From f65508690d92301e0f6e27ce76a46d28780272ea Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 24 Jan 2025 16:33:54 +0100 Subject: [PATCH 001/105] Update publish-winget action to use Komac directly (#2228) ## Changes For the most recent release, I had to re-run the "publish-winget" action a couple of times before it passed. The underlying issue that causes the failure should be solved by the latest version of the action, but upon inspection of the latest version, I found that it always installs the latest version of [Komac](https://github.com/russellbanks/Komac). To both fix the issue and lock this down further, I updated our action to call Komac directly instead of relying on a separate action to do this for us. ## Tests Successful run in https://github.com/databricks/cli/actions/runs/12951529979. --- .github/workflows/publish-winget.yml | 68 +++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml index eb9a72eda..cbd24856b 100644 --- a/.github/workflows/publish-winget.yml +++ b/.github/workflows/publish-winget.yml @@ -10,19 +10,65 @@ on: jobs: publish-to-winget-pkgs: runs-on: - group: databricks-protected-runner-group - labels: windows-server-latest + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco environment: release steps: - - uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # v2 - with: - identifier: Databricks.DatabricksCLI - installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases - token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }} - fork-user: eng-dev-ecosystem-bot + - name: Checkout repository and submodules + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - # Use the tag from the input, or the ref name if the input is not provided. - # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command. - release-tag: ${{ inputs.tag || github.ref_name }} + # When updating the version of komac, make sure to update the checksum in the next step. + # Find both at https://github.com/russellbanks/Komac/releases. + - name: Download komac binary + run: | + curl -s -L -o $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz https://github.com/russellbanks/Komac/releases/download/v2.9.0/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz + + - name: Verify komac binary + run: | + echo "d07a12831ad5418fee715488542a98ce3c0e591d05c850dd149fe78432be8c4c $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c - + + - name: Untar komac binary to temporary path + run: | + mkdir -p $RUNNER_TEMP/komac + tar -xzf $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz -C $RUNNER_TEMP/komac + + - name: Add komac to PATH + run: echo "$RUNNER_TEMP/komac" >> $GITHUB_PATH + + - name: Confirm komac version + run: komac --version + + # Use the tag from the input, or the ref name if the input is not provided. + # The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command. + - name: Strip "v" prefix from version + id: strip_version + run: echo "version=$(echo ${{ inputs.tag || github.ref_name }} | sed 's/^v//')" >> "$GITHUB_OUTPUT" + + - name: Get URLs of signed Windows binaries + id: get_windows_urls + run: | + urls=$( + gh api https://api.github.com/repos/databricks/cli/releases/tags/${{ inputs.tag || github.ref_name }} | \ + jq -r .assets[].browser_download_url | \ + grep -E '_windows_.*-signed\.zip$' | \ + tr '\n' ' ' + ) + if [ -z "$urls" ]; then + echo "No signed Windows binaries found" >&2 + exit 1 + fi + echo "urls=$urls" >> "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to Winget + run: | + komac update Databricks.DatabricksCLI \ + --version ${{ steps.strip_version.outputs.version }} \ + --submit \ + --urls ${{ steps.get_windows_urls.outputs.urls }} \ + env: + KOMAC_FORK_OWNER: eng-dev-ecosystem-bot + GITHUB_TOKEN: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }} From 468660dc45bd1deac4d37fb914d4a6224aa1a27e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 24 Jan 2025 16:53:06 +0100 Subject: [PATCH 002/105] Add an acc test covering failures when reading .git (#2223) ## Changes - New test covering failures in reading .git. One case results in error, some result in warning (not shown). - New helper withdir runs commands in a subdirectory. ## Tests New acceptance test. --- .../bundle/git-permerror/databricks.yml | 2 + acceptance/bundle/git-permerror/output.txt | 78 +++++++++++++++++++ acceptance/bundle/git-permerror/script | 25 ++++++ acceptance/bundle/git-permerror/test.toml | 5 ++ acceptance/script.prepare | 11 +++ 5 files changed, 121 insertions(+) create mode 100644 acceptance/bundle/git-permerror/databricks.yml create mode 100644 acceptance/bundle/git-permerror/output.txt create mode 100644 acceptance/bundle/git-permerror/script create mode 100644 acceptance/bundle/git-permerror/test.toml diff --git a/acceptance/bundle/git-permerror/databricks.yml b/acceptance/bundle/git-permerror/databricks.yml new file mode 100644 index 000000000..83e0acda8 --- /dev/null +++ b/acceptance/bundle/git-permerror/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: git-permerror diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt new file mode 100644 index 000000000..2b52134ab --- /dev/null +++ b/acceptance/bundle/git-permerror/output.txt @@ -0,0 +1,78 @@ +=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case. + +>>> chmod 000 .git + +>>> $CLI bundle validate +Error: unable to load repository specific gitconfig: open config: permission denied + +Name: git-permerror +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/git-permerror/default + +Found 1 error + +Exit code: 1 + +>>> $CLI bundle validate -o json +Error: unable to load repository specific gitconfig: open config: permission denied + + +Exit code: 1 +{ + "bundle_root_path": ".", + "inferred": true +} + +>>> withdir subdir/a/b $CLI bundle validate -o json +Error: unable to load repository specific gitconfig: open config: permission denied + + +Exit code: 1 +{ + "bundle_root_path": ".", + "inferred": true +} + + +=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case. + +>>> chmod 000 .git/HEAD + +>>> $CLI bundle validate -o json +{ + "bundle_root_path": ".", + "inferred": true +} + +>>> withdir subdir/a/b $CLI bundle validate -o json +{ + "bundle_root_path": ".", + "inferred": true +} + + +=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case. + +>>> chmod 000 .git/config + +>>> $CLI bundle validate -o json +Error: unable to load repository specific gitconfig: open config: permission denied + + +Exit code: 1 +{ + "bundle_root_path": ".", + "inferred": true +} + +>>> withdir subdir/a/b $CLI bundle validate -o json +Error: unable to load repository specific gitconfig: open config: permission denied + + +Exit code: 1 +{ + "bundle_root_path": ".", + "inferred": true +} diff --git a/acceptance/bundle/git-permerror/script b/acceptance/bundle/git-permerror/script new file mode 100644 index 000000000..782cbf5bc --- /dev/null +++ b/acceptance/bundle/git-permerror/script @@ -0,0 +1,25 @@ +mkdir myrepo +cd myrepo +cp ../databricks.yml . +git-repo-init +mkdir -p subdir/a/b + +printf "=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case.\n" +trace chmod 000 .git +errcode trace $CLI bundle validate +errcode trace $CLI bundle validate -o json | jq .bundle.git +errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git + +printf "\n\n=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case.\n" +chmod 700 .git +trace chmod 000 .git/HEAD +errcode trace $CLI bundle validate -o json | jq .bundle.git +errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git + +printf "\n\n=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case.\n" +chmod 666 .git/HEAD +trace chmod 000 .git/config +errcode trace $CLI bundle validate -o json | jq .bundle.git +errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git + +rm -fr .git diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml new file mode 100644 index 000000000..3f96e551c --- /dev/null +++ b/acceptance/bundle/git-permerror/test.toml @@ -0,0 +1,5 @@ +Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct" + +[GOOS] +# This test relies on chmod which does not work on Windows +windows = false diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 87910654d..b814a1260 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -47,3 +47,14 @@ title() { local label="$1" printf "\n=== %s" "$label" } + +withdir() { + local dir="$1" + shift + local orig_dir="$(pwd)" + cd "$dir" || return $? + "$@" + local exit_code=$? + cd "$orig_dir" || return $? + return $exit_code +} From b3d98fe66664cb85c750364afce9b1ea0785417f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 08:45:09 +0100 Subject: [PATCH 003/105] acc: Print replacements on error and rm duplicates (#2230) ## Changes - File comparison files in acceptance test, print the contents of all applied replacements. Do it once per test. - Remove duplicate entries in replacement list. ## Tests Manually, change out files of existing test, you'll get this printed once, after first assertion: ``` acceptance_test.go:307: Available replacements: REPL /Users/denis\.bilenko/work/cli/acceptance/build/databricks => $$CLI REPL /private/var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAccept598522733/001 => $$TMPHOME ... ``` --- acceptance/acceptance_test.go | 17 ++++++++++++++--- libs/testdiff/replacement.go | 6 +++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a1c41c5e6..11fd3f2ee 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -228,9 +228,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont formatOutput(out, err) require.NoError(t, out.Close()) + printedRepls := false + // Compare expected outputs for relPath := range outputs { - doComparison(t, repls, dir, tmpDir, relPath) + doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } // Make sure there are not unaccounted for new files @@ -245,12 +247,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: - doComparison(t, repls, dir, tmpDir, relPath) + doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } } } -func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string) { +func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { pathRef := filepath.Join(dirRef, relPath) pathNew := filepath.Join(dirNew, relPath) bufRef, okRef := readIfExists(t, pathRef) @@ -295,6 +297,15 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN t.Logf("Overwriting existing output file: %s", relPath) testutil.WriteFile(t, pathRef, valueNew) } + + if !equal && printedRepls != nil && !*printedRepls { + *printedRepls = true + var items []string + for _, item := range repls.Repls { + items = append(items, fmt.Sprintf("REPL %s => %s", item.Old, item.New)) + } + t.Log("Available replacements:\n" + strings.Join(items, "\n")) + } } // Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ... diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 865192662..b512374a3 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -76,7 +76,11 @@ func (r *ReplacementsContext) Set(old, new string) { if err == nil { encodedOld, err := json.Marshal(old) if err == nil { - r.appendLiteral(trimQuotes(string(encodedOld)), trimQuotes(string(encodedNew))) + encodedStrNew := trimQuotes(string(encodedNew)) + encodedStrOld := trimQuotes(string(encodedOld)) + if encodedStrNew != new || encodedStrOld != old { + r.appendLiteral(encodedStrOld, encodedStrNew) + } } } From 82b0dd36d682b1b11260e05e8a5c6aeccb65c255 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 09:17:22 +0100 Subject: [PATCH 004/105] Add acceptance/selftest, showcasing basic features (#2229) Also make TestInprocessMode use this test. --- acceptance/README.md | 2 ++ acceptance/acceptance_test.go | 7 +------ acceptance/selftest/out.hello.txt | 1 + acceptance/selftest/output.txt | 30 ++++++++++++++++++++++++++++++ acceptance/selftest/script | 21 +++++++++++++++++++++ acceptance/selftest/test.toml | 11 +++++++++++ 6 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 acceptance/selftest/out.hello.txt create mode 100644 acceptance/selftest/output.txt create mode 100644 acceptance/selftest/script create mode 100644 acceptance/selftest/test.toml diff --git a/acceptance/README.md b/acceptance/README.md index 42a37d253..75ac1d5fc 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -17,3 +17,5 @@ For more complex tests one can also use: - `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script. - `trace` helper: prints the arguments before executing the command. - custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`. + +See [selftest](./selftest) for a toy test. diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 11fd3f2ee..6b70c6a7f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -60,12 +60,7 @@ func TestInprocessMode(t *testing.T) { if InprocessMode { t.Skip("Already tested by TestAccept") } - if runtime.GOOS == "windows" { - // - catalogs A catalog is the first layer of Unity Catalog’s three-level namespace. - // + catalogs A catalog is the first layer of Unity Catalog�s three-level namespace. - t.Skip("Fails on CI on unicode characters") - } - require.NotZero(t, testAccept(t, true, "help")) + require.Equal(t, 1, testAccept(t, true, "selftest")) } func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { diff --git a/acceptance/selftest/out.hello.txt b/acceptance/selftest/out.hello.txt new file mode 100644 index 000000000..e427984d4 --- /dev/null +++ b/acceptance/selftest/out.hello.txt @@ -0,0 +1 @@ +HELLO diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt new file mode 100644 index 000000000..d1830e01f --- /dev/null +++ b/acceptance/selftest/output.txt @@ -0,0 +1,30 @@ +=== Capturing STDERR +>>> python3 -c import sys; sys.stderr.write("STDERR\n") +STDERR + +=== Capturing STDOUT +>>> python3 -c import sys; sys.stderr.write("STDOUT\n") +STDOUT + +=== Capturing exit code +>>> errcode python3 -c raise SystemExit(5) + +Exit code: 5 + +=== Capturing exit code (alt) +>>> python3 -c raise SystemExit(7) + +Exit code: 7 + +=== Capturing pwd +>>> python3 -c import os; print(os.getcwd()) +$TMPDIR + +=== Capturing subdir +>>> mkdir -p subdir/a/b/c + +>>> withdir subdir/a/b/c python3 -c import os; print(os.getcwd()) +$TMPDIR/subdir/a/b/c + +=== Custom output files - everything starting with out is captured and compared +>>> echo HELLO diff --git a/acceptance/selftest/script b/acceptance/selftest/script new file mode 100644 index 000000000..89201d925 --- /dev/null +++ b/acceptance/selftest/script @@ -0,0 +1,21 @@ +printf "=== Capturing STDERR" +trace python3 -c 'import sys; sys.stderr.write("STDERR\n")' + +printf "\n=== Capturing STDOUT" +trace python3 -c 'import sys; sys.stderr.write("STDOUT\n")' + +printf "\n=== Capturing exit code" +trace errcode python3 -c 'raise SystemExit(5)' + +printf "\n=== Capturing exit code (alt)" +errcode trace python3 -c 'raise SystemExit(7)' + +printf "\n=== Capturing pwd" +trace python3 -c 'import os; print(os.getcwd())' + +printf "\n=== Capturing subdir" +trace mkdir -p subdir/a/b/c +trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' | sed 's/\\/\//g' + +printf "\n=== Custom output files - everything starting with out is captured and compared" +trace echo HELLO > out.hello.txt diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml new file mode 100644 index 000000000..d867a4fd7 --- /dev/null +++ b/acceptance/selftest/test.toml @@ -0,0 +1,11 @@ +# Badness = "Brief description of what's wrong with the test output, if anything" + +#[GOOS] +# Disable on Windows +#windows = false + +# Disable on Mac +#mac = false + +# Disable on Linux +#linux = false From 1cb32eca907872556b94890e3666ffac531a0f29 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 10:11:06 +0100 Subject: [PATCH 005/105] acc: Support custom replacements (#2231) ## Changes - Ability to extend a list of replacements via test.toml - Modify selftest to both demo this feature and to get rid of sed on Windows. ## Tests Acceptance tests. I'm also using it https://github.com/databricks/cli/pull/2213 for things like pid. --- acceptance/acceptance_test.go | 1 + acceptance/config_test.go | 5 +++++ acceptance/selftest/output.txt | 5 +++++ acceptance/selftest/script | 7 ++++++- acceptance/selftest/test.toml | 9 +++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6b70c6a7f..e48bd9908 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -190,6 +190,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } repls.SetPathWithParents(tmpDir, "$TMPDIR") + repls.Repls = append(repls.Repls, config.Repls...) scriptContents := readMergedScriptContents(t, dir) testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents) diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 49dce06ba..41866c4a7 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/BurntSushi/toml" + "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" ) @@ -24,6 +25,10 @@ type TestConfig struct { // Which OSes the test is enabled on. Each string is compared against runtime.GOOS. // If absent, default to true. GOOS map[string]bool + + // List of additional replacements to apply on this test. + // Old is a regexp, New is a replacement expression. + Repls []testdiff.Replacement } // FindConfig finds the closest config file. diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt index d1830e01f..9fdfbc1e7 100644 --- a/acceptance/selftest/output.txt +++ b/acceptance/selftest/output.txt @@ -28,3 +28,8 @@ $TMPDIR/subdir/a/b/c === Custom output files - everything starting with out is captured and compared >>> echo HELLO + +=== Custom regex can be specified in [[Repl]] section +1234 +CUSTOM_NUMBER_REGEX +123456 diff --git a/acceptance/selftest/script b/acceptance/selftest/script index 89201d925..665726167 100644 --- a/acceptance/selftest/script +++ b/acceptance/selftest/script @@ -15,7 +15,12 @@ trace python3 -c 'import os; print(os.getcwd())' printf "\n=== Capturing subdir" trace mkdir -p subdir/a/b/c -trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' | sed 's/\\/\//g' +trace withdir subdir/a/b/c python3 -c 'import os; print(os.getcwd())' printf "\n=== Custom output files - everything starting with out is captured and compared" trace echo HELLO > out.hello.txt + +printf "\n=== Custom regex can be specified in [[Repl]] section\n" +echo 1234 +echo 12345 +echo 123456 diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml index d867a4fd7..9607ec5df 100644 --- a/acceptance/selftest/test.toml +++ b/acceptance/selftest/test.toml @@ -9,3 +9,12 @@ # Disable on Linux #linux = false + +[[Repls]] +Old = '\b[0-9]{5}\b' +New = "CUSTOM_NUMBER_REGEX" + +[[Repls]] +# Fix path with reverse slashes in the output for Windows. +Old = '\$TMPDIR\\subdir\\a\\b\\c' +New = '$$TMPDIR/subdir/a/b/c' From 6e8f0ea8afeecf86c3edd42d0ccccbacf25353d2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 11:33:16 +0100 Subject: [PATCH 006/105] CI: Move ruff to 'lint' job (#2232) This is where it belongs and also there is no need to run it 3 times. --- .github/workflows/push.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c3a314d69..2a8a68862 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -60,12 +60,6 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0 - - name: Run ruff - uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1 - with: - version: "0.9.1" - args: "format --check" - - name: Set go env run: | echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV @@ -80,7 +74,7 @@ jobs: - name: Run tests with coverage run: make cover - golangci: + linters: needs: cleanups name: lint runs-on: ubuntu-latest @@ -105,6 +99,11 @@ jobs: with: version: v1.63.4 args: --timeout=15m + - name: Run ruff + uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1 + with: + version: "0.9.1" + args: "format --check" validate-bundle-schema: needs: cleanups From b7dd70b8b3c59d64ab7b54805750b532b0d75f07 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 13:22:40 +0100 Subject: [PATCH 007/105] acc: Add a couple of error tests for 'bundle init' (#2233) This captures how we log errors related to subprocess run and what does the output look like. --- acceptance/bundle/templates/wrong-path/output.txt | 3 +++ acceptance/bundle/templates/wrong-path/script | 2 ++ acceptance/bundle/templates/wrong-path/test.toml | 1 + acceptance/bundle/templates/wrong-url/output.txt | 5 +++++ acceptance/bundle/templates/wrong-url/script | 2 ++ acceptance/bundle/templates/wrong-url/test.toml | 7 +++++++ 6 files changed, 20 insertions(+) create mode 100644 acceptance/bundle/templates/wrong-path/output.txt create mode 100644 acceptance/bundle/templates/wrong-path/script create mode 100644 acceptance/bundle/templates/wrong-path/test.toml create mode 100644 acceptance/bundle/templates/wrong-url/output.txt create mode 100644 acceptance/bundle/templates/wrong-url/script create mode 100644 acceptance/bundle/templates/wrong-url/test.toml diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates/wrong-path/output.txt new file mode 100644 index 000000000..0a6fdfc84 --- /dev/null +++ b/acceptance/bundle/templates/wrong-path/output.txt @@ -0,0 +1,3 @@ +Error: not a bundle template: expected to find a template schema file at databricks_template_schema.json + +Exit code: 1 diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates/wrong-path/script new file mode 100644 index 000000000..00c05927a --- /dev/null +++ b/acceptance/bundle/templates/wrong-path/script @@ -0,0 +1,2 @@ +export NO_COLOR=1 +$CLI bundle init /DOES/NOT/EXIST diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates/wrong-path/test.toml new file mode 100644 index 000000000..4bbcb5100 --- /dev/null +++ b/acceptance/bundle/templates/wrong-path/test.toml @@ -0,0 +1 @@ +Badness = 'The error message should include full path: "expected to find a template schema file at databricks_template_schema.json"' diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates/wrong-url/output.txt new file mode 100644 index 000000000..b78cf4b68 --- /dev/null +++ b/acceptance/bundle/templates/wrong-url/output.txt @@ -0,0 +1,5 @@ +Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world $TMPDIR_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '$TMPDIR_GPARENT/world-123456'... +fatal: unable to access 'https://invalid-domain-123.databricks.com/hello/world/': Could not resolve host: invalid-domain-123.databricks.com + + +Exit code: 1 diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates/wrong-url/script new file mode 100644 index 000000000..e9bc0f4f6 --- /dev/null +++ b/acceptance/bundle/templates/wrong-url/script @@ -0,0 +1,2 @@ +export NO_COLOR=1 +$CLI bundle init https://invalid-domain-123.databricks.com/hello/world diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates/wrong-url/test.toml new file mode 100644 index 000000000..0bb24bf1a --- /dev/null +++ b/acceptance/bundle/templates/wrong-url/test.toml @@ -0,0 +1,7 @@ +[[Repls]] +Old = '\\' +New = '/' + +[[Repls]] +Old = '/world-[0-9]+' +New = '/world-123456' From 4595c6f1b5d4890b6c9a1e13257319d52954dfe5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:11:07 +0100 Subject: [PATCH 008/105] Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 (#2238) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.55.0 to 0.56.1.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.56.1

Bug Fixes

  • Do not send query parameters when set to zero value (#1136).

v0.56.0

Bug Fixes

  • Support Query parameters for all HTTP operations (#1124).

Internal Changes

  • Add download target to MakeFile (#1125).
  • Delete examples/mocking module (#1126).
  • Scope the traversing directory in the Recursive list workspace test (#1120).

API Changes:

... (truncated)

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.56.1

Bug Fixes

  • Do not send query parameters when set to zero value (#1136).

[Release] Release v0.56.0

Bug Fixes

  • Support Query parameters for all HTTP operations (#1124).

Internal Changes

  • Add download target to MakeFile (#1125).
  • Delete examples/mocking module (#1126).
  • Scope the traversing directory in the Recursive list workspace test (#1120).

API Changes:

... (truncated)

Commits
  • bf617bb [Release] Release v0.56.1 (#1137)
  • 18cebf1 [Fix] Do not send query parameters when set to zero value (#1136)
  • 28ff749 [Release] Release v0.56.0 (#1134)
  • 1134540 [Internal] Add download target to MakeFile (#1125)
  • e079db9 [Fix] Support Query parameters for all HTTP operations (#1124)
  • 1045fb9 [Internal] Delete examples/mocking module (#1126)
  • 914ab6b [Internal] Scope the traversing directory in the Recursive list workspace tes...
  • See full diff in compare view

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.55.0&new-version=0.56.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- .codegen/service.go.tmpl | 20 +- .gitattributes | 1 + bundle/deploy/terraform/convert_test.go | 4 +- .../convert_model_serving_endpoint_test.go | 2 +- .../internal/schema/annotations_openapi.yml | 367 +++++++++++------- .../schema/annotations_openapi_overrides.yml | 11 + bundle/schema/jsonschema.json | 175 +++++---- .../custom-app-integration.go | 1 + cmd/api/api.go | 2 +- .../access-control/access-control.go | 109 ++++++ cmd/workspace/cmd.go | 2 + cmd/workspace/providers/providers.go | 4 +- cmd/workspace/recipients/recipients.go | 96 ++--- .../serving-endpoints/serving-endpoints.go | 111 +++++- go.mod | 2 +- go.sum | 4 +- integration/cmd/sync/sync_test.go | 2 +- libs/filer/files_client.go | 4 +- libs/filer/workspace_files_client.go | 5 +- .../workspace_files_extensions_client_test.go | 2 +- libs/git/info.go | 1 + 22 files changed, 588 insertions(+), 339 deletions(-) create mode 100755 cmd/workspace/access-control/access-control.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index dfe78790a..588cf9d63 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -779817ed8d63031f5ea761fbd25ee84f38feec0d \ No newline at end of file +0be1b914249781b5e903b7676fd02255755bc851 \ No newline at end of file diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index 0c9fa089a..2f4987b13 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -109,16 +109,19 @@ var {{.CamelName}}Overrides []func( {{- end }} ) +{{- $excludeFromJson := list "http-request"}} + func new{{.PascalName}}() *cobra.Command { cmd := &cobra.Command{} + {{- $canUseJson := and .CanUseJson (not (in $excludeFromJson .KebabName )) -}} {{- if .Request}} var {{.CamelName}}Req {{.Service.Package.Name}}.{{.Request.PascalName}} {{- if .RequestBodyField }} {{.CamelName}}Req.{{.RequestBodyField.PascalName}} = &{{.Service.Package.Name}}.{{.RequestBodyField.Entity.PascalName}}{} {{- end }} - {{- if .CanUseJson}} + {{- if $canUseJson}} var {{.CamelName}}Json flags.JsonFlag {{- end}} {{- end}} @@ -135,7 +138,7 @@ func new{{.PascalName}}() *cobra.Command { {{- $request = .RequestBodyField.Entity -}} {{- end -}} {{if $request }}// TODO: short flags - {{- if .CanUseJson}} + {{- if $canUseJson}} cmd.Flags().Var(&{{.CamelName}}Json, "json", `either inline JSON string or @path/to/file.json with request body`) {{- end}} {{$method := .}} @@ -177,7 +180,7 @@ func new{{.PascalName}}() *cobra.Command { {{- $hasRequiredArgs := and (not $hasIdPrompt) $hasPosArgs -}} {{- $hasSingleRequiredRequestBodyFieldWithPrompt := and (and $hasIdPrompt $request) (eq 1 (len $request.RequiredRequestBodyFields)) -}} {{- $onlyPathArgsRequiredAsPositionalArguments := and $request (eq (len .RequiredPositionalArguments) (len $request.RequiredPathFields)) -}} - {{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and .CanUseJson (or $request.HasRequiredRequestBodyFields )) -}} + {{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and $canUseJson (or $request.HasRequiredRequestBodyFields )) -}} {{- $hasCustomArgHandler := or $hasRequiredArgs $hasDifferentArgsWithJsonFlag -}} {{- $atleastOneArgumentWithDescription := false -}} @@ -239,7 +242,7 @@ func new{{.PascalName}}() *cobra.Command { ctx := cmd.Context() {{if .Service.IsAccounts}}a := root.AccountClient(ctx){{else}}w := root.WorkspaceClient(ctx){{end}} {{- if .Request }} - {{ if .CanUseJson }} + {{ if $canUseJson }} if cmd.Flags().Changed("json") { diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req{{ if .RequestBodyField }}.{{.RequestBodyField.PascalName}}{{ end }}) if diags.HasError() { @@ -255,7 +258,7 @@ func new{{.PascalName}}() *cobra.Command { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") }{{- end}} {{- if $hasPosArgs }} - {{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else { + {{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} else { {{- end}} {{- if $hasIdPrompt}} if len(args) == 0 { @@ -279,9 +282,9 @@ func new{{.PascalName}}() *cobra.Command { {{$method := .}} {{- range $arg, $field := .RequiredPositionalArguments}} - {{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt)}} + {{- template "args-scan" (dict "Arg" $arg "Field" $field "Method" $method "HasIdPrompt" $hasIdPrompt "ExcludeFromJson" $excludeFromJson)}} {{- end -}} - {{- if and .CanUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} + {{- if and $canUseJson $hasSingleRequiredRequestBodyFieldWithPrompt }} } {{- end}} @@ -392,7 +395,8 @@ func new{{.PascalName}}() *cobra.Command { {{- $method := .Method -}} {{- $arg := .Arg -}} {{- $hasIdPrompt := .HasIdPrompt -}} - {{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $method.CanUseJson) }} + {{ $canUseJson := and $method.CanUseJson (not (in .ExcludeFromJson $method.KebabName)) }} + {{- $optionalIfJsonIsUsed := and (not $hasIdPrompt) (and $field.IsRequestBodyField $canUseJson) }} {{- if $optionalIfJsonIsUsed }} if !cmd.Flags().Changed("json") { {{- end }} diff --git a/.gitattributes b/.gitattributes index 0a8ddf3cb..ebe94ed8e 100755 --- a/.gitattributes +++ b/.gitattributes @@ -31,6 +31,7 @@ cmd/account/users/users.go linguist-generated=true cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true cmd/account/workspaces/workspaces.go linguist-generated=true +cmd/workspace/access-control/access-control.go linguist-generated=true cmd/workspace/aibi-dashboard-embedding-access-policy/aibi-dashboard-embedding-access-policy.go linguist-generated=true cmd/workspace/aibi-dashboard-embedding-approved-domains/aibi-dashboard-embedding-approved-domains.go linguist-generated=true cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index ffe55db71..afc1fb22a 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -419,7 +419,7 @@ func TestBundleToTerraformModelServing(t *testing.T) { src := resources.ModelServingEndpoint{ CreateServingEndpoint: &serving.CreateServingEndpoint{ Name: "name", - Config: serving.EndpointCoreConfigInput{ + Config: &serving.EndpointCoreConfigInput{ ServedModels: []serving.ServedModelInput{ { ModelName: "model_name", @@ -474,7 +474,7 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) { // and as such observed the `omitempty` tag. // The new method leverages [dyn.Value] where any field that is not // explicitly set is not part of the value. - Config: serving.EndpointCoreConfigInput{ + Config: &serving.EndpointCoreConfigInput{ ServedModels: []serving.ServedModelInput{ { ModelName: "model_name", diff --git a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go index d46350bb7..98cf2dc22 100644 --- a/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_model_serving_endpoint_test.go @@ -17,7 +17,7 @@ func TestConvertModelServingEndpoint(t *testing.T) { src := resources.ModelServingEndpoint{ CreateServingEndpoint: &serving.CreateServingEndpoint{ Name: "name", - Config: serving.EndpointCoreConfigInput{ + Config: &serving.EndpointCoreConfigInput{ ServedModels: []serving.ServedModelInput{ { ModelName: "model_name", diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 8ff5c9253..d5a9bf69e 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -353,12 +353,12 @@ github.com/databricks/cli/bundle/config/resources.MlflowModel: github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "ai_gateway": "description": |- - The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now. + The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. "config": "description": |- The core config of the serving endpoint. "name": - "description": | + "description": |- The name of the serving endpoint. This field is required and must be unique across a Databricks workspace. An endpoint name can consist of alphanumeric characters, dashes, and underscores. "rate_limits": @@ -1974,6 +1974,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask: Parameters passed to the main method. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. + "run_as_repl": + "description": |- + Deprecated. A value of `false` is no longer supported. github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask: "parameters": "description": |- @@ -2684,27 +2687,36 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScd github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig: "ai21labs_api_key": "description": |- - The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. + The Databricks secret key reference for an AI21 Labs API key. If you + prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. + You must provide an API key using one of the following fields: + `ai21labs_api_key` or `ai21labs_api_key_plaintext`. "ai21labs_api_key_plaintext": "description": |- - An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. + An AI21 Labs API key provided as a plaintext string. If you prefer to + reference your key using Databricks Secrets, see `ai21labs_api_key`. You + must provide an API key using one of the following fields: + `ai21labs_api_key` or `ai21labs_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig: "guardrails": "description": |- Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. "inference_table_config": "description": |- - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. + Configuration for payload logging using inference tables. + Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. "rate_limits": "description": |- Configuration for rate limits which can be set to limit endpoint traffic. "usage_tracking_config": "description": |- - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. + Configuration to enable usage tracking using system tables. + These tables allow you to monitor operational usage on endpoints and their associated costs. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters: "invalid_keywords": "description": |- - List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content. + List of invalid keywords. + AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content. "pii": "description": |- Configuration for guardrail PII filter. @@ -2713,15 +2725,14 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame Indicates whether the safety filter is enabled. "valid_topics": "description": |- - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. + The list of allowed topics. + Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior: "behavior": "description": |- - Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned. + Configuration for input guardrail filters. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior: "_": - "description": |- - Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned. "enum": - |- NONE @@ -2737,30 +2748,32 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails: github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig: "catalog_name": "description": |- - The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name. + The name of the catalog in Unity Catalog. Required when enabling inference tables. + NOTE: On update, you have to disable inference table first in order to change the catalog name. "enabled": "description": |- Indicates whether the inference table is enabled. "schema_name": "description": |- - The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name. + The name of the schema in Unity Catalog. Required when enabling inference tables. + NOTE: On update, you have to disable inference table first in order to change the schema name. "table_name_prefix": "description": |- - The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name. + The prefix of the table in Unity Catalog. + NOTE: On update, you have to disable inference table first in order to change the prefix name. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit: "calls": "description": |- Used to specify how many calls are allowed for a key within the renewal_period. "key": "description": |- - Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, + with 'endpoint' being the default if not specified. "renewal_period": "description": |- Renewal period field for a rate limit. Currently, only 'minute' is supported. github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey: "_": - "description": |- - Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. "enum": - |- user @@ -2768,8 +2781,6 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey: endpoint github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod: "_": - "description": |- - Renewal period field for a rate limit. Currently, only 'minute' is supported. "enum": - |- minute @@ -2780,26 +2791,43 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingCo github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig: "aws_access_key_id": "description": |- - The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + The Databricks secret key reference for an AWS access key ID with + permissions to interact with Bedrock services. If you prefer to paste + your API key directly, see `aws_access_key_id_plaintext`. You must provide an API + key using one of the following fields: `aws_access_key_id` or + `aws_access_key_id_plaintext`. "aws_access_key_id_plaintext": "description": |- - An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + An AWS access key ID with permissions to interact with Bedrock services + provided as a plaintext string. If you prefer to reference your key using + Databricks Secrets, see `aws_access_key_id`. You must provide an API key + using one of the following fields: `aws_access_key_id` or + `aws_access_key_id_plaintext`. "aws_region": "description": |- The AWS region to use. Bedrock has to be enabled there. "aws_secret_access_key": "description": |- - The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + The Databricks secret key reference for an AWS secret access key paired + with the access key ID, with permissions to interact with Bedrock + services. If you prefer to paste your API key directly, see + `aws_secret_access_key_plaintext`. You must provide an API key using one + of the following fields: `aws_secret_access_key` or + `aws_secret_access_key_plaintext`. "aws_secret_access_key_plaintext": "description": |- - An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + An AWS secret access key paired with the access key ID, with permissions + to interact with Bedrock services provided as a plaintext string. If you + prefer to reference your key using Databricks Secrets, see + `aws_secret_access_key`. You must provide an API key using one of the + following fields: `aws_secret_access_key` or + `aws_secret_access_key_plaintext`. "bedrock_provider": "description": |- - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. + The underlying provider in Amazon Bedrock. Supported values (case + insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider: "_": - "description": |- - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. "enum": - |- anthropic @@ -2812,10 +2840,16 @@ github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedro github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig: "anthropic_api_key": "description": |- - The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. + The Databricks secret key reference for an Anthropic API key. If you + prefer to paste your API key directly, see `anthropic_api_key_plaintext`. + You must provide an API key using one of the following fields: + `anthropic_api_key` or `anthropic_api_key_plaintext`. "anthropic_api_key_plaintext": "description": |- - The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. + The Anthropic API key provided as a plaintext string. If you prefer to + reference your key using Databricks Secrets, see `anthropic_api_key`. You + must provide an API key using one of the following fields: + `anthropic_api_key` or `anthropic_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput: "catalog_name": "description": |- @@ -2831,42 +2865,58 @@ github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput: The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. github.com/databricks/databricks-sdk-go/service/serving.CohereConfig: "cohere_api_base": - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + "description": |- + This is an optional field to provide a customized base URL for the Cohere + API. If left unspecified, the standard Cohere base URL is used. "cohere_api_key": "description": |- - The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. + The Databricks secret key reference for a Cohere API key. If you prefer + to paste your API key directly, see `cohere_api_key_plaintext`. You must + provide an API key using one of the following fields: `cohere_api_key` or + `cohere_api_key_plaintext`. "cohere_api_key_plaintext": "description": |- - The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. + The Cohere API key provided as a plaintext string. If you prefer to + reference your key using Databricks Secrets, see `cohere_api_key`. You + must provide an API key using one of the following fields: + `cohere_api_key` or `cohere_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig: "databricks_api_token": - "description": | - The Databricks secret key reference for a Databricks API token that corresponds to a user or service - principal with Can Query access to the model serving endpoint pointed to by this external model. - If you prefer to paste your API key directly, see `databricks_api_token_plaintext`. - You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. + "description": |- + The Databricks secret key reference for a Databricks API token that + corresponds to a user or service principal with Can Query access to the + model serving endpoint pointed to by this external model. If you prefer + to paste your API key directly, see `databricks_api_token_plaintext`. You + must provide an API key using one of the following fields: + `databricks_api_token` or `databricks_api_token_plaintext`. "databricks_api_token_plaintext": - "description": | - The Databricks API token that corresponds to a user or service - principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string. - If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`. - You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. + "description": |- + The Databricks API token that corresponds to a user or service principal + with Can Query access to the model serving endpoint pointed to by this + external model provided as a plaintext string. If you prefer to reference + your key using Databricks Secrets, see `databricks_api_token`. You must + provide an API key using one of the following fields: + `databricks_api_token` or `databricks_api_token_plaintext`. "databricks_workspace_url": - "description": | - The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model. + "description": |- + The URL of the Databricks workspace containing the model serving endpoint + pointed to by this external model. github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput: "auto_capture_config": "description": |- Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. + Note: this field is deprecated for creating new provisioned throughput endpoints, + or updating existing provisioned throughput endpoints that never have inference table configured; + in these cases please use AI Gateway to manage inference tables. "served_entities": "description": |- - A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities. + The list of served entities under the serving endpoint config. "served_models": "description": |- - (Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models. + (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. "traffic_config": "description": |- - The traffic config defining how invocations to the serving endpoint should be routed. + The traffic configuration associated with the serving endpoint config. github.com/databricks/databricks-sdk-go/service/serving.EndpointTag: "key": "description": |- @@ -2903,17 +2953,13 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModel: "description": |- PaLM Config. Only required if the provider is 'palm'. "provider": - "description": | - The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', - 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.", + "description": |- + The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'. "task": "description": |- The task type of the external model. github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider: "_": - "description": | - The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', - 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.", "enum": - |- ai21labs @@ -2934,70 +2980,114 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider: github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig: "private_key": "description": |- - The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext` + The Databricks secret key reference for a private key for the service + account which has access to the Google Cloud Vertex AI Service. See [Best + practices for managing service account keys]. If you prefer to paste your + API key directly, see `private_key_plaintext`. You must provide an API + key using one of the following fields: `private_key` or + `private_key_plaintext` + + [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys "private_key_plaintext": "description": |- - The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`. + The private key for the service account which has access to the Google + Cloud Vertex AI Service provided as a plaintext secret. See [Best + practices for managing service account keys]. If you prefer to reference + your key using Databricks Secrets, see `private_key`. You must provide an + API key using one of the following fields: `private_key` or + `private_key_plaintext`. + + [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys "project_id": "description": |- - This is the Google Cloud project id that the service account is associated with. + This is the Google Cloud project id that the service account is + associated with. "region": "description": |- - This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions. + This is the region for the Google Cloud Vertex AI Service. See [supported + regions] for more details. Some models are only available in specific + regions. + + [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig: + "_": + "description": |- + Configs needed to create an OpenAI model route. "microsoft_entra_client_id": - "description": | - This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID. + "description": |- + This field is only required for Azure AD OpenAI and is the Microsoft + Entra Client ID. "microsoft_entra_client_secret": - "description": | - The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication. - If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`. - You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + "description": |- + The Databricks secret key reference for a client secret used for + Microsoft Entra ID authentication. If you prefer to paste your client + secret directly, see `microsoft_entra_client_secret_plaintext`. You must + provide an API key using one of the following fields: + `microsoft_entra_client_secret` or + `microsoft_entra_client_secret_plaintext`. "microsoft_entra_client_secret_plaintext": - "description": | - The client secret used for Microsoft Entra ID authentication provided as a plaintext string. - If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`. - You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + "description": |- + The client secret used for Microsoft Entra ID authentication provided as + a plaintext string. If you prefer to reference your key using Databricks + Secrets, see `microsoft_entra_client_secret`. You must provide an API key + using one of the following fields: `microsoft_entra_client_secret` or + `microsoft_entra_client_secret_plaintext`. "microsoft_entra_tenant_id": - "description": | - This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID. + "description": |- + This field is only required for Azure AD OpenAI and is the Microsoft + Entra Tenant ID. "openai_api_base": - "description": | - This is a field to provide a customized base URl for the OpenAI API. - For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service - provided by Azure. - For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used. + "description": |- + This is a field to provide a customized base URl for the OpenAI API. For + Azure OpenAI, this field is required, and is the base URL for the Azure + OpenAI API service provided by Azure. For other OpenAI API types, this + field is optional, and if left unspecified, the standard OpenAI base URL + is used. "openai_api_key": "description": |- - The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + The Databricks secret key reference for an OpenAI API key using the + OpenAI or Azure service. If you prefer to paste your API key directly, + see `openai_api_key_plaintext`. You must provide an API key using one of + the following fields: `openai_api_key` or `openai_api_key_plaintext`. "openai_api_key_plaintext": "description": |- - The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + The OpenAI API key using the OpenAI or Azure service provided as a + plaintext string. If you prefer to reference your key using Databricks + Secrets, see `openai_api_key`. You must provide an API key using one of + the following fields: `openai_api_key` or `openai_api_key_plaintext`. "openai_api_type": - "description": | - This is an optional field to specify the type of OpenAI API to use. - For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security - access validation protocol. For access token validation, use azure. For authentication using Azure Active + "description": |- + This is an optional field to specify the type of OpenAI API to use. For + Azure OpenAI, this field is required, and adjust this parameter to + represent the preferred security access validation protocol. For access + token validation, use azure. For authentication using Azure Active Directory (Azure AD) use, azuread. "openai_api_version": - "description": | - This is an optional field to specify the OpenAI API version. - For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to - utilize, specified by a date. + "description": |- + This is an optional field to specify the OpenAI API version. For Azure + OpenAI, this field is required, and is the version of the Azure OpenAI + service to utilize, specified by a date. "openai_deployment_name": - "description": | - This field is only required for Azure OpenAI and is the name of the deployment resource for the - Azure OpenAI service. + "description": |- + This field is only required for Azure OpenAI and is the name of the + deployment resource for the Azure OpenAI service. "openai_organization": - "description": | - This is an optional field to specify the organization in OpenAI or Azure OpenAI. + "description": |- + This is an optional field to specify the organization in OpenAI or Azure + OpenAI. github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig: "palm_api_key": "description": |- - The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. + The Databricks secret key reference for a PaLM API key. If you prefer to + paste your API key directly, see `palm_api_key_plaintext`. You must + provide an API key using one of the following fields: `palm_api_key` or + `palm_api_key_plaintext`. "palm_api_key_plaintext": "description": |- - The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. + The PaLM API key provided as a plaintext string. If you prefer to + reference your key using Databricks Secrets, see `palm_api_key`. You must + provide an API key using one of the following fields: `palm_api_key` or + `palm_api_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.RateLimit: "calls": "description": |- @@ -3010,8 +3100,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimit: Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey: "_": - "description": |- - Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. "enum": - |- user @@ -3019,8 +3107,6 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey: endpoint github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod: "_": - "description": |- - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. "enum": - |- minute @@ -3033,21 +3119,15 @@ github.com/databricks/databricks-sdk-go/service/serving.Route: The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive. github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_name": - "description": | - The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), - or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of - __catalog_name__.__schema_name__.__model_name__. - "entity_version": "description": |- - The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC. + The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**. + "entity_version": {} "environment_vars": - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" + "description": |- + An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}` "external_model": - "description": | - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) - can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, - it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. - The task type of all external models within an endpoint must be the same. + "description": |- + The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. "instance_profile_arn": "description": |- ARN of the instance profile that the served entity uses to access AWS resources. @@ -3058,68 +3138,46 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "description": |- The minimum tokens per second that the endpoint can scale down to. "name": - "description": | - The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. - If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other - entities, it defaults to -. + "description": |- + The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version. "scale_to_zero_enabled": "description": |- Whether the compute resources for the served entity should scale down to zero. "workload_size": - "description": | - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. - A single unit of provisioned concurrency can process one request at a time. - Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). - If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + "description": |- + The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. "workload_type": - "description": | - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is - "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. - See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). + "description": |- + The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput: "environment_vars": - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`" + "description": |- + An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}` "instance_profile_arn": "description": |- - ARN of the instance profile that the served model will use to access AWS resources. + ARN of the instance profile that the served entity uses to access AWS resources. "max_provisioned_throughput": "description": |- The maximum tokens per second that the endpoint can scale up to. "min_provisioned_throughput": "description": |- The minimum tokens per second that the endpoint can scale down to. - "model_name": - "description": | - The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model, - in the form of __catalog_name__.__schema_name__.__model_name__. - "model_version": - "description": |- - The version of the model in Databricks Model Registry or Unity Catalog to be served. + "model_name": {} + "model_version": {} "name": - "description": | - The name of a served model. It must be unique across an endpoint. If not specified, this field will default to -. - A served model name can consist of alphanumeric characters, dashes, and underscores. + "description": |- + The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version. "scale_to_zero_enabled": "description": |- - Whether the compute resources for the served model should scale down to zero. + Whether the compute resources for the served entity should scale down to zero. "workload_size": - "description": | - The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between. - A single unit of provisioned concurrency can process one request at a time. - Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). - If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0. + "description": |- + The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. "workload_type": - "description": | - The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is - "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. - See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). + "description": |- + The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize: "_": - "description": | - The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between. - A single unit of provisioned concurrency can process one request at a time. - Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). - If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0. "enum": - |- Small @@ -3129,17 +3187,26 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkload Large github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType: "_": - "description": | - The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is - "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. - See the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). "enum": - |- CPU + - |- + GPU_MEDIUM - |- GPU_SMALL + - |- + GPU_LARGE + - |- + MULTIGPU_MEDIUM +github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType: + "_": + "enum": + - |- + CPU - |- GPU_MEDIUM + - |- + GPU_SMALL - |- GPU_LARGE - |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 120a12543..323432fa3 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -197,3 +197,14 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: + "entity_version": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput: + "model_name": + "description": |- + PLACEHOLDER + "model_version": + "description": |- + PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4a3b56814..17a621ba0 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -546,7 +546,7 @@ "type": "object", "properties": { "ai_gateway": { - "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.", + "description": "The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig" }, "config": { @@ -554,7 +554,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" }, "name": { - "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n", + "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.", "$ref": "#/$defs/string" }, "permissions": { @@ -575,7 +575,6 @@ }, "additionalProperties": false, "required": [ - "config", "name" ] }, @@ -4142,6 +4141,10 @@ "parameters": { "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", "$ref": "#/$defs/slice/string" + }, + "run_as_repl": { + "description": "Deprecated. A value of `false` is no longer supported.", + "$ref": "#/$defs/bool" } }, "additionalProperties": false @@ -5502,11 +5505,11 @@ "type": "object", "properties": { "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.", + "description": "The Databricks secret key reference for an AI21 Labs API key. If you\nprefer to paste your API key directly, see `ai21labs_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.", "$ref": "#/$defs/string" }, "ai21labs_api_key_plaintext": { - "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`.", + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.", "$ref": "#/$defs/string" } }, @@ -5528,7 +5531,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails" }, "inference_table_config": { - "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", + "description": "Configuration for payload logging using inference tables.\nUse these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig" }, "rate_limits": { @@ -5536,7 +5539,7 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" }, "usage_tracking_config": { - "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.", + "description": "Configuration to enable usage tracking using system tables.\nThese tables allow you to monitor operational usage on endpoints and their associated costs.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig" } }, @@ -5554,7 +5557,7 @@ "type": "object", "properties": { "invalid_keywords": { - "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", + "description": "List of invalid keywords.\nAI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", "$ref": "#/$defs/slice/string" }, "pii": { @@ -5566,7 +5569,7 @@ "$ref": "#/$defs/bool" }, "valid_topics": { - "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.", + "description": "The list of allowed topics.\nGiven a chat request, this guardrail flags the request if its topic is not in the allowed topics.", "$ref": "#/$defs/slice/string" } }, @@ -5584,14 +5587,11 @@ "type": "object", "properties": { "behavior": { - "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", + "description": "Configuration for input guardrail filters.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior" } }, - "additionalProperties": false, - "required": [ - "behavior" - ] + "additionalProperties": false }, { "type": "string", @@ -5603,7 +5603,6 @@ "oneOf": [ { "type": "string", - "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", "enum": [ "NONE", "BLOCK" @@ -5643,7 +5642,7 @@ "type": "object", "properties": { "catalog_name": { - "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.", + "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the catalog name.", "$ref": "#/$defs/string" }, "enabled": { @@ -5651,11 +5650,11 @@ "$ref": "#/$defs/bool" }, "schema_name": { - "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.", + "description": "The name of the schema in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the schema name.", "$ref": "#/$defs/string" }, "table_name_prefix": { - "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.", + "description": "The prefix of the table in Unity Catalog.\nNOTE: On update, you have to disable inference table first in order to change the prefix name.", "$ref": "#/$defs/string" } }, @@ -5674,10 +5673,10 @@ "properties": { "calls": { "description": "Used to specify how many calls are allowed for a key within the renewal_period.", - "$ref": "#/$defs/int" + "$ref": "#/$defs/int64" }, "key": { - "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,\nwith 'endpoint' being the default if not specified.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey" }, "renewal_period": { @@ -5701,7 +5700,6 @@ "oneOf": [ { "type": "string", - "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", "enum": [ "user", "endpoint" @@ -5717,7 +5715,6 @@ "oneOf": [ { "type": "string", - "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", "enum": [ "minute" ] @@ -5752,11 +5749,11 @@ "type": "object", "properties": { "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "description": "The Databricks secret key reference for an AWS access key ID with\npermissions to interact with Bedrock services. If you prefer to paste\nyour API key directly, see `aws_access_key_id_plaintext`. You must provide an API\nkey using one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.", "$ref": "#/$defs/string" }, "aws_access_key_id_plaintext": { - "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.", "$ref": "#/$defs/string" }, "aws_region": { @@ -5764,15 +5761,15 @@ "$ref": "#/$defs/string" }, "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "description": "The Databricks secret key reference for an AWS secret access key paired\nwith the access key ID, with permissions to interact with Bedrock\nservices. If you prefer to paste your API key directly, see\n`aws_secret_access_key_plaintext`. You must provide an API key using one\nof the following fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.", "$ref": "#/$defs/string" }, "aws_secret_access_key_plaintext": { - "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.", "$ref": "#/$defs/string" }, "bedrock_provider": { - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", + "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider" } }, @@ -5792,7 +5789,6 @@ "oneOf": [ { "type": "string", - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", "enum": [ "anthropic", "cohere", @@ -5812,11 +5808,11 @@ "type": "object", "properties": { "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "description": "The Databricks secret key reference for an Anthropic API key. If you\nprefer to paste your API key directly, see `anthropic_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.", "$ref": "#/$defs/string" }, "anthropic_api_key_plaintext": { - "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.", "$ref": "#/$defs/string" } }, @@ -5864,15 +5860,15 @@ "type": "object", "properties": { "cohere_api_base": { - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n", + "description": "This is an optional field to provide a customized base URL for the Cohere\nAPI. If left unspecified, the standard Cohere base URL is used.", "$ref": "#/$defs/string" }, "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "description": "The Databricks secret key reference for a Cohere API key. If you prefer\nto paste your API key directly, see `cohere_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `cohere_api_key` or\n`cohere_api_key_plaintext`.", "$ref": "#/$defs/string" }, "cohere_api_key_plaintext": { - "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.", "$ref": "#/$defs/string" } }, @@ -5890,15 +5886,15 @@ "type": "object", "properties": { "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "description": "The Databricks secret key reference for a Databricks API token that\ncorresponds to a user or service principal with Can Query access to the\nmodel serving endpoint pointed to by this external model. If you prefer\nto paste your API key directly, see `databricks_api_token_plaintext`. You\nmust provide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.", "$ref": "#/$defs/string" }, "databricks_api_token_plaintext": { - "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.", "$ref": "#/$defs/string" }, "databricks_workspace_url": { - "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n", + "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.", "$ref": "#/$defs/string" } }, @@ -5919,19 +5915,19 @@ "type": "object", "properties": { "auto_capture_config": { - "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", + "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.\nNote: this field is deprecated for creating new provisioned throughput endpoints,\nor updating existing provisioned throughput endpoints that never have inference table configured;\nin these cases please use AI Gateway to manage inference tables.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput" }, "served_entities": { - "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", + "description": "The list of served entities under the serving endpoint config.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" }, "served_models": { - "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", + "description": "(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" }, "traffic_config": { - "description": "The traffic config defining how invocations to the serving endpoint should be routed.", + "description": "The traffic configuration associated with the serving endpoint config.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig" } }, @@ -6010,7 +6006,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig" }, "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider" }, "task": { @@ -6035,7 +6031,6 @@ "oneOf": [ { "type": "string", - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", "enum": [ "ai21labs", "anthropic", @@ -6059,23 +6054,27 @@ "type": "object", "properties": { "private_key": { - "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`", + "description": "The Databricks secret key reference for a private key for the service\naccount which has access to the Google Cloud Vertex AI Service. See [Best\npractices for managing service account keys]. If you prefer to paste your\nAPI key directly, see `private_key_plaintext`. You must provide an API\nkey using one of the following fields: `private_key` or\n`private_key_plaintext`\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys", "$ref": "#/$defs/string" }, "private_key_plaintext": { - "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`.", + "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys", "$ref": "#/$defs/string" }, "project_id": { - "description": "This is the Google Cloud project id that the service account is associated with.", + "description": "This is the Google Cloud project id that the service account is\nassociated with.", "$ref": "#/$defs/string" }, "region": { - "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions.", + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported\nregions] for more details. Some models are only available in specific\nregions.\n\n[supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations", "$ref": "#/$defs/string" } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "project_id", + "region" + ] }, { "type": "string", @@ -6087,49 +6086,50 @@ "oneOf": [ { "type": "object", + "description": "Configs needed to create an OpenAI model route.", "properties": { "microsoft_entra_client_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n", + "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Client ID.", "$ref": "#/$defs/string" }, "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n", + "description": "The Databricks secret key reference for a client secret used for\nMicrosoft Entra ID authentication. If you prefer to paste your client\nsecret directly, see `microsoft_entra_client_secret_plaintext`. You must\nprovide an API key using one of the following fields:\n`microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.", "$ref": "#/$defs/string" }, "microsoft_entra_client_secret_plaintext": { - "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n", + "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.", "$ref": "#/$defs/string" }, "microsoft_entra_tenant_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n", + "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.", "$ref": "#/$defs/string" }, "openai_api_base": { - "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n", + "description": "This is a field to provide a customized base URl for the OpenAI API. For\nAzure OpenAI, this field is required, and is the base URL for the Azure\nOpenAI API service provided by Azure. For other OpenAI API types, this\nfield is optional, and if left unspecified, the standard OpenAI base URL\nis used.", "$ref": "#/$defs/string" }, "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.", + "description": "The Databricks secret key reference for an OpenAI API key using the\nOpenAI or Azure service. If you prefer to paste your API key directly,\nsee `openai_api_key_plaintext`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.", "$ref": "#/$defs/string" }, "openai_api_key_plaintext": { - "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`.", + "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.", "$ref": "#/$defs/string" }, "openai_api_type": { - "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n", + "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.", "$ref": "#/$defs/string" }, "openai_api_version": { - "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n", + "description": "This is an optional field to specify the OpenAI API version. For Azure\nOpenAI, this field is required, and is the version of the Azure OpenAI\nservice to utilize, specified by a date.", "$ref": "#/$defs/string" }, "openai_deployment_name": { - "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n", + "description": "This field is only required for Azure OpenAI and is the name of the\ndeployment resource for the Azure OpenAI service.", "$ref": "#/$defs/string" }, "openai_organization": { - "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n", + "description": "This is an optional field to specify the organization in OpenAI or Azure\nOpenAI.", "$ref": "#/$defs/string" } }, @@ -6147,11 +6147,11 @@ "type": "object", "properties": { "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.", + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to\npaste your API key directly, see `palm_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.", "$ref": "#/$defs/string" }, "palm_api_key_plaintext": { - "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`.", + "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.", "$ref": "#/$defs/string" } }, @@ -6170,7 +6170,7 @@ "properties": { "calls": { "description": "Used to specify how many calls are allowed for a key within the renewal_period.", - "$ref": "#/$defs/int" + "$ref": "#/$defs/int64" }, "key": { "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", @@ -6197,7 +6197,6 @@ "oneOf": [ { "type": "string", - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", "enum": [ "user", "endpoint" @@ -6213,7 +6212,6 @@ "oneOf": [ { "type": "string", - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", "enum": [ "minute" ] @@ -6256,19 +6254,18 @@ "type": "object", "properties": { "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n", + "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.", "$ref": "#/$defs/string" }, "entity_version": { - "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.", "$ref": "#/$defs/string" }, "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", "$ref": "#/$defs/map/string" }, "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel" }, "instance_profile_arn": { @@ -6284,7 +6281,7 @@ "$ref": "#/$defs/int" }, "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n", + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", "$ref": "#/$defs/string" }, "scale_to_zero_enabled": { @@ -6292,12 +6289,12 @@ "$ref": "#/$defs/bool" }, "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n", + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.", "$ref": "#/$defs/string" }, "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", - "$ref": "#/$defs/string" + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType" } }, "additionalProperties": false @@ -6314,11 +6311,11 @@ "type": "object", "properties": { "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", "$ref": "#/$defs/map/string" }, "instance_profile_arn": { - "description": "ARN of the instance profile that the served model will use to access AWS resources.", + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", "$ref": "#/$defs/string" }, "max_provisioned_throughput": { @@ -6330,27 +6327,25 @@ "$ref": "#/$defs/int" }, "model_name": { - "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n", "$ref": "#/$defs/string" }, "model_version": { - "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.", "$ref": "#/$defs/string" }, "name": { - "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n", + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", "$ref": "#/$defs/string" }, "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served model should scale down to zero.", + "description": "Whether the compute resources for the served entity should scale down to zero.", "$ref": "#/$defs/bool" }, "workload_size": { - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize" }, "workload_type": { - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType" } }, @@ -6371,7 +6366,6 @@ "oneOf": [ { "type": "string", - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", "enum": [ "Small", "Medium", @@ -6388,11 +6382,28 @@ "oneOf": [ { "type": "string", - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", "enum": [ "CPU", - "GPU_SMALL", "GPU_MEDIUM", + "GPU_SMALL", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServingModelWorkloadType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CPU", + "GPU_MEDIUM", + "GPU_SMALL", "GPU_LARGE", "MULTIGPU_MEDIUM" ] diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index 1eec1018e..43e458bc6 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -307,6 +307,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: redirect_urls + // TODO: array: scopes // TODO: complex arg: token_access_policy cmd.Use = "update INTEGRATION_ID" diff --git a/cmd/api/api.go b/cmd/api/api.go index c3a3eb0b6..fad8a026f 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -62,7 +62,7 @@ func makeCommand(method string) *cobra.Command { var response any headers := map[string]string{"Content-Type": "application/json"} - err = api.Do(cmd.Context(), method, path, headers, request, &response) + err = api.Do(cmd.Context(), method, path, headers, nil, request, &response) if err != nil { return err } diff --git a/cmd/workspace/access-control/access-control.go b/cmd/workspace/access-control/access-control.go new file mode 100755 index 000000000..7668265fb --- /dev/null +++ b/cmd/workspace/access-control/access-control.go @@ -0,0 +1,109 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package access_control + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "access-control", + Short: `Rule based Access Control for Databricks Resources.`, + Long: `Rule based Access Control for Databricks Resources.`, + GroupID: "iam", + Annotations: map[string]string{ + "package": "iam", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCheckPolicy()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start check-policy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var checkPolicyOverrides []func( + *cobra.Command, + *iam.CheckPolicyRequest, +) + +func newCheckPolicy() *cobra.Command { + cmd := &cobra.Command{} + + var checkPolicyReq iam.CheckPolicyRequest + var checkPolicyJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&checkPolicyJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: resource_info + + cmd.Use = "check-policy" + cmd.Short = `Check access policy to a resource.` + cmd.Long = `Check access policy to a resource.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := checkPolicyJson.Unmarshal(&checkPolicyReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.AccessControl.CheckPolicy(ctx, checkPolicyReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range checkPolicyOverrides { + fn(cmd, &checkPolicyReq) + } + + return cmd +} + +// end service AccessControl diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index f07d0cf76..c447bd736 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -3,6 +3,7 @@ package workspace import ( + access_control "github.com/databricks/cli/cmd/workspace/access-control" alerts "github.com/databricks/cli/cmd/workspace/alerts" alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy" apps "github.com/databricks/cli/cmd/workspace/apps" @@ -96,6 +97,7 @@ import ( func All() []*cobra.Command { var out []*cobra.Command + out = append(out, access_control.New()) out = append(out, alerts.New()) out = append(out, alerts_legacy.New()) out = append(out, apps.New()) diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index 504beac5e..4d6262cff 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -64,7 +64,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the provider.`) - cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`) + cmd.Flags().StringVar(&createReq.RecipientProfileStr, "recipient-profile-str", createReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`) cmd.Use = "create NAME AUTHENTICATION_TYPE" cmd.Short = `Create an auth provider.` @@ -430,7 +430,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the provider.`) cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the provider.`) cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of Provider owner.`) - cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN** or not provided.`) + cmd.Flags().StringVar(&updateReq.RecipientProfileStr, "recipient-profile-str", updateReq.RecipientProfileStr, `This field is required when the __authentication_type__ is **TOKEN**, **OAUTH_CLIENT_CREDENTIALS** or not provided.`) cmd.Use = "update NAME" cmd.Short = `Update a provider.` diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index 56abd2014..6d6ce42f1 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -91,7 +91,7 @@ func newCreate() *cobra.Command { cmd.Long = `Create a share recipient. Creates a new recipient with the delta sharing authentication type in the - metastore. The caller must be a metastore admin or has the + metastore. The caller must be a metastore admin or have the **CREATE_RECIPIENT** privilege on the metastore. Arguments: @@ -186,28 +186,16 @@ func newDelete() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down." - names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Name of the recipient") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have name of the recipient") - } deleteReq.Name = args[0] err = w.Recipients.Delete(ctx, deleteReq) @@ -258,28 +246,16 @@ func newGet() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down." - names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Name of the recipient") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have name of the recipient") - } getReq.Name = args[0] response, err := w.Recipients.Get(ctx, getReq) @@ -384,7 +360,7 @@ func newRotateToken() *cobra.Command { the provided token info. The caller must be the owner of the recipient. Arguments: - NAME: The name of the recipient. + NAME: The name of the Recipient. EXISTING_TOKEN_EXPIRE_IN_SECONDS: The expiration time of the bearer token in ISO 8601 format. This will set the expiration_time of existing token only to a smaller timestamp, it cannot extend the expiration_time. Use 0 to expire the existing token @@ -479,28 +455,16 @@ func newSharePermissions() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down." - names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "The name of the Recipient") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have the name of the recipient") - } sharePermissionsReq.Name = args[0] response, err := w.Recipients.SharePermissions(ctx, sharePermissionsReq) @@ -560,6 +524,11 @@ func newUpdate() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -577,30 +546,13 @@ func newUpdate() *cobra.Command { } } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No NAME argument specified. Loading names for Recipients drop-down." - names, err := w.Recipients.RecipientInfoNameToMetastoreIdMap(ctx, sharing.ListRecipientsRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Recipients drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Name of the recipient") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have name of the recipient") - } updateReq.Name = args[0] - err = w.Recipients.Update(ctx, updateReq) + response, err := w.Recipients.Update(ctx, updateReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index cc99177c7..034133623 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -49,6 +49,7 @@ func New() *cobra.Command { cmd.AddCommand(newGetOpenApi()) cmd.AddCommand(newGetPermissionLevels()) cmd.AddCommand(newGetPermissions()) + cmd.AddCommand(newHttpRequest()) cmd.AddCommand(newList()) cmd.AddCommand(newLogs()) cmd.AddCommand(newPatch()) @@ -153,16 +154,34 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: complex arg: ai_gateway + // TODO: complex arg: config // TODO: array: rate_limits cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`) // TODO: array: tags - cmd.Use = "create" + cmd.Use = "create NAME" cmd.Short = `Create a new serving endpoint.` - cmd.Long = `Create a new serving endpoint.` + cmd.Long = `Create a new serving endpoint. + + Arguments: + NAME: The name of the serving endpoint. This field is required and must be + unique across a Databricks workspace. An endpoint name can consist of + alphanumeric characters, dashes, and underscores.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -179,8 +198,9 @@ func newCreate() *cobra.Command { return err } } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + if !cmd.Flags().Changed("json") { + createReq.Name = args[0] } wait, err := w.ServingEndpoints.Create(ctx, createReq) @@ -233,10 +253,7 @@ func newDelete() *cobra.Command { cmd.Use = "delete NAME" cmd.Short = `Delete a serving endpoint.` - cmd.Long = `Delete a serving endpoint. - - Arguments: - NAME: The name of the serving endpoint. This field is required.` + cmd.Long = `Delete a serving endpoint.` cmd.Annotations = make(map[string]string) @@ -432,11 +449,12 @@ func newGetOpenApi() *cobra.Command { getOpenApiReq.Name = args[0] - err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq) + response, err := w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq) if err != nil { return err } - return nil + defer response.Contents.Close() + return cmdio.Render(ctx, response.Contents) } // Disable completions since they are not applicable. @@ -568,6 +586,77 @@ func newGetPermissions() *cobra.Command { return cmd } +// start http-request command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var httpRequestOverrides []func( + *cobra.Command, + *serving.ExternalFunctionRequest, +) + +func newHttpRequest() *cobra.Command { + cmd := &cobra.Command{} + + var httpRequestReq serving.ExternalFunctionRequest + + // TODO: short flags + + cmd.Flags().StringVar(&httpRequestReq.Headers, "headers", httpRequestReq.Headers, `Additional headers for the request.`) + cmd.Flags().StringVar(&httpRequestReq.Json, "json", httpRequestReq.Json, `The JSON payload to send in the request body.`) + cmd.Flags().StringVar(&httpRequestReq.Params, "params", httpRequestReq.Params, `Query parameters for the request.`) + + cmd.Use = "http-request CONNECTION_NAME METHOD PATH" + cmd.Short = `Make external services call using the credentials stored in UC Connection.` + cmd.Long = `Make external services call using the credentials stored in UC Connection. + + Arguments: + CONNECTION_NAME: The connection name to use. This is required to identify the external + connection. + METHOD: The HTTP method to use (e.g., 'GET', 'POST'). + PATH: The relative path for the API endpoint. This is required.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + httpRequestReq.ConnectionName = args[0] + _, err = fmt.Sscan(args[1], &httpRequestReq.Method) + if err != nil { + return fmt.Errorf("invalid METHOD: %s", args[1]) + } + httpRequestReq.Path = args[2] + + response, err := w.ServingEndpoints.HttpRequest(ctx, httpRequestReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range httpRequestOverrides { + fn(cmd, &httpRequestReq) + } + + return cmd +} + // start list command // Slice with functions to override default command behavior. @@ -849,7 +938,7 @@ func newPutAiGateway() *cobra.Command { cmd.Long = `Update AI Gateway of a serving endpoint. Used to update the AI Gateway of a serving endpoint. NOTE: Only external model - endpoints are currently supported. + and provisioned throughput endpoints are currently supported. Arguments: NAME: The name of the serving endpoint whose AI Gateway is being updated. This diff --git a/go.mod b/go.mod index 930963f89..bd8997190 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.55.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.56.1 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 diff --git a/go.sum b/go.sum index d025b3947..dec1d40b2 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.55.0 h1:ReziD6spzTDltM0ml80LggKo27F3oUjgTinCFDJDnak= -github.com/databricks/databricks-sdk-go v0.55.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= +github.com/databricks/databricks-sdk-go v0.56.1 h1:sgweTRvAQaI8EPrfDnVdAB0lNX6L5uTT720SlMMQI2U= +github.com/databricks/databricks-sdk-go v0.56.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index 632497054..88e6ed89a 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -158,7 +158,7 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected var res []byte a.c.Eventually(func() bool { - err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &res) + err = apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &res) require.NoError(a.t, err) actualContent := string(res) return actualContent == expectedContent diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 88bbadd32..7102b6e29 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -148,7 +148,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, overwrite := slices.Contains(mode, OverwriteIfExists) urlPath = fmt.Sprintf("%s?overwrite=%t", urlPath, overwrite) headers := map[string]string{"Content-Type": "application/octet-stream"} - err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, reader, nil) + err = w.apiClient.Do(ctx, http.MethodPut, urlPath, headers, nil, reader, nil) // Return early on success. if err == nil { @@ -176,7 +176,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err } var reader io.ReadCloser - err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader) + err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &reader) // Return early on success. if err == nil { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 8d5148edd..1d514f13b 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -106,7 +106,7 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) { // as an interface to allow for mocking in tests. type apiClient interface { Do(ctx context.Context, method, path string, - headers map[string]string, request, response any, + headers map[string]string, queryString map[string]any, request, response any, visitors ...func(*http.Request) error) error } @@ -156,7 +156,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } - err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, body, nil) + err = w.apiClient.Do(ctx, http.MethodPost, urlPath, nil, nil, body, nil) // Return early on success. if err == nil { @@ -341,6 +341,7 @@ func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileIn http.MethodGet, "/api/2.0/workspace/get-status", nil, + nil, map[string]string{ "path": absPath, "return_export_info": "true", diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go index 9ea837fa9..e9fde4762 100644 --- a/libs/filer/workspace_files_extensions_client_test.go +++ b/libs/filer/workspace_files_extensions_client_test.go @@ -17,7 +17,7 @@ type mockApiClient struct { } func (m *mockApiClient) Do(ctx context.Context, method, path string, - headers map[string]string, request, response any, + headers map[string]string, queryString map[string]any, request, response any, visitors ...func(*http.Request) error, ) error { args := m.Called(ctx, method, path, headers, request, response, visitors) diff --git a/libs/git/info.go b/libs/git/info.go index 46e57be48..dc4af9b6d 100644 --- a/libs/git/info.go +++ b/libs/git/info.go @@ -66,6 +66,7 @@ func fetchRepositoryInfoAPI(ctx context.Context, path string, w *databricks.Work http.MethodGet, apiEndpoint, nil, + nil, map[string]string{ "path": path, "return_git_info": "true", From 65fbbd9a7c75a2404fa3d4956560ab037535d779 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 14:22:08 +0100 Subject: [PATCH 009/105] libs/python: Remove DetectInterpreters (#2234) ## Changes - Remove DetectInterpreters from DetectExecutable call: python3 or python should always be on on the PATH. We don't need to detect non-standard situations like python3.10 is present but python3 is not. - I moved DetectInterpreters to cmd/labs where it is still used. This is a follow up to https://github.com/databricks/cli/pull/2034 ## Tests Existing tests. --- cmd/labs/project/installer.go | 3 +-- .../labs/project}/interpreters.go | 2 +- .../labs/project}/interpreters_unix_test.go | 2 +- .../labs/project}/interpreters_win_test.go | 2 +- .../testdata/other-binaries-filtered/python | 0 .../other-binaries-filtered/python3-whatever | 0 .../other-binaries-filtered/python3.10 | 0 .../other-binaries-filtered/python3.10.100 | 0 .../other-binaries-filtered/python3.11 | 0 .../other-binaries-filtered/python4.8 | 0 .../testdata/other-binaries-filtered/python5 | 0 .../testdata/other-binaries-filtered/python6 | 0 .../testdata/other-binaries-filtered/python7 | 0 .../testdata/other-binaries-filtered/pythonw | 0 .../other-binaries-filtered/real-python3.11.4 | 0 .../testdata/other-binaries-filtered/whatever | 0 .../testdata/world-writeable/python8.4 | 0 libs/python/detect.go | 22 +------------------ libs/python/detect_unix_test.go | 12 ++-------- libs/python/detect_win_test.go | 2 +- 20 files changed, 8 insertions(+), 37 deletions(-) rename {libs/python => cmd/labs/project}/interpreters.go (99%) rename {libs/python => cmd/labs/project}/interpreters_unix_test.go (99%) rename {libs/python => cmd/labs/project}/interpreters_win_test.go (97%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3-whatever (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.10 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.10.100 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python3.11 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python4.8 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python5 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python6 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/python7 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/pythonw (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/real-python3.11.4 (100%) rename {libs/python => cmd/labs/project}/testdata/other-binaries-filtered/whatever (100%) rename {libs/python => cmd/labs/project}/testdata/world-writeable/python8.4 (100%) diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 7d31623bb..05f7d68aa 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/process" - "github.com/databricks/cli/libs/python" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/sql" @@ -223,7 +222,7 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr feedback := cmdio.Spinner(ctx) defer close(feedback) feedback <- "Detecting all installed Python interpreters on the system" - pythonInterpreters, err := python.DetectInterpreters(ctx) + pythonInterpreters, err := DetectInterpreters(ctx) if err != nil { return fmt.Errorf("detect: %w", err) } diff --git a/libs/python/interpreters.go b/cmd/labs/project/interpreters.go similarity index 99% rename from libs/python/interpreters.go rename to cmd/labs/project/interpreters.go index 6071309a8..00f099ed4 100644 --- a/libs/python/interpreters.go +++ b/cmd/labs/project/interpreters.go @@ -1,4 +1,4 @@ -package python +package project import ( "context" diff --git a/libs/python/interpreters_unix_test.go b/cmd/labs/project/interpreters_unix_test.go similarity index 99% rename from libs/python/interpreters_unix_test.go rename to cmd/labs/project/interpreters_unix_test.go index 57adc9279..a5bbb6468 100644 --- a/libs/python/interpreters_unix_test.go +++ b/cmd/labs/project/interpreters_unix_test.go @@ -1,6 +1,6 @@ //go:build unix -package python +package project import ( "context" diff --git a/libs/python/interpreters_win_test.go b/cmd/labs/project/interpreters_win_test.go similarity index 97% rename from libs/python/interpreters_win_test.go rename to cmd/labs/project/interpreters_win_test.go index f99981529..2316daa30 100644 --- a/libs/python/interpreters_win_test.go +++ b/cmd/labs/project/interpreters_win_test.go @@ -1,6 +1,6 @@ //go:build windows -package python +package project import ( "context" diff --git a/libs/python/testdata/other-binaries-filtered/python b/cmd/labs/project/testdata/other-binaries-filtered/python similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python rename to cmd/labs/project/testdata/other-binaries-filtered/python diff --git a/libs/python/testdata/other-binaries-filtered/python3-whatever b/cmd/labs/project/testdata/other-binaries-filtered/python3-whatever similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python3-whatever rename to cmd/labs/project/testdata/other-binaries-filtered/python3-whatever diff --git a/libs/python/testdata/other-binaries-filtered/python3.10 b/cmd/labs/project/testdata/other-binaries-filtered/python3.10 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python3.10 rename to cmd/labs/project/testdata/other-binaries-filtered/python3.10 diff --git a/libs/python/testdata/other-binaries-filtered/python3.10.100 b/cmd/labs/project/testdata/other-binaries-filtered/python3.10.100 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python3.10.100 rename to cmd/labs/project/testdata/other-binaries-filtered/python3.10.100 diff --git a/libs/python/testdata/other-binaries-filtered/python3.11 b/cmd/labs/project/testdata/other-binaries-filtered/python3.11 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python3.11 rename to cmd/labs/project/testdata/other-binaries-filtered/python3.11 diff --git a/libs/python/testdata/other-binaries-filtered/python4.8 b/cmd/labs/project/testdata/other-binaries-filtered/python4.8 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python4.8 rename to cmd/labs/project/testdata/other-binaries-filtered/python4.8 diff --git a/libs/python/testdata/other-binaries-filtered/python5 b/cmd/labs/project/testdata/other-binaries-filtered/python5 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python5 rename to cmd/labs/project/testdata/other-binaries-filtered/python5 diff --git a/libs/python/testdata/other-binaries-filtered/python6 b/cmd/labs/project/testdata/other-binaries-filtered/python6 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python6 rename to cmd/labs/project/testdata/other-binaries-filtered/python6 diff --git a/libs/python/testdata/other-binaries-filtered/python7 b/cmd/labs/project/testdata/other-binaries-filtered/python7 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/python7 rename to cmd/labs/project/testdata/other-binaries-filtered/python7 diff --git a/libs/python/testdata/other-binaries-filtered/pythonw b/cmd/labs/project/testdata/other-binaries-filtered/pythonw similarity index 100% rename from libs/python/testdata/other-binaries-filtered/pythonw rename to cmd/labs/project/testdata/other-binaries-filtered/pythonw diff --git a/libs/python/testdata/other-binaries-filtered/real-python3.11.4 b/cmd/labs/project/testdata/other-binaries-filtered/real-python3.11.4 similarity index 100% rename from libs/python/testdata/other-binaries-filtered/real-python3.11.4 rename to cmd/labs/project/testdata/other-binaries-filtered/real-python3.11.4 diff --git a/libs/python/testdata/other-binaries-filtered/whatever b/cmd/labs/project/testdata/other-binaries-filtered/whatever similarity index 100% rename from libs/python/testdata/other-binaries-filtered/whatever rename to cmd/labs/project/testdata/other-binaries-filtered/whatever diff --git a/libs/python/testdata/world-writeable/python8.4 b/cmd/labs/project/testdata/world-writeable/python8.4 similarity index 100% rename from libs/python/testdata/world-writeable/python8.4 rename to cmd/labs/project/testdata/world-writeable/python8.4 diff --git a/libs/python/detect.go b/libs/python/detect.go index e86d9d621..75158da65 100644 --- a/libs/python/detect.go +++ b/libs/python/detect.go @@ -39,27 +39,7 @@ func DetectExecutable(ctx context.Context) (string, error) { // // See https://github.com/pyenv/pyenv#understanding-python-version-selection - out, err := exec.LookPath(GetExecutable()) - - // most of the OS'es have python3 in $PATH, but for those which don't, - // we perform the latest version lookup - if err != nil && !errors.Is(err, exec.ErrNotFound) { - return "", err - } - if out != "" { - return out, nil - } - // otherwise, detect all interpreters and pick the least that satisfies - // minimal version requirements - all, err := DetectInterpreters(ctx) - if err != nil { - return "", err - } - interpreter, err := all.AtLeast("3.8") - if err != nil { - return "", err - } - return interpreter.Path, nil + return exec.LookPath(GetExecutable()) } // DetectVEnvExecutable returns the path to the python3 executable inside venvPath, diff --git a/libs/python/detect_unix_test.go b/libs/python/detect_unix_test.go index a962e1f55..1774aa108 100644 --- a/libs/python/detect_unix_test.go +++ b/libs/python/detect_unix_test.go @@ -16,24 +16,16 @@ func TestDetectsViaPathLookup(t *testing.T) { assert.NotEmpty(t, py) } -func TestDetectsViaListing(t *testing.T) { - t.Setenv("PATH", "testdata/other-binaries-filtered") - ctx := context.Background() - py, err := DetectExecutable(ctx) - assert.NoError(t, err) - assert.Equal(t, "testdata/other-binaries-filtered/python3.10", py) -} - func TestDetectFailsNoInterpreters(t *testing.T) { t.Setenv("PATH", "testdata") ctx := context.Background() _, err := DetectExecutable(ctx) - assert.Equal(t, ErrNoPythonInterpreters, err) + assert.Error(t, err) } func TestDetectFailsNoMinimalVersion(t *testing.T) { t.Setenv("PATH", "testdata/no-python3") ctx := context.Background() _, err := DetectExecutable(ctx) - assert.EqualError(t, err, "cannot find Python greater or equal to v3.8.0") + assert.Error(t, err) } diff --git a/libs/python/detect_win_test.go b/libs/python/detect_win_test.go index 2ef811a4b..7b2ee281e 100644 --- a/libs/python/detect_win_test.go +++ b/libs/python/detect_win_test.go @@ -20,5 +20,5 @@ func TestDetectFailsNoInterpreters(t *testing.T) { t.Setenv("PATH", "testdata") ctx := context.Background() _, err := DetectExecutable(ctx) - assert.ErrorIs(t, err, ErrNoPythonInterpreters) + assert.Error(t, err) } From 52bf7e388a80beb95d248dc623cfda3cf5d5e137 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 15:25:56 +0100 Subject: [PATCH 010/105] acc: Propagate user's UV_CACHE_DIR to tests (#2239) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a speed up in 0.5s but it is still 4.4s, so something else is slow there. Benchmarking bundle/templates/experimental-jobs-as-code: ``` # Without UV_CACHE_DIR ~/work/cli/acceptance/bundle/templates/experimental-jobs-as-code % hyperfine --warmup 2 'testme -count=1' Benchmark 1: testme -count=1 Time (mean ± σ): 4.950 s ± 0.079 s [User: 2.730 s, System: 8.524 s] Range (min … max): 4.838 s … 5.076 s 10 runs # With UV_CACHE_DIR ~/work/cli/acceptance/bundle/templates/experimental-jobs-as-code % hyperfine --warmup 2 'testme -count=1' Benchmark 1: testme -count=1 Time (mean ± σ): 4.410 s ± 0.049 s [User: 2.669 s, System: 8.710 s] Range (min … max): 4.324 s … 4.467 s 10 runs ``` --- acceptance/acceptance_test.go | 17 +++++++++++++++++ .../templates/experimental-jobs-as-code/script | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e48bd9908..47295b47a 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -100,6 +100,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // Prevent CLI from downloading terraform in each test: t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) + // Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically + uvCache := getUVDefaultCacheDir(t) + t.Setenv("UV_CACHE_DIR", uvCache) + ctx := context.Background() cloudEnv := os.Getenv("CLOUD_ENV") @@ -486,3 +490,16 @@ func ListDir(t *testing.T, src string) []string { } return files } + +func getUVDefaultCacheDir(t *testing.T) string { + // According to uv docs https://docs.astral.sh/uv/concepts/cache/#caching-in-continuous-integration + // the default cache directory is + // "A system-appropriate cache directory, e.g., $XDG_CACHE_HOME/uv or $HOME/.cache/uv on Unix and %LOCALAPPDATA%\uv\cache on Windows" + cacheDir, err := os.UserCacheDir() + require.NoError(t, err) + if runtime.GOOS == "windows" { + return cacheDir + "\\uv\\cache" + } else { + return cacheDir + "/uv" + } +} diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index af28b9d0a..0223b3326 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -3,7 +3,7 @@ trace $CLI bundle init experimental-jobs-as-code --config-file ./input.json --ou cd output/my_jobs_as_code # silence uv output because it's non-deterministic -uv sync 2> /dev/null +uv sync -q # remove version constraint because it always creates a warning on dev builds cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new From 67d1413db5b84df6643f3c1571abae13da14c6e2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 15:34:53 +0100 Subject: [PATCH 011/105] Add default regex for DEV_VERSION (#2241) ## Changes - Replace development version with $DEV_VERSION - Update experimental-jobs-as-code to make use of it. ## Tests - Existing tests. - Using this in https://github.com/databricks/cli/pull/2213 --- acceptance/acceptance_test.go | 1 + .../bundle/templates/experimental-jobs-as-code/output.txt | 2 ++ .../output/my_jobs_as_code/databricks.yml | 1 + .../bundle/templates/experimental-jobs-as-code/script | 4 ---- acceptance/selftest/output.txt | 4 ++++ acceptance/selftest/script | 3 +++ libs/testdiff/replacement.go | 7 +++++++ 7 files changed, 18 insertions(+), 4 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 47295b47a..5eb08f674 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -128,6 +128,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { testdiff.PrepareReplacementsUser(t, &repls, *user) testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient) testdiff.PrepareReplacementsUUID(t, &repls) + testdiff.PrepareReplacementsDevVersion(t, &repls) testDirs := getTests(t) require.NotEmpty(t, testDirs) diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt index 1aa8a94d5..10aca003e 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt @@ -10,6 +10,8 @@ Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> $CLI bundle validate -t dev --output json +Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: $DEV_VERSION + { "jobs": { "my_jobs_as_code_job": { diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml index fd87aa381..a1a93d95c 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml @@ -3,6 +3,7 @@ bundle: name: my_jobs_as_code uuid: + databricks_cli_version: ">= 0.238.0" experimental: python: diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index 0223b3326..10188aabd 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -5,10 +5,6 @@ cd output/my_jobs_as_code # silence uv output because it's non-deterministic uv sync -q -# remove version constraint because it always creates a warning on dev builds -cat databricks.yml | grep -v databricks_cli_version > databricks.yml.new -mv databricks.yml.new databricks.yml - trace $CLI bundle validate -t dev --output json | jq ".resources" rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt index 9fdfbc1e7..91aa8c33e 100644 --- a/acceptance/selftest/output.txt +++ b/acceptance/selftest/output.txt @@ -33,3 +33,7 @@ $TMPDIR/subdir/a/b/c 1234 CUSTOM_NUMBER_REGEX 123456 + +=== Testing --version +>>> $CLI --version +Databricks CLI v$DEV_VERSION diff --git a/acceptance/selftest/script b/acceptance/selftest/script index 665726167..bccf30e71 100644 --- a/acceptance/selftest/script +++ b/acceptance/selftest/script @@ -24,3 +24,6 @@ printf "\n=== Custom regex can be specified in [[Repl]] section\n" echo 1234 echo 12345 echo 123456 + +printf "\n=== Testing --version" +trace $CLI --version diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index b512374a3..40e7e72b4 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -23,6 +23,8 @@ var ( uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) numIdRegex = regexp.MustCompile(`[0-9]{3,}`) privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`) + // Version could v0.0.0-dev+21e1aacf518a or just v0.0.0-dev (the latter is currently the case on Windows) + devVersionRegex = regexp.MustCompile(`0\.0\.0-dev(\+[a-f0-9]{10,16})?`) ) type Replacement struct { @@ -211,3 +213,8 @@ func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsC t.Helper() r.append(privatePathRegex, "/tmp/.../$3") } + +func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) { + t.Helper() + r.append(devVersionRegex, "$$DEV_VERSION") +} From be908ee1a17abe36c573a24ac83033243c154379 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 27 Jan 2025 16:28:33 +0100 Subject: [PATCH 012/105] Add acceptance test for 'experimental.scripts' (#2240) --- acceptance/bundle/scripts/databricks.yml | 11 +++++ acceptance/bundle/scripts/myscript.py | 8 ++++ acceptance/bundle/scripts/output.txt | 52 ++++++++++++++++++++++++ acceptance/bundle/scripts/script | 3 ++ acceptance/server_test.go | 4 ++ bundle/scripts/scripts_test.go | 51 ----------------------- 6 files changed, 78 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/scripts/databricks.yml create mode 100644 acceptance/bundle/scripts/myscript.py create mode 100644 acceptance/bundle/scripts/output.txt create mode 100644 acceptance/bundle/scripts/script delete mode 100644 bundle/scripts/scripts_test.go diff --git a/acceptance/bundle/scripts/databricks.yml b/acceptance/bundle/scripts/databricks.yml new file mode 100644 index 000000000..6421e2b59 --- /dev/null +++ b/acceptance/bundle/scripts/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: scripts + +experimental: + scripts: + preinit: "python3 ./myscript.py $EXITCODE preinit" + postinit: "python3 ./myscript.py 0 postinit" + prebuild: "python3 ./myscript.py 0 prebuild" + postbuild: "python3 ./myscript.py 0 postbuild" + predeploy: "python3 ./myscript.py 0 predeploy" + postdeploy: "python3 ./myscript.py 0 postdeploy" diff --git a/acceptance/bundle/scripts/myscript.py b/acceptance/bundle/scripts/myscript.py new file mode 100644 index 000000000..d10f497e1 --- /dev/null +++ b/acceptance/bundle/scripts/myscript.py @@ -0,0 +1,8 @@ +import sys + +info = " ".join(sys.argv[1:]) +sys.stderr.write(f"from myscript.py {info}: hello stderr!\n") +sys.stdout.write(f"from myscript.py {info}: hello stdout!\n") + +exitcode = int(sys.argv[1]) +sys.exit(exitcode) diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt new file mode 100644 index 000000000..ec5978380 --- /dev/null +++ b/acceptance/bundle/scripts/output.txt @@ -0,0 +1,52 @@ + +>>> EXITCODE=0 errcode $CLI bundle validate +Executing 'preinit' script +from myscript.py 0 preinit: hello stdout! +from myscript.py 0 preinit: hello stderr! +Executing 'postinit' script +from myscript.py 0 postinit: hello stdout! +from myscript.py 0 postinit: hello stderr! +Name: scripts +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/scripts/default + +Validation OK! + +>>> EXITCODE=1 errcode $CLI bundle validate +Executing 'preinit' script +from myscript.py 1 preinit: hello stdout! +from myscript.py 1 preinit: hello stderr! +Error: failed to execute script: exit status 1 + +Name: scripts + +Found 1 error + +Exit code: 1 + +>>> EXITCODE=0 errcode $CLI bundle deploy +Executing 'preinit' script +from myscript.py 0 preinit: hello stdout! +from myscript.py 0 preinit: hello stderr! +Executing 'postinit' script +from myscript.py 0 postinit: hello stdout! +from myscript.py 0 postinit: hello stderr! +Executing 'prebuild' script +from myscript.py 0 prebuild: hello stdout! +from myscript.py 0 prebuild: hello stderr! +Executing 'postbuild' script +from myscript.py 0 postbuild: hello stdout! +from myscript.py 0 postbuild: hello stderr! +Executing 'predeploy' script +from myscript.py 0 predeploy: hello stdout! +from myscript.py 0 predeploy: hello stderr! +Error: unable to deploy to /Workspace/Users/$USERNAME/.bundle/scripts/default/state as $USERNAME. +Please make sure the current user or one of their groups is listed under the permissions of this bundle. +For assistance, contact the owners of this project. +They may need to redeploy the bundle to apply the new permissions. +Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. + + +Exit code: 1 diff --git a/acceptance/bundle/scripts/script b/acceptance/bundle/scripts/script new file mode 100644 index 000000000..de07d277e --- /dev/null +++ b/acceptance/bundle/scripts/script @@ -0,0 +1,3 @@ +trace EXITCODE=0 errcode $CLI bundle validate +trace EXITCODE=1 errcode $CLI bundle validate +trace EXITCODE=0 errcode $CLI bundle deploy diff --git a/acceptance/server_test.go b/acceptance/server_test.go index eb8cbb24a..dbc55c03f 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -146,4 +146,8 @@ func AddHandlers(server *TestServer) { }, }, nil }) + + server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, error) { + return "{}", nil + }) } diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go deleted file mode 100644 index 0c92bc2c3..000000000 --- a/bundle/scripts/scripts_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package scripts - -import ( - "bufio" - "context" - "strings" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/exec" - "github.com/stretchr/testify/require" -) - -func TestExecutesHook(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Experimental: &config.Experimental{ - Scripts: map[config.ScriptHook]config.Command{ - config.ScriptPreBuild: "echo 'Hello'", - }, - }, - }, - } - - executor, err := exec.NewCommandExecutor(b.BundleRootPath) - require.NoError(t, err) - _, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild) - require.NoError(t, err) - - reader := bufio.NewReader(out) - line, err := reader.ReadString('\n') - - require.NoError(t, err) - require.Equal(t, "Hello", strings.TrimSpace(line)) -} - -func TestExecuteMutator(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Experimental: &config.Experimental{ - Scripts: map[config.ScriptHook]config.Command{ - config.ScriptPreBuild: "echo 'Hello'", - }, - }, - }, - } - - diags := bundle.Apply(context.Background(), b, Execute(config.ScriptPreInit)) - require.NoError(t, diags.Error()) -} From 60709e3d48a711b931d341196120f4450ee78499 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 11:15:32 +0100 Subject: [PATCH 013/105] acc: Restore unexpected output error (#2243) ## Changes Restore original behaviour of acceptance tests: any unaccounted for files trigger an error (not just those that start with "out"). This got changed in https://github.com/databricks/cli/pull/2146/files#diff-2bb968d823f4afb825e1dcea2879bdbdedf2b7c15d4e77f47905691b14246a04L196 which started only checking files starting with "out*" and skipping everything else. ## Tests Existing tests. --- acceptance/acceptance_test.go | 1 + acceptance/bundle/git-permerror/script | 3 ++- acceptance/bundle/syncroot/dotdot-git/script | 4 +++- acceptance/bundle/syncroot/dotdot-git/test.toml | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/syncroot/dotdot-git/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 5eb08f674..2d67fb269 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -245,6 +245,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := outputs[relPath]; ok { continue } + t.Errorf("Unexpected output: %s", relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: diff --git a/acceptance/bundle/git-permerror/script b/acceptance/bundle/git-permerror/script index 782cbf5bc..3a9b4db24 100644 --- a/acceptance/bundle/git-permerror/script +++ b/acceptance/bundle/git-permerror/script @@ -22,4 +22,5 @@ trace chmod 000 .git/config errcode trace $CLI bundle validate -o json | jq .bundle.git errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git -rm -fr .git +cd .. +rm -fr myrepo diff --git a/acceptance/bundle/syncroot/dotdot-git/script b/acceptance/bundle/syncroot/dotdot-git/script index 0706a1d5e..278e77101 100644 --- a/acceptance/bundle/syncroot/dotdot-git/script +++ b/acceptance/bundle/syncroot/dotdot-git/script @@ -3,4 +3,6 @@ mkdir myrepo cd myrepo cp ../databricks.yml . git-repo-init -$CLI bundle validate | sed 's/\\\\/\//g' +errcode $CLI bundle validate +cd .. +rm -fr myrepo diff --git a/acceptance/bundle/syncroot/dotdot-git/test.toml b/acceptance/bundle/syncroot/dotdot-git/test.toml new file mode 100644 index 000000000..f57f83ee4 --- /dev/null +++ b/acceptance/bundle/syncroot/dotdot-git/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\\\\myrepo' +New = '/myrepo' From 11436faafe5361bd390fa04dc699807e31db6144 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 11:22:29 +0100 Subject: [PATCH 014/105] acc: Avoid reading and applying replacements on large files; validate utf8 (#2244) ## Changes - Do not start replacement / comparison if file is too large or not valid utf-8. - This helps to prevent replacements if there is accidentally a large binary (e.g. terraform). ## Tests Found this problem when working on https://github.com/databricks/cli/pull/2242 -- the tests tried to applied replacements on terraform binary and crashed. With this change, an error is reported instead. --- acceptance/acceptance_test.go | 51 ++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 2d67fb269..877c7239d 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -15,6 +15,7 @@ import ( "strings" "testing" "time" + "unicode/utf8" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" @@ -44,6 +45,7 @@ const ( EntryPointScript = "script" CleanupScript = "script.cleanup" PrepareScript = "script.prepare" + MaxFileSize = 100_000 ) var Scripts = map[string]bool{ @@ -257,15 +259,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { pathRef := filepath.Join(dirRef, relPath) pathNew := filepath.Join(dirNew, relPath) - bufRef, okRef := readIfExists(t, pathRef) - bufNew, okNew := readIfExists(t, pathNew) + bufRef, okRef := tryReading(t, pathRef) + bufNew, okNew := tryReading(t, pathNew) if !okRef && !okNew { - t.Errorf("Both files are missing: %s, %s", pathRef, pathNew) + t.Errorf("Both files are missing or have errors: %s, %s", pathRef, pathNew) return } - valueRef := testdiff.NormalizeNewlines(string(bufRef)) - valueNew := testdiff.NormalizeNewlines(string(bufNew)) + valueRef := testdiff.NormalizeNewlines(bufRef) + valueNew := testdiff.NormalizeNewlines(bufNew) // Apply replacements to the new value only. // The reference value is stored after applying replacements. @@ -323,14 +325,14 @@ func readMergedScriptContents(t *testing.T, dir string) string { cleanups := []string{} for { - x, ok := readIfExists(t, filepath.Join(dir, CleanupScript)) + x, ok := tryReading(t, filepath.Join(dir, CleanupScript)) if ok { - cleanups = append(cleanups, string(x)) + cleanups = append(cleanups, x) } - x, ok = readIfExists(t, filepath.Join(dir, PrepareScript)) + x, ok = tryReading(t, filepath.Join(dir, PrepareScript)) if ok { - prepares = append(prepares, string(x)) + prepares = append(prepares, x) } if dir == "" || dir == "." { @@ -417,16 +419,33 @@ func formatOutput(w io.Writer, err error) { } } -func readIfExists(t *testing.T, path string) ([]byte, bool) { - data, err := os.ReadFile(path) - if err == nil { - return data, true +func tryReading(t *testing.T, path string) (string, bool) { + info, err := os.Stat(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s: %s", path, err) + } + return "", false } - if !errors.Is(err, os.ErrNotExist) { - t.Fatalf("%s: %s", path, err) + if info.Size() > MaxFileSize { + t.Errorf("%s: ignoring, too large: %d", path, info.Size()) + return "", false } - return []byte{}, false + + data, err := os.ReadFile(path) + if err != nil { + // already checked ErrNotExist above + t.Errorf("%s: %s", path, err) + return "", false + } + + if !utf8.Valid(data) { + t.Errorf("%s: not valid utf-8", path) + return "", false + } + + return string(data), true } func CopyDir(src, dst string, inputs, outputs map[string]bool) error { From 3ffac800071a397763bddb49e22c1aca4f55573c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 11:23:44 +0100 Subject: [PATCH 015/105] acc: Use real terraform when CLOUD_ENV is set (#2245) ## Changes - If CLOUD_ENV is set to do not override with dummy value. This allows running acceptance tests as integration tests. - Needed for https://github.com/databricks/cli/pull/2242 ## Tests Manually run the test suite against dogfood. `CLOUD_ENV=aws go test ./acceptance` --- acceptance/acceptance_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 877c7239d..b4b27f201 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -99,9 +99,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { repls.SetPath(tempHomeDir, "$TMPHOME") t.Logf("$TMPHOME=%v", tempHomeDir) - // Prevent CLI from downloading terraform in each test: - t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) - // Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically uvCache := getUVDefaultCacheDir(t) t.Setenv("UV_CACHE_DIR", uvCache) @@ -119,6 +116,9 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { homeDir := t.TempDir() // Do not read user's ~/.databrickscfg t.Setenv(env.HomeEnvVar(), homeDir) + + // Prevent CLI from downloading terraform in each test: + t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) } workspaceClient, err := databricks.NewWorkspaceClient() From 65e4f79dfec84f45689ec3241da62ca3660112e6 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:24:23 +0530 Subject: [PATCH 016/105] Switch to using `[` from `<` in text replacements (#2224) ## Changes Noticed this when working on https://github.com/databricks/cli/pull/2221. `<` is a special HTML character that is encoded during text replacement when using `AssertEqualTexts`. ## Tests N/A --- .../dbt-sql/output/my_dbt_sql/databricks.yml | 2 +- .../output/my_default_python/databricks.yml | 2 +- .../my_default_python/scratch/exploration.ipynb | 2 +- .../output/my_default_python/src/dlt_pipeline.ipynb | 6 +++--- .../output/my_default_python/src/notebook.ipynb | 4 ++-- .../default-sql/output/my_default_sql/databricks.yml | 2 +- .../output/my_default_sql/scratch/exploration.ipynb | 2 +- .../output/my_jobs_as_code/databricks.yml | 2 +- .../output/my_jobs_as_code/src/notebook.ipynb | 4 ++-- .../bundle/testdata/default_python/bundle_deploy.txt | 2 +- .../testdata/default_python/bundle_summary.txt | 12 ++++++------ libs/testdiff/replacement.go | 4 ++-- libs/testdiff/replacement_test.go | 4 ++-- 13 files changed, 24 insertions(+), 24 deletions(-) diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml index 1962bc543..cdf3704b9 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml @@ -3,7 +3,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_dbt_sql - uuid: + uuid: [UUID] include: - resources/*.yml diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml index 9deca9cf5..3fa777219 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml @@ -2,7 +2,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_default_python - uuid: + uuid: [UUID] include: - resources/*.yml diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb index 3b2fef4b4..a12773d4e 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb +++ b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb @@ -20,7 +20,7 @@ "rowLimit": 10000 }, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb index 36e993af7..8a02183e7 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb +++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb @@ -6,7 +6,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } @@ -24,7 +24,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } @@ -47,7 +47,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb index 0d560443b..472ccb219 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb +++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb @@ -6,7 +6,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } @@ -37,7 +37,7 @@ "rowLimit": 10000 }, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml index ab857287e..16292bc84 100644 --- a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml @@ -2,7 +2,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_default_sql - uuid: + uuid: [UUID] include: - resources/*.yml diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb index c3fd072e5..f3976c1de 100644 --- a/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb @@ -7,7 +7,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml index a1a93d95c..54e69a256 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml @@ -2,7 +2,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_jobs_as_code - uuid: + uuid: [UUID] databricks_cli_version: ">= 0.238.0" experimental: diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb index 9bc3f1560..227c7cc55 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/src/notebook.ipynb @@ -6,7 +6,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } @@ -37,7 +37,7 @@ "rowLimit": 10000 }, "inputWidgets": {}, - "nuid": "", + "nuid": "[UUID]", "showTitle": false, "title": "" } diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt index eef0b79b3..d7b8cede9 100644 --- a/integration/bundle/testdata/default_python/bundle_deploy.txt +++ b/integration/bundle/testdata/default_python/bundle_deploy.txt @@ -1,5 +1,5 @@ Building project_name_$UNIQUE_PRJ... -Uploading project_name_$UNIQUE_PRJ-0.0.1+.-py3-none-any.whl... +Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files... Deploying resources... Updating deployment state... diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 318cd2543..88ccdc496 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -16,7 +16,7 @@ "enabled": false } }, - "uuid": "" + "uuid": "[UUID]" }, "include": [ "resources/project_name_$UNIQUE_PRJ.job.yml", @@ -74,7 +74,7 @@ ] }, "format": "MULTI_TASK", - "id": "", + "id": "[NUMID]", "job_clusters": [ { "job_cluster_key": "job_cluster", @@ -141,7 +141,7 @@ "unit": "DAYS" } }, - "url": "$DATABRICKS_URL/jobs/?o=" + "url": "$DATABRICKS_URL/jobs/[NUMID]?o=[NUMID]" } }, "pipelines": { @@ -155,7 +155,7 @@ "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" }, "development": true, - "id": "", + "id": "[UUID]", "libraries": [ { "notebook": { @@ -165,7 +165,7 @@ ], "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline", "target": "project_name_$UNIQUE_PRJ_dev", - "url": "$DATABRICKS_URL/pipelines/?o=" + "url": "$DATABRICKS_URL/pipelines/[UUID]?o=[NUMID]" } } }, @@ -183,4 +183,4 @@ "dev": "$USERNAME" } } -} \ No newline at end of file +} diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 40e7e72b4..ce5476a57 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -201,12 +201,12 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam. func PrepareReplacementsUUID(t testutil.TestingT, r *ReplacementsContext) { t.Helper() - r.append(uuidRegex, "") + r.append(uuidRegex, "[UUID]") } func PrepareReplacementsNumber(t testutil.TestingT, r *ReplacementsContext) { t.Helper() - r.append(numIdRegex, "") + r.append(numIdRegex, "[NUMID]") } func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsContext) { diff --git a/libs/testdiff/replacement_test.go b/libs/testdiff/replacement_test.go index de247c03e..1b6c5fe2d 100644 --- a/libs/testdiff/replacement_test.go +++ b/libs/testdiff/replacement_test.go @@ -25,7 +25,7 @@ func TestReplacement_UUID(t *testing.T) { PrepareReplacementsUUID(t, &repls) - assert.Equal(t, "", repls.Replace("123e4567-e89b-12d3-a456-426614174000")) + assert.Equal(t, "[UUID]", repls.Replace("123e4567-e89b-12d3-a456-426614174000")) } func TestReplacement_Number(t *testing.T) { @@ -34,7 +34,7 @@ func TestReplacement_Number(t *testing.T) { PrepareReplacementsNumber(t, &repls) assert.Equal(t, "12", repls.Replace("12")) - assert.Equal(t, "", repls.Replace("123")) + assert.Equal(t, "[NUMID]", repls.Replace("123")) } func TestReplacement_TemporaryDirectory(t *testing.T) { From 5971bd5c1ac0997a88f56dd4ccc88acf501e5267 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 15:00:41 +0100 Subject: [PATCH 017/105] acc: Disable git hooks (#2249) Otherwise hooks from universe and custom hooks run in tests. --- acceptance/script.prepare | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/script.prepare b/acceptance/script.prepare index b814a1260..ca47cdbff 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -39,6 +39,7 @@ git-repo-init() { git config core.autocrlf false git config user.name "Tester" git config user.email "tester@databricks.com" + git config core.hooksPath no-hooks git add databricks.yml git commit -qm 'Add databricks.yml' } From 025622540809702994aaefdb1e387a6552c00afa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 15:12:47 +0100 Subject: [PATCH 018/105] acc: Exclude secrets from replacements (#2250) They should never be printed by CLI anyway. --- libs/testdiff/replacement.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index ce5476a57..7077e611b 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -146,25 +146,16 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") - r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL") r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") - r.Set(w.Config.Token, "$DATABRICKS_TOKEN") r.Set(w.Config.Username, "$DATABRICKS_USERNAME") - r.Set(w.Config.Password, "$DATABRICKS_PASSWORD") r.SetPath(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") - r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS") r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") - r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET") - // r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID") r.Set(w.Config.AzureClientID, testerName) r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") - r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL") - r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN") r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") - r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET") r.SetPath(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") // This is set to words like "path" that happen too frequently // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") From 4ba222ab3632c45e488e88d3c54b6e05cbfe441b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 28 Jan 2025 15:22:56 +0100 Subject: [PATCH 019/105] Fix env_overrides not to use variables in workspace.profile (#2251) This does not work when this test is run against cloud. Needed for https://github.com/databricks/cli/pull/2242 --- acceptance/bundle/variables/env_overrides/databricks.yml | 7 ++++--- acceptance/bundle/variables/env_overrides/output.txt | 3 ++- acceptance/bundle/variables/env_overrides/script | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/variables/env_overrides/databricks.yml b/acceptance/bundle/variables/env_overrides/databricks.yml index 560513bc3..e5fc7fcc4 100644 --- a/acceptance/bundle/variables/env_overrides/databricks.yml +++ b/acceptance/bundle/variables/env_overrides/databricks.yml @@ -18,12 +18,13 @@ variables: description: variable with lookup lookup: cluster_policy: wrong-cluster-policy + + result: + default: ${var.a} ${var.b} + bundle: name: test bundle -workspace: - profile: ${var.a} ${var.b} - targets: env-with-single-variable-override: variables: diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index 1ee9ef625..06e6e518b 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -36,5 +36,6 @@ Exit code: 1 "b": "prod-b", "d": "4321", "e": "1234", - "f": "9876" + "f": "9876", + "result": "default-a prod-b" } diff --git a/acceptance/bundle/variables/env_overrides/script b/acceptance/bundle/variables/env_overrides/script index 30919fd8a..3965d1564 100644 --- a/acceptance/bundle/variables/env_overrides/script +++ b/acceptance/bundle/variables/env_overrides/script @@ -1,6 +1,6 @@ -trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile -trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile -trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile +trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .variables.result.value +trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .variables.result.value +trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .variables.result.value trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment trace errcode $CLI bundle validate -t env-using-an-undefined-variable trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)' From 099e9bed0f2250e3dcece80e6e64d8873c75e74d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jan 2025 15:34:44 +0100 Subject: [PATCH 020/105] Upgrade TF provider to 1.64.1 (#2247) ## Changes - Added support for `no_compute` in Apps - Added support for `run_as_repl` for job tasks --- bundle/internal/tf/codegen/schema/version.go | 2 +- .../internal/tf/schema/data_source_serving_endpoints.go | 8 ++++---- bundle/internal/tf/schema/resource_app.go | 1 + bundle/internal/tf/schema/resource_job.go | 2 ++ bundle/internal/tf/schema/resource_model_serving.go | 8 ++++---- bundle/internal/tf/schema/resource_recipient.go | 1 + bundle/internal/tf/schema/root.go | 2 +- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 677b8fc10..393afd6ed 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.63.0" +const ProviderVersion = "1.64.1" diff --git a/bundle/internal/tf/schema/data_source_serving_endpoints.go b/bundle/internal/tf/schema/data_source_serving_endpoints.go index bdfd778e0..973989216 100644 --- a/bundle/internal/tf/schema/data_source_serving_endpoints.go +++ b/bundle/internal/tf/schema/data_source_serving_endpoints.go @@ -3,7 +3,7 @@ package schema type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInputPii struct { - Behavior string `json:"behavior"` + Behavior string `json:"behavior,omitempty"` } type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInput struct { @@ -14,7 +14,7 @@ type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsInput struct { } type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsOutputPii struct { - Behavior string `json:"behavior"` + Behavior string `json:"behavior,omitempty"` } type DataSourceServingEndpointsEndpointsAiGatewayGuardrailsOutput struct { @@ -87,8 +87,8 @@ type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelDatabri type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct { PrivateKey string `json:"private_key,omitempty"` PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"` - ProjectId string `json:"project_id,omitempty"` - Region string `json:"region,omitempty"` + ProjectId string `json:"project_id"` + Region string `json:"region"` } type DataSourceServingEndpointsEndpointsConfigServedEntitiesExternalModelOpenaiConfig struct { diff --git a/bundle/internal/tf/schema/resource_app.go b/bundle/internal/tf/schema/resource_app.go index 14c93b793..cbce5ab0e 100644 --- a/bundle/internal/tf/schema/resource_app.go +++ b/bundle/internal/tf/schema/resource_app.go @@ -91,6 +91,7 @@ type ResourceApp struct { DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` Description string `json:"description,omitempty"` Name string `json:"name"` + NoCompute bool `json:"no_compute,omitempty"` PendingDeployment *ResourceAppPendingDeployment `json:"pending_deployment,omitempty"` Resources []ResourceAppResources `json:"resources,omitempty"` ServicePrincipalClientId string `json:"service_principal_client_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 63c8aeb7b..da277b5c1 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -904,6 +904,7 @@ type ResourceJobTaskForEachTaskTaskSparkJarTask struct { JarUri string `json:"jar_uri,omitempty"` MainClassName string `json:"main_class_name,omitempty"` Parameters []string `json:"parameters,omitempty"` + RunAsRepl bool `json:"run_as_repl,omitempty"` } type ResourceJobTaskForEachTaskTaskSparkPythonTask struct { @@ -1299,6 +1300,7 @@ type ResourceJobTaskSparkJarTask struct { JarUri string `json:"jar_uri,omitempty"` MainClassName string `json:"main_class_name,omitempty"` Parameters []string `json:"parameters,omitempty"` + RunAsRepl bool `json:"run_as_repl,omitempty"` } type ResourceJobTaskSparkPythonTask struct { diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 71cf8925d..2025de34c 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -3,7 +3,7 @@ package schema type ResourceModelServingAiGatewayGuardrailsInputPii struct { - Behavior string `json:"behavior"` + Behavior string `json:"behavior,omitempty"` } type ResourceModelServingAiGatewayGuardrailsInput struct { @@ -14,7 +14,7 @@ type ResourceModelServingAiGatewayGuardrailsInput struct { } type ResourceModelServingAiGatewayGuardrailsOutputPii struct { - Behavior string `json:"behavior"` + Behavior string `json:"behavior,omitempty"` } type ResourceModelServingAiGatewayGuardrailsOutput struct { @@ -94,8 +94,8 @@ type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServing type ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct { PrivateKey string `json:"private_key,omitempty"` PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"` - ProjectId string `json:"project_id,omitempty"` - Region string `json:"region,omitempty"` + ProjectId string `json:"project_id"` + Region string `json:"region"` } type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct { diff --git a/bundle/internal/tf/schema/resource_recipient.go b/bundle/internal/tf/schema/resource_recipient.go index 91de4df76..4c8f2c7e7 100644 --- a/bundle/internal/tf/schema/resource_recipient.go +++ b/bundle/internal/tf/schema/resource_recipient.go @@ -29,6 +29,7 @@ type ResourceRecipient struct { CreatedAt int `json:"created_at,omitempty"` CreatedBy string `json:"created_by,omitempty"` DataRecipientGlobalMetastoreId string `json:"data_recipient_global_metastore_id,omitempty"` + ExpirationTime int `json:"expiration_time,omitempty"` Id string `json:"id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 7dd3f9210..2ac852355 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.63.0" +const ProviderVersion = "1.64.1" func NewRoot() *Root { return &Root{ From 413ca5c13471d007edc607a815850f0b31dc32cb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jan 2025 18:17:37 +0100 Subject: [PATCH 021/105] Do not wait for app compute to start on `bundle deploy` (#2144) ## Changes This allows DABs to avoid waiting for the compute to start when app is initially created as part of "bundle deploy" which significantly improves deploy time. Always set no_compute to true for apps ## Tests Covered by `TestDeployBundleWithApp`, currently fails until TF provider is upgraded to the version supporting `no_compute` option --- bundle/apps/slow_deploy_message.go | 29 ------------------- bundle/deploy/terraform/tfdyn/convert_app.go | 6 ++++ .../terraform/tfdyn/convert_app_test.go | 2 ++ bundle/phases/deploy.go | 1 - integration/bundle/apps_test.go | 11 ++++--- .../bundle/testdata/apps/bundle_deploy.txt | 1 - 6 files changed, 13 insertions(+), 37 deletions(-) delete mode 100644 bundle/apps/slow_deploy_message.go diff --git a/bundle/apps/slow_deploy_message.go b/bundle/apps/slow_deploy_message.go deleted file mode 100644 index 87275980a..000000000 --- a/bundle/apps/slow_deploy_message.go +++ /dev/null @@ -1,29 +0,0 @@ -package apps - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" -) - -type slowDeployMessage struct{} - -// TODO: needs to be removed when when no_compute option becomes available in TF provider and used in DABs -// See https://github.com/databricks/cli/pull/2144 -func (v *slowDeployMessage) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if len(b.Config.Resources.Apps) > 0 { - cmdio.LogString(ctx, "Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning.") - } - - return nil -} - -func (v *slowDeployMessage) Name() string { - return "apps.SlowDeployMessage" -} - -func SlowDeployMessage() bundle.Mutator { - return &slowDeployMessage{} -} diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index dcba0809b..b3d599f15 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -38,6 +38,12 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out return err } + // We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created. + vout, err = dyn.Set(vout, "no_compute", dyn.V(true)) + if err != nil { + return err + } + // Add the converted resource to the output. out.App[key] = vout.AsAny() diff --git a/bundle/deploy/terraform/tfdyn/convert_app_test.go b/bundle/deploy/terraform/tfdyn/convert_app_test.go index be8152cc6..cdf56f8ed 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_app_test.go @@ -63,6 +63,7 @@ func TestConvertApp(t *testing.T) { assert.Equal(t, map[string]any{ "description": "app description", "name": "app_id", + "no_compute": true, "resources": []any{ map[string]any{ "name": "job1", @@ -136,6 +137,7 @@ func TestConvertAppWithNoDescription(t *testing.T) { assert.Equal(t, map[string]any{ "name": "app_id", "description": "", // Due to Apps API always returning a description field, we set it in the output as well to avoid permanent TF drift + "no_compute": true, "resources": []any{ map[string]any{ "name": "job1", diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b59ce9f89..c6ec04962 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -130,7 +130,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { // mutators need informed consent if they are potentially destructive. deployCore := bundle.Defer( bundle.Seq( - apps.SlowDeployMessage(), bundle.LogString("Deploying resources..."), terraform.Apply(), ), diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go index 23cd784be..01ab52e90 100644 --- a/integration/bundle/apps_test.go +++ b/integration/bundle/apps_test.go @@ -18,12 +18,6 @@ import ( func TestDeployBundleWithApp(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) - // TODO: should only skip app run when app can be created with no_compute option. - if testing.Short() { - t.Log("Skip the app creation and run in short mode") - return - } - if testutil.GetCloud(t) == testutil.GCP { t.Skip("Skipping test for GCP cloud because /api/2.0/apps is temporarily unavailable there.") } @@ -106,6 +100,11 @@ env: - name: JOB_ID value: "%d"`, job.JobId)) + if testing.Short() { + t.Log("Skip the app run in short mode") + return + } + // Try to run the app _, out := runResourceWithStderr(t, ctx, root, "test_app") require.Contains(t, out, app.Url) diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt index b077f327d..211164174 100644 --- a/integration/bundle/testdata/apps/bundle_deploy.txt +++ b/integration/bundle/testdata/apps/bundle_deploy.txt @@ -1,5 +1,4 @@ Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files... -Note: Databricks apps included in this bundle may increase initial deployment time due to compute provisioning. Deploying resources... Updating deployment state... Deployment complete! From 124515e8d2105a3d2ec071dadcb30bf792ba9cad Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:12:21 +0530 Subject: [PATCH 022/105] Move TestServer from acceptance to libs/testserver (#2255) ## Changes Just a move, no changes. As recommended here: https://github.com/databricks/cli/pull/2226#discussion_r1932152627 ## Tests N/A --- acceptance/acceptance_test.go | 3 +- acceptance/cmd_server_test.go | 3 +- acceptance/server_test.go | 56 +++---------------------------- libs/testserver/server.go | 63 +++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 libs/testserver/server.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index b4b27f201..91ad09e9e 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testdiff" + "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/require" ) @@ -107,7 +108,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cloudEnv := os.Getenv("CLOUD_ENV") if cloudEnv == "" { - server := StartServer(t) + server := testserver.New(t) AddHandlers(server) // Redirect API access to local server: t.Setenv("DATABRICKS_HOST", server.URL) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 28feec1bd..3f5a6356e 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -8,10 +8,11 @@ import ( "testing" "github.com/databricks/cli/internal/testcli" + "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/require" ) -func StartCmdServer(t *testing.T) *TestServer { +func StartCmdServer(t *testing.T) *testserver.Server { server := StartServer(t) server.Handle("/", func(r *http.Request) (any, error) { q := r.URL.Query() diff --git a/acceptance/server_test.go b/acceptance/server_test.go index dbc55c03f..66de5dcbf 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -1,73 +1,25 @@ package acceptance_test import ( - "encoding/json" "net/http" - "net/http/httptest" "testing" + "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" ) -type TestServer struct { - *httptest.Server - Mux *http.ServeMux -} - -type HandlerFunc func(r *http.Request) (any, error) - -func NewTestServer() *TestServer { - mux := http.NewServeMux() - server := httptest.NewServer(mux) - - return &TestServer{ - Server: server, - Mux: mux, - } -} - -func (s *TestServer) Handle(pattern string, handler HandlerFunc) { - s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, err := handler(r) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - - var respBytes []byte - - respString, ok := resp.(string) - if ok { - respBytes = []byte(respString) - } else { - respBytes, err = json.MarshalIndent(resp, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - - if _, err := w.Write(respBytes); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - }) -} - -func StartServer(t *testing.T) *TestServer { - server := NewTestServer() +func StartServer(t *testing.T) *testserver.Server { + server := testserver.New(t) t.Cleanup(func() { server.Close() }) return server } -func AddHandlers(server *TestServer) { +func AddHandlers(server *testserver.Server) { server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ diff --git a/libs/testserver/server.go b/libs/testserver/server.go new file mode 100644 index 000000000..10269af8f --- /dev/null +++ b/libs/testserver/server.go @@ -0,0 +1,63 @@ +package testserver + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/databricks/cli/internal/testutil" +) + +type Server struct { + *httptest.Server + Mux *http.ServeMux + + t testutil.TestingT +} + +func New(t testutil.TestingT) *Server { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + + return &Server{ + Server: server, + Mux: mux, + t: t, + } +} + +type HandlerFunc func(req *http.Request) (resp any, err error) + +func (s *Server) Close() { + s.Server.Close() +} + +func (s *Server) Handle(pattern string, handler HandlerFunc) { + s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { + resp, err := handler(r) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + + var respBytes []byte + + respString, ok := resp.(string) + if ok { + respBytes = []byte(respString) + } else { + respBytes, err = json.MarshalIndent(resp, "", " ") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + + if _, err := w.Write(respBytes); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) +} From 884b5f26ed148c431a0dfea6333bff9b293f8ed1 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:32:08 +0530 Subject: [PATCH 023/105] Set bundle auth configuration in command context (#2195) ## Changes This change is required to enable tracking execution time telemetry for bundle commands. In order to track execution time for the command generally, we need to have the databricks auth configuration available at this section of the code: https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/root/root.go#L99 In order to do this we can rely on the `configUsed` context key. Most commands rely on the `root.MustWorkspaceClient` function which automatically sets the client config in the `configUsed` context key. Bundle commands, however, do not do so. They instead store their workspace clients in the `&bundle.Bundle{}` object. With this PR, the `configUsed` context key will be set for all `bundle` commands. Functionally nothing changes. ## Tests Existing tests. Also manually verified that either `root.MustConfigureBundle` or `utils.ConfigureBundleWithVariables` is called for all bundle commands (except `bundle init`) thus ensuring this context key would be set for all bundle commands. refs for the functions: 1. `root.MustConfigureBundle`: https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/root/bundle.go#L88 2. `utils.ConfigureBundleWithVariables`: https://github.com/databricks/cli/blob/41bbd89257285707b3c3df9b9e5b92d6bcf8f1d1/cmd/bundle/utils/utils.go#L19 --------- Co-authored-by: Pieter Noordhuis --- bundle/bundle.go | 31 ++++--- .../mutator/initialize_workspace_client.go | 26 ------ bundle/phases/initialize.go | 1 - cmd/root/auth.go | 2 +- cmd/root/bundle.go | 16 ++++ cmd/root/bundle_test.go | 85 ++++++++----------- 6 files changed, 69 insertions(+), 92 deletions(-) delete mode 100644 bundle/config/mutator/initialize_workspace_client.go diff --git a/bundle/bundle.go b/bundle/bundle.go index e715b8b2c..9cb8916f5 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -72,6 +72,7 @@ type Bundle struct { // It can be initialized on demand after loading the configuration. clientOnce sync.Once client *databricks.WorkspaceClient + clientErr error // Files that are synced to the workspace.file_path Files []fileset.File @@ -134,23 +135,25 @@ func TryLoad(ctx context.Context) (*Bundle, error) { return Load(ctx, root) } -func (b *Bundle) InitializeWorkspaceClient() (*databricks.WorkspaceClient, error) { - client, err := b.Config.Workspace.Client() - if err != nil { - return nil, fmt.Errorf("cannot resolve bundle auth configuration: %w", err) - } - return client, nil +func (b *Bundle) WorkspaceClientE() (*databricks.WorkspaceClient, error) { + b.clientOnce.Do(func() { + var err error + b.client, err = b.Config.Workspace.Client() + if err != nil { + b.clientErr = fmt.Errorf("cannot resolve bundle auth configuration: %w", err) + } + }) + + return b.client, b.clientErr } func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient { - b.clientOnce.Do(func() { - var err error - b.client, err = b.InitializeWorkspaceClient() - if err != nil { - panic(err) - } - }) - return b.client + client, err := b.WorkspaceClientE() + if err != nil { + panic(err) + } + + return client } // SetWorkpaceClient sets the workspace client for this bundle. diff --git a/bundle/config/mutator/initialize_workspace_client.go b/bundle/config/mutator/initialize_workspace_client.go deleted file mode 100644 index 5c905f40c..000000000 --- a/bundle/config/mutator/initialize_workspace_client.go +++ /dev/null @@ -1,26 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" -) - -type initializeWorkspaceClient struct{} - -func InitializeWorkspaceClient() bundle.Mutator { - return &initializeWorkspaceClient{} -} - -func (m *initializeWorkspaceClient) Name() string { - return "InitializeWorkspaceClient" -} - -// Apply initializes the workspace client for the bundle. We do this here so -// downstream calls to b.WorkspaceClient() do not panic if there's an error in the -// auth configuration. -func (m *initializeWorkspaceClient) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - _, err := b.InitializeWorkspaceClient() - return diag.FromErr(err) -} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index c5b875196..afd6def3f 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -34,7 +34,6 @@ func Initialize() bundle.Mutator { // If it is an ancestor, this updates all paths to be relative to the sync root path. mutator.SyncInferRoot(), - mutator.InitializeWorkspaceClient(), mutator.PopulateCurrentUser(), mutator.LoadGitDetails(), diff --git a/cmd/root/auth.go b/cmd/root/auth.go index 49abfd414..4fcfbb4d8 100644 --- a/cmd/root/auth.go +++ b/cmd/root/auth.go @@ -209,7 +209,7 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error { if b != nil { ctx = context.WithValue(ctx, &configUsed, b.Config.Workspace.Config()) cmd.SetContext(ctx) - client, err := b.InitializeWorkspaceClient() + client, err := b.WorkspaceClientE() if err != nil { return err } diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index 8b98f2cf2..5842526f3 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -81,6 +81,22 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag // Configure the workspace profile if the flag has been set. diags = diags.Extend(configureProfile(cmd, b)) + if diags.HasError() { + return b, diags + } + + // Set the auth configuration in the command context. This can be used + // downstream to initialize a API client. + // + // Note that just initializing a workspace client and loading auth configuration + // is a fast operation. It does not perform network I/O or invoke processes (for example the Azure CLI). + client, err := b.WorkspaceClientE() + if err != nil { + return b, diags.Extend(diag.FromErr(err)) + } + ctx = context.WithValue(ctx, &configUsed, client.Config) + cmd.SetContext(ctx) + return b, diags } diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 1998b19e6..3517b02e4 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -8,7 +8,6 @@ import ( "runtime" "testing" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/internal/testutil" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -38,7 +37,7 @@ func emptyCommand(t *testing.T) *cobra.Command { return cmd } -func setupWithHost(t *testing.T, cmd *cobra.Command, host string) *bundle.Bundle { +func setupWithHost(t *testing.T, cmd *cobra.Command, host string) error { setupDatabricksCfg(t) rootPath := t.TempDir() @@ -51,12 +50,11 @@ workspace: err := os.WriteFile(filepath.Join(rootPath, "databricks.yml"), []byte(contents), 0o644) require.NoError(t, err) - b, diags := MustConfigureBundle(cmd) - require.NoError(t, diags.Error()) - return b + _, diags := MustConfigureBundle(cmd) + return diags.Error() } -func setupWithProfile(t *testing.T, cmd *cobra.Command, profile string) *bundle.Bundle { +func setupWithProfile(t *testing.T, cmd *cobra.Command, profile string) error { setupDatabricksCfg(t) rootPath := t.TempDir() @@ -69,29 +67,25 @@ workspace: err := os.WriteFile(filepath.Join(rootPath, "databricks.yml"), []byte(contents), 0o644) require.NoError(t, err) - b, diags := MustConfigureBundle(cmd) - require.NoError(t, diags.Error()) - return b + _, diags := MustConfigureBundle(cmd) + return diags.Error() } func TestBundleConfigureDefault(t *testing.T) { testutil.CleanupEnvironment(t) cmd := emptyCommand(t) - b := setupWithHost(t, cmd, "https://x.com") - - client, err := b.InitializeWorkspaceClient() + err := setupWithHost(t, cmd, "https://x.com") require.NoError(t, err) - assert.Equal(t, "https://x.com", client.Config.Host) + + assert.Equal(t, "https://x.com", ConfigUsed(cmd.Context()).Host) } func TestBundleConfigureWithMultipleMatches(t *testing.T) { testutil.CleanupEnvironment(t) cmd := emptyCommand(t) - b := setupWithHost(t, cmd, "https://a.com") - - _, err := b.InitializeWorkspaceClient() + err := setupWithHost(t, cmd, "https://a.com") assert.ErrorContains(t, err, "multiple profiles matched: PROFILE-1, PROFILE-2") } @@ -101,9 +95,8 @@ func TestBundleConfigureWithNonExistentProfileFlag(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("NOEXIST") require.NoError(t, err) - b := setupWithHost(t, cmd, "https://x.com") - _, err = b.InitializeWorkspaceClient() + err = setupWithHost(t, cmd, "https://x.com") assert.ErrorContains(t, err, "has no NOEXIST profile configured") } @@ -113,9 +106,8 @@ func TestBundleConfigureWithMismatchedProfile(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("PROFILE-1") require.NoError(t, err) - b := setupWithHost(t, cmd, "https://x.com") - _, err = b.InitializeWorkspaceClient() + err = setupWithHost(t, cmd, "https://x.com") assert.ErrorContains(t, err, "config host mismatch: profile uses host https://a.com, but CLI configured to use https://x.com") } @@ -125,12 +117,11 @@ func TestBundleConfigureWithCorrectProfile(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("PROFILE-1") require.NoError(t, err) - b := setupWithHost(t, cmd, "https://a.com") + err = setupWithHost(t, cmd, "https://a.com") - client, err := b.InitializeWorkspaceClient() require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "PROFILE-1", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile) } func TestBundleConfigureWithMismatchedProfileEnvVariable(t *testing.T) { @@ -138,9 +129,8 @@ func TestBundleConfigureWithMismatchedProfileEnvVariable(t *testing.T) { t.Setenv("DATABRICKS_CONFIG_PROFILE", "PROFILE-1") cmd := emptyCommand(t) - b := setupWithHost(t, cmd, "https://x.com") - _, err := b.InitializeWorkspaceClient() + err := setupWithHost(t, cmd, "https://x.com") assert.ErrorContains(t, err, "config host mismatch: profile uses host https://a.com, but CLI configured to use https://x.com") } @@ -151,12 +141,11 @@ func TestBundleConfigureWithProfileFlagAndEnvVariable(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("PROFILE-1") require.NoError(t, err) - b := setupWithHost(t, cmd, "https://a.com") - client, err := b.InitializeWorkspaceClient() + err = setupWithHost(t, cmd, "https://a.com") require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "PROFILE-1", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile) } func TestBundleConfigureProfileDefault(t *testing.T) { @@ -164,13 +153,12 @@ func TestBundleConfigureProfileDefault(t *testing.T) { // The profile in the databricks.yml file is used cmd := emptyCommand(t) - b := setupWithProfile(t, cmd, "PROFILE-1") - client, err := b.InitializeWorkspaceClient() + err := setupWithProfile(t, cmd, "PROFILE-1") require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "a", client.Config.Token) - assert.Equal(t, "PROFILE-1", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "a", ConfigUsed(cmd.Context()).Token) + assert.Equal(t, "PROFILE-1", ConfigUsed(cmd.Context()).Profile) } func TestBundleConfigureProfileFlag(t *testing.T) { @@ -180,13 +168,12 @@ func TestBundleConfigureProfileFlag(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("PROFILE-2") require.NoError(t, err) - b := setupWithProfile(t, cmd, "PROFILE-1") - client, err := b.InitializeWorkspaceClient() + err = setupWithProfile(t, cmd, "PROFILE-1") require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "b", client.Config.Token) - assert.Equal(t, "PROFILE-2", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token) + assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile) } func TestBundleConfigureProfileEnvVariable(t *testing.T) { @@ -195,13 +182,12 @@ func TestBundleConfigureProfileEnvVariable(t *testing.T) { // The DATABRICKS_CONFIG_PROFILE environment variable takes precedence over the profile in the databricks.yml file t.Setenv("DATABRICKS_CONFIG_PROFILE", "PROFILE-2") cmd := emptyCommand(t) - b := setupWithProfile(t, cmd, "PROFILE-1") - client, err := b.InitializeWorkspaceClient() + err := setupWithProfile(t, cmd, "PROFILE-1") require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "b", client.Config.Token) - assert.Equal(t, "PROFILE-2", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token) + assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile) } func TestBundleConfigureProfileFlagAndEnvVariable(t *testing.T) { @@ -212,13 +198,12 @@ func TestBundleConfigureProfileFlagAndEnvVariable(t *testing.T) { cmd := emptyCommand(t) err := cmd.Flag("profile").Value.Set("PROFILE-2") require.NoError(t, err) - b := setupWithProfile(t, cmd, "PROFILE-1") - client, err := b.InitializeWorkspaceClient() + err = setupWithProfile(t, cmd, "PROFILE-1") require.NoError(t, err) - assert.Equal(t, "https://a.com", client.Config.Host) - assert.Equal(t, "b", client.Config.Token) - assert.Equal(t, "PROFILE-2", client.Config.Profile) + assert.Equal(t, "https://a.com", ConfigUsed(cmd.Context()).Host) + assert.Equal(t, "b", ConfigUsed(cmd.Context()).Token) + assert.Equal(t, "PROFILE-2", ConfigUsed(cmd.Context()).Profile) } func TestTargetFlagFull(t *testing.T) { From 30f57d3b49aa30ecd9d12bcb416c883035c70f44 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:19:53 +0530 Subject: [PATCH 024/105] Add protos for bundle telemetry (#2209) ## Changes These types correspond to the telemetry protobufs defined in universe. ## Tests No tests are needed since this PR only adds the type bindings. --------- Co-authored-by: Pieter Noordhuis --- libs/telemetry/api.go | 31 +++++++++ libs/telemetry/protos/README.md | 2 + libs/telemetry/protos/bundle_deploy.go | 77 +++++++++++++++++++++ libs/telemetry/protos/bundle_init.go | 37 ++++++++++ libs/telemetry/protos/databricks_cli_log.go | 35 ++++++++++ libs/telemetry/protos/enum.go | 26 +++++++ libs/telemetry/protos/frontend_log.go | 22 ++++++ 7 files changed, 230 insertions(+) create mode 100644 libs/telemetry/api.go create mode 100644 libs/telemetry/protos/README.md create mode 100644 libs/telemetry/protos/bundle_deploy.go create mode 100644 libs/telemetry/protos/bundle_init.go create mode 100644 libs/telemetry/protos/databricks_cli_log.go create mode 100644 libs/telemetry/protos/enum.go create mode 100644 libs/telemetry/protos/frontend_log.go diff --git a/libs/telemetry/api.go b/libs/telemetry/api.go new file mode 100644 index 000000000..4e3f6c861 --- /dev/null +++ b/libs/telemetry/api.go @@ -0,0 +1,31 @@ +package telemetry + +// RequestBody is the request body type bindings for the /telemetry-ext API endpoint. +type RequestBody struct { + // Timestamp in millis for when the log was uploaded. + UploadTime int64 `json:"uploadTime"` + + // DO NOT USE. This is the legacy field for logging in usage logs (not lumberjack). + // We keep this around because the API endpoint works only if this field is serialized + // to an empty array. + Items []string `json:"items"` + + // JSON encoded strings containing the proto logs. Since it's represented as a + // string here, the values here end up being double JSON encoded in the final + // request body. + // + // Any logs here will be logged in our lumberjack tables as long as a corresponding + // protobuf is defined in universe. + ProtoLogs []string `json:"protoLogs"` +} + +// ResponseBody is the response body type bindings for the /telemetry-ext API endpoint. +type ResponseBody struct { + Errors []LogError `json:"errors"` + NumProtoSuccess int64 `json:"numProtoSuccess"` +} + +type LogError struct { + Message string `json:"message"` + ErrorType string `json:"errorType"` +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md new file mode 100644 index 000000000..7dcc75e17 --- /dev/null +++ b/libs/telemetry/protos/README.md @@ -0,0 +1,2 @@ +The types in this package are equivalent to the lumberjack protos defined in Universe. +You can find all lumberjack protos for the Databricks CLI in the `proto/logs/frontend/databricks_cli` directory. diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go new file mode 100644 index 000000000..f3c3a360b --- /dev/null +++ b/libs/telemetry/protos/bundle_deploy.go @@ -0,0 +1,77 @@ +package protos + +type BundleDeployEvent struct { + // UUID associated with the bundle itself. Set in the `bundle.uuid` field in the bundle configuration. + BundleUuid string `json:"bundle_uuid,omitempty"` + + ResourceCount int64 `json:"resource_count,omitempty"` + ResourceJobCount int64 `json:"resource_job_count,omitempty"` + ResourcePipelineCount int64 `json:"resource_pipeline_count,omitempty"` + ResourceModelCount int64 `json:"resource_model_count,omitempty"` + ResourceExperimentCount int64 `json:"resource_experiment_count,omitempty"` + ResourceModelServingEndpointCount int64 `json:"resource_model_serving_endpoint_count,omitempty"` + ResourceRegisteredModelCount int64 `json:"resource_registered_model_count,omitempty"` + ResourceQualityMonitorCount int64 `json:"resource_quality_monitor_count,omitempty"` + ResourceSchemaCount int64 `json:"resource_schema_count,omitempty"` + ResourceVolumeCount int64 `json:"resource_volume_count,omitempty"` + ResourceClusterCount int64 `json:"resource_cluster_count,omitempty"` + ResourceDashboardCount int64 `json:"resource_dashboard_count,omitempty"` + ResourceAppCount int64 `json:"resource_app_count,omitempty"` + + // IDs of resources managed by the bundle. Some resources like volumes or schemas + // do not expose a numerical or UUID identifier and are tracked by name. Those + // resources are not tracked here since the names are PII. + ResourceJobIDs []string `json:"resource_job_ids,omitempty"` + ResourcePipelineIDs []string `json:"resource_pipeline_ids,omitempty"` + ResourceClusterIDs []string `json:"resource_cluster_ids,omitempty"` + ResourceDashboardIDs []string `json:"resource_dashboard_ids,omitempty"` + + Experimental *BundleDeployExperimental `json:"experimental,omitempty"` +} + +// These metrics are experimental and are often added in an adhoc manner. There +// are no guarantees for these metrics and they maybe removed in the future without +// any notice. +type BundleDeployExperimental struct { + // Number of configuration files in the bundle. + ConfigurationFileCount int64 `json:"configuration_file_count,omitempty"` + + // Size in bytes of the Terraform state file + TerraformStateSizeBytes int64 `json:"terraform_state_size_bytes,omitempty"` + + // Number of variables in the bundle + VariableCount int64 `json:"variable_count,omitempty"` + ComplexVariableCount int64 `json:"complex_variable_count,omitempty"` + LookupVariableCount int64 `json:"lookup_variable_count,omitempty"` + + // Number of targets in the bundle + TargetCount int64 `json:"target_count,omitempty"` + + // Whether a field is set or not. If a configuration field is not present in this + // map then it is not tracked by this field. + // Keys are the full path of the field in the configuration tree. + // Examples: "bundle.terraform.exec_path", "bundle.git.branch" etc. + SetFields []BoolMapEntry `json:"set_fields,omitempty"` + + // Values for boolean configuration fields like `experimental.python_wheel_wrapper` + // We don't need to define protos to track boolean values and can simply write those + // values to this map to track them. + BoolValues []BoolMapEntry `json:"bool_values,omitempty"` + + BundleMode BundleMode `json:"bundle_mode,omitempty"` + + WorkspaceArtifactPathType BundleDeployArtifactPathType `json:"workspace_artifact_path_type,omitempty"` + + // Execution time per mutator for a selected subset of mutators. + BundleMutatorExecutionTimeMs []IntMapEntry `json:"bundle_mutator_execution_time_ms,omitempty"` +} + +type BoolMapEntry struct { + Key string `json:"key,omitempty"` + Value bool `json:"value,omitempty"` +} + +type IntMapEntry struct { + Key string `json:"key,omitempty"` + Value int64 `json:"value,omitempty"` +} diff --git a/libs/telemetry/protos/bundle_init.go b/libs/telemetry/protos/bundle_init.go new file mode 100644 index 000000000..47308a267 --- /dev/null +++ b/libs/telemetry/protos/bundle_init.go @@ -0,0 +1,37 @@ +package protos + +type BundleInitEvent struct { + // UUID associated with the DAB itself. This is serialized into the DAB + // when a user runs `databricks bundle init` and all subsequent deployments of + // that DAB can then be associated with this init event. + BundleUuid string `json:"bundle_uuid,omitempty"` + + // Name of the template initialized when the user ran `databricks bundle init` + // This is only populated when the template is a first party template like + // mlops-stacks or default-python. + TemplateName string `json:"template_name,omitempty"` + + // Arguments used by the user to initialize the template. Only enum + // values will be set here by the Databricks CLI. + // + // We use a generic map representation here because a bundle template's args are + // managed in the template itself and maintaining a copy typed schema for it here + // will be untenable in the long term. + TemplateEnumArgs []BundleInitTemplateEnumArg `json:"template_enum_args,omitempty"` +} + +type BundleInitTemplateEnumArg struct { + // Valid key values for the template. These correspond to the keys specified in + // the "properties" section of the `databricks_template_schema.json` file. + // + // Note: `databricks_template_schema.json` contains a JSON schema type specification + // for the arguments that the template accepts. + Key string `json:"key"` + + // Value that the user set for the field. This is only populated for properties + // that have the "enum" field specified in the JSON schema type specification. + // + // The Databricks CLI ensures that the value here is one of the "enum" values from + // the template specification. + Value string `json:"value"` +} diff --git a/libs/telemetry/protos/databricks_cli_log.go b/libs/telemetry/protos/databricks_cli_log.go new file mode 100644 index 000000000..9e4e59596 --- /dev/null +++ b/libs/telemetry/protos/databricks_cli_log.go @@ -0,0 +1,35 @@ +package protos + +type ExecutionContext struct { + // UUID generated by the CLI for every CLI command run. This is also set in the HTTP user + // agent under the key "cmd-exec-id" and can be used to correlate frontend_log table + // with the http_access_log table. + CmdExecID string `json:"cmd_exec_id,omitempty"` + + // Version of the Databricks CLI used. + Version string `json:"version,omitempty"` + + // Command that was run by the user. Eg: bundle_deploy, fs_cp etc. + Command string `json:"command,omitempty"` + + // Lowercase string name for the operating system. Same value + // as the one set in `runtime.GOOS` in Golang. + OperatingSystem string `json:"operating_system,omitempty"` + + // Version of DBR from which CLI is being run. + // Only set when the CLI is being run from a Databricks cluster. + DbrVersion string `json:"dbr_version,omitempty"` + + // If true, the CLI is being run from a Databricks notebook / cluster web terminal. + FromWebTerminal bool `json:"from_web_terminal,omitempty"` + + // Time taken for the CLI command to execute. + ExecutionTimeMs int64 `json:"execution_time_ms,omitempty"` + + // Exit code of the CLI command. + ExitCode int64 `json:"exit_code,omitempty"` +} + +type CliTestEvent struct { + Name DummyCliEnum `json:"name,omitempty"` +} diff --git a/libs/telemetry/protos/enum.go b/libs/telemetry/protos/enum.go new file mode 100644 index 000000000..7f6780cb6 --- /dev/null +++ b/libs/telemetry/protos/enum.go @@ -0,0 +1,26 @@ +package protos + +type DummyCliEnum string + +const ( + DummyCliEnumUnspecified DummyCliEnum = "DUMMY_CLI_ENUM_UNSPECIFIED" + DummyCliEnumValue1 DummyCliEnum = "VALUE1" + DummyCliEnumValue2 DummyCliEnum = "VALUE2" + DummyCliEnumValue3 DummyCliEnum = "VALUE3" +) + +type BundleMode string + +const ( + BundleModeUnspecified BundleMode = "TYPE_UNSPECIFIED" + BundleModeDevelopment BundleMode = "DEVELOPMENT" + BundleModeProduction BundleMode = "PRODUCTION" +) + +type BundleDeployArtifactPathType string + +const ( + BundleDeployArtifactPathTypeUnspecified BundleDeployArtifactPathType = "TYPE_UNSPECIFIED" + BundleDeployArtifactPathTypeWorkspace BundleDeployArtifactPathType = "WORKSPACE_FILE_SYSTEM" + BundleDeployArtifactPathTypeVolume BundleDeployArtifactPathType = "UC_VOLUME" +) diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go new file mode 100644 index 000000000..7e6ab1012 --- /dev/null +++ b/libs/telemetry/protos/frontend_log.go @@ -0,0 +1,22 @@ +package protos + +// This corresponds to the FrontendLog lumberjack proto in universe. +// FrontendLog is the top-level struct for any client-side logs at Databricks. +type FrontendLog struct { + // A UUID for the log event generated from the CLI. + FrontendLogEventID string `json:"frontend_log_event_id,omitempty"` + + Entry FrontendLogEntry `json:"entry,omitempty"` +} + +type FrontendLogEntry struct { + DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitempty"` +} + +type DatabricksCliLog struct { + ExecutionContext *ExecutionContext `json:"execution_context,omitempty"` + + CliTestEvent *CliTestEvent `json:"cli_test_event,omitempty"` + BundleInitEvent *BundleInitEvent `json:"bundle_init_event,omitempty"` + BundleDeployEvent *BundleDeployEvent `json:"bundle_deploy_event,omitempty"` +} From 708c4fbb7af82ff75065e95ca4b50e451c45a315 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 29 Jan 2025 13:14:21 +0100 Subject: [PATCH 025/105] Autogenerated documentation for bundle config (#2033) ## Changes Documentation autogeneration tool. This tool uses same annotations_*.yml files as in json-schema Result will go [there](https://docs.databricks.com/en/dev-tools/bundles/reference.html) and [there](https://docs.databricks.com/en/dev-tools/bundles/resources.html#cluster) ## Tests Manually --- Makefile | 5 +- bundle/docsgen/README.md | 79 + bundle/docsgen/main.go | 135 + bundle/docsgen/markdown.go | 99 + bundle/docsgen/nodes.go | 228 + bundle/docsgen/nodes_test.go | 120 + bundle/docsgen/output/reference.md | 1336 +++ bundle/docsgen/output/resources.md | 8189 +++++++++++++++++ bundle/docsgen/refs.go | 97 + bundle/docsgen/renderer.go | 51 + bundle/docsgen/templates/reference.md | 10 + bundle/docsgen/templates/resources.md | 70 + bundle/internal/annotation/descriptor.go | 12 + bundle/internal/annotation/file.go | 44 + bundle/internal/schema/annotations.go | 74 +- bundle/internal/schema/annotations.yml | 318 +- .../schema/annotations_openapi_overrides.yml | 355 + bundle/internal/schema/annotations_test.go | 4 + bundle/internal/schema/main_test.go | 5 +- bundle/internal/schema/parser.go | 23 +- bundle/schema/jsonschema.json | 170 +- libs/jsonschema/from_type.go | 4 + libs/jsonschema/schema.go | 4 + 23 files changed, 11065 insertions(+), 367 deletions(-) create mode 100644 bundle/docsgen/README.md create mode 100644 bundle/docsgen/main.go create mode 100644 bundle/docsgen/markdown.go create mode 100644 bundle/docsgen/nodes.go create mode 100644 bundle/docsgen/nodes_test.go create mode 100644 bundle/docsgen/output/reference.md create mode 100644 bundle/docsgen/output/resources.md create mode 100644 bundle/docsgen/refs.go create mode 100644 bundle/docsgen/renderer.go create mode 100644 bundle/docsgen/templates/reference.md create mode 100644 bundle/docsgen/templates/resources.md create mode 100644 bundle/internal/annotation/descriptor.go create mode 100644 bundle/internal/annotation/file.go diff --git a/Makefile b/Makefile index 00dadcb0c..d30ccef14 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ vendor: schema: go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json +docs: + go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen + INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h integration: @@ -56,4 +59,4 @@ integration: integration-short: $(INTEGRATION) -short -.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover +.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs diff --git a/bundle/docsgen/README.md b/bundle/docsgen/README.md new file mode 100644 index 000000000..c8484ab64 --- /dev/null +++ b/bundle/docsgen/README.md @@ -0,0 +1,79 @@ +## docs-autogen + +1. Install [Golang](https://go.dev/doc/install) +2. Run `make vendor docs` from the repo +3. See generated documents in `./bundle/docsgen/output` directory +4. To change descriptions update content in `./bundle/internal/schema/annotations.yml` or `./bundle/internal/schema/annotations_openapi_overrides.yml` and re-run `make docs` + +For simpler usage run it together with copy command to move resulting files to local `docs` repo. Note that it will overwrite any local changes in affected files. Example: + +``` +make docs && cp bundle/docgen/output/*.md ../docs/source/dev-tools/bundles +``` + +To change intro sections for files update them in `templates/` directory + +### Annotation file structure + +```yaml +"": + "": + description: Description of the property, only plain text is supported + markdown_description: Description with markdown support, if defined it will override the value in docs and in JSON-schema + markdown_examples: Custom block for any example, in free form, Markdown is supported + title: JSON-schema title, not used in docs + default: Default value of the property, not used in docs + enum: Possible values of enum-type, not used in docs +``` + +Descriptions with `PLACEHOLDER` value are not displayed in docs and JSON-schema + +All relative links like `[_](/dev-tools/bundles/settings.md#cluster_id)` are kept as is in docs but converted to absolute links in JSON schema + +To change description for type itself (not its fields) use `"_"`: + +```yaml +github.com/databricks/cli/bundle/config/resources.Cluster: + "_": + "markdown_description": |- + The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). +``` + +### Example annotation + +```yaml +github.com/databricks/cli/bundle/config.Bundle: + "cluster_id": + "description": |- + The ID of a cluster to use to run the bundle. + "markdown_description": |- + The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id). + "compute_id": + "description": |- + PLACEHOLDER + "databricks_cli_version": + "description": |- + The Databricks CLI version to use for the bundle. + "markdown_description": |- + The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version). + "deployment": + "description": |- + The definition of the bundle deployment + "markdown_description": |- + The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md). + "git": + "description": |- + The Git version control details that are associated with your bundle. + "markdown_description": |- + The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git). + "name": + "description": |- + The name of the bundle. + "uuid": + "description": |- + PLACEHOLDER +``` + +### TODO + +Add file watcher to track changes in the annotation files and re-run `make docs` script automtically diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go new file mode 100644 index 000000000..ad737feea --- /dev/null +++ b/bundle/docsgen/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "fmt" + "log" + "os" + "path" + "reflect" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/jsonschema" +) + +const ( + rootFileName = "reference.md" + resourcesFileName = "resources.md" +) + +func main() { + if len(os.Args) != 3 { + fmt.Println("Usage: go run main.go ") + os.Exit(1) + } + + annotationDir := os.Args[1] + docsDir := os.Args[2] + outputDir := path.Join(docsDir, "output") + templatesDir := path.Join(docsDir, "templates") + + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + log.Fatal(err) + } + } + + rootHeader, err := os.ReadFile(path.Join(templatesDir, rootFileName)) + if err != nil { + log.Fatal(err) + } + err = generateDocs( + []string{path.Join(annotationDir, "annotations.yml")}, + path.Join(outputDir, rootFileName), + reflect.TypeOf(config.Root{}), + string(rootHeader), + ) + if err != nil { + log.Fatal(err) + } + resourcesHeader, err := os.ReadFile(path.Join(templatesDir, resourcesFileName)) + if err != nil { + log.Fatal(err) + } + err = generateDocs( + []string{path.Join(annotationDir, "annotations_openapi.yml"), path.Join(annotationDir, "annotations_openapi_overrides.yml"), path.Join(annotationDir, "annotations.yml")}, + path.Join(outputDir, resourcesFileName), + reflect.TypeOf(config.Resources{}), + string(resourcesHeader), + ) + if err != nil { + log.Fatal(err) + } +} + +func generateDocs(inputPaths []string, outputPath string, rootType reflect.Type, header string) error { + annotations, err := annotation.LoadAndMerge(inputPaths) + if err != nil { + log.Fatal(err) + } + + // schemas is used to resolve references to schemas + schemas := map[string]*jsonschema.Schema{} + // ownFields is used to track fields that are defined in the annotation file and should be included in the docs page + ownFields := map[string]bool{} + + s, err := jsonschema.FromType(rootType, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ + func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + _, isOwnField := annotations[jsonschema.TypePath(typ)] + if isOwnField { + ownFields[jsonschema.TypePath(typ)] = true + } + + refPath := getPath(typ) + shouldHandle := strings.HasPrefix(refPath, "github.com") + if !shouldHandle { + schemas[jsonschema.TypePath(typ)] = &s + return s + } + + a := annotations[refPath] + if a == nil { + a = map[string]annotation.Descriptor{} + } + + rootTypeAnnotation, ok := a["_"] + if ok { + assignAnnotation(&s, rootTypeAnnotation) + } + + for k, v := range s.Properties { + assignAnnotation(v, a[k]) + } + + schemas[jsonschema.TypePath(typ)] = &s + return s + }, + }) + if err != nil { + log.Fatal(err) + } + + nodes := buildNodes(s, schemas, ownFields) + err = buildMarkdown(nodes, outputPath, header) + if err != nil { + log.Fatal(err) + } + return nil +} + +func getPath(typ reflect.Type) string { + return typ.PkgPath() + "." + typ.Name() +} + +func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { + if a.Description != "" && a.Description != annotation.Placeholder { + s.Description = a.Description + } + if a.MarkdownDescription != "" { + s.MarkdownDescription = a.MarkdownDescription + } + if a.MarkdownExamples != "" { + s.Examples = []any{a.MarkdownExamples} + } +} diff --git a/bundle/docsgen/markdown.go b/bundle/docsgen/markdown.go new file mode 100644 index 000000000..6e3b42b65 --- /dev/null +++ b/bundle/docsgen/markdown.go @@ -0,0 +1,99 @@ +package main + +import ( + "fmt" + "log" + "os" + "strings" +) + +func buildMarkdown(nodes []rootNode, outputFile, header string) error { + m := newMardownRenderer() + m = m.PlainText(header) + for _, node := range nodes { + m = m.LF() + if node.TopLevel { + m = m.H2(node.Title) + } else { + m = m.H3(node.Title) + } + m = m.LF() + + if node.Type != "" { + m = m.PlainText(fmt.Sprintf("**`Type: %s`**", node.Type)) + m = m.LF() + } + m = m.PlainText(node.Description) + m = m.LF() + + if len(node.ObjectKeyAttributes) > 0 { + n := pickLastWord(node.Title) + n = removePluralForm(n) + m = m.CodeBlocks("yaml", fmt.Sprintf("%ss:\n <%s-name>:\n <%s-field-name>: <%s-field-value>", n, n, n, n)) + m = m.LF() + m = buildAttributeTable(m, node.ObjectKeyAttributes) + } else if len(node.ArrayItemAttributes) > 0 { + m = m.LF() + m = buildAttributeTable(m, node.ArrayItemAttributes) + } else if len(node.Attributes) > 0 { + m = m.LF() + m = buildAttributeTable(m, node.Attributes) + } + + if node.Example != "" { + m = m.LF() + m = m.PlainText("**Example**") + m = m.LF() + m = m.PlainText(node.Example) + } + } + + f, err := os.Create(outputFile) + if err != nil { + log.Fatal(err) + } + _, err = f.WriteString(m.String()) + if err != nil { + log.Fatal(err) + } + return f.Close() +} + +func pickLastWord(s string) string { + words := strings.Split(s, ".") + return words[len(words)-1] +} + +// Build a custom table which we use in Databricks website +func buildAttributeTable(m *markdownRenderer, attributes []attributeNode) *markdownRenderer { + m = m.LF() + m = m.PlainText(".. list-table::") + m = m.PlainText(" :header-rows: 1") + m = m.LF() + + m = m.PlainText(" * - Key") + m = m.PlainText(" - Type") + m = m.PlainText(" - Description") + m = m.LF() + + for _, a := range attributes { + m = m.PlainText(" * - " + fmt.Sprintf("`%s`", a.Title)) + m = m.PlainText(" - " + a.Type) + m = m.PlainText(" - " + formatDescription(a)) + m = m.LF() + } + return m +} + +func formatDescription(a attributeNode) string { + s := strings.ReplaceAll(a.Description, "\n", " ") + if a.Link != "" { + if strings.HasSuffix(s, ".") { + s += " " + } else if s != "" { + s += ". " + } + s += fmt.Sprintf("See [_](#%s).", a.Link) + } + return s +} diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go new file mode 100644 index 000000000..68ed86450 --- /dev/null +++ b/bundle/docsgen/nodes.go @@ -0,0 +1,228 @@ +package main + +import ( + "sort" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +// rootNode is an intermediate representation of resolved JSON-schema item that is used to generate documentation +// Every schema node goes follows this conversion `JSON-schema -> rootNode -> markdown text` +type rootNode struct { + Title string + Description string + Attributes []attributeNode + Example string + ObjectKeyAttributes []attributeNode + ArrayItemAttributes []attributeNode + TopLevel bool + Type string +} + +type attributeNode struct { + Title string + Type string + Description string + Link string +} + +type rootProp struct { + // k is the name of the property + k string + // v is the corresponding json-schema node + v *jsonschema.Schema + // topLevel is true only for direct properties of the schema of root type (e.g. config.Root or config.Resources) + // Example: config.Root has . + topLevel bool + // circular indicates if property was added by recursive type, e.g. task.for_each_task.task.for_each_task + // These entries don't expand further and don't add any new nodes from their properties + circular bool +} + +const MapType = "Map" + +// buildNodes converts JSON-schema to a flat list of rootNode items that are then used to generate markdown documentation +// It recursively traverses the schema expanding the resulting list with new items for every properties of nodes `object` and `array` type +func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFields map[string]bool) []rootNode { + rootProps := []rootProp{} + for k, v := range s.Properties { + rootProps = append(rootProps, rootProp{k, v, true, false}) + } + nodes := make([]rootNode, 0, len(rootProps)) + visited := make(map[string]bool) + + for i := 0; i < len(rootProps); i++ { + item := rootProps[i] + k := item.k + v := item.v + + if visited[k] { + continue + } + visited[k] = true + + v = resolveRefs(v, refs) + node := rootNode{ + Title: k, + Description: getDescription(v, item.topLevel), + TopLevel: item.topLevel, + Example: getExample(v), + Type: getHumanReadableType(v.Type), + } + + hasProperties := len(v.Properties) > 0 + if hasProperties { + node.Attributes = getAttributes(v.Properties, refs, ownFields, k, item.circular) + } + + mapValueType := getMapValueType(v, refs) + if mapValueType != nil { + d := getDescription(mapValueType, true) + if d != "" { + node.Description = d + } + if node.Example == "" { + node.Example = getExample(mapValueType) + } + node.ObjectKeyAttributes = getAttributes(mapValueType.Properties, refs, ownFields, getMapKeyPrefix(k), item.circular) + } + + arrayItemType := resolveRefs(v.Items, refs) + if arrayItemType != nil { + node.ArrayItemAttributes = getAttributes(arrayItemType.Properties, refs, ownFields, k, item.circular) + } + + nodes = append(nodes, node) + + // Whether we should add new root props from the children of the current JSON-schema node to include their definitions to this document + shouldAddNewProps := !item.circular + if shouldAddNewProps { + newProps := []rootProp{} + // Adds node with definition for the properties. Example: + // bundle: + // prop-name: + if hasProperties { + newProps = append(newProps, extractNodes(k, v.Properties, refs, ownFields)...) + } + + // Adds node with definition for the type of array item. Example: + // permissions: + // - + if arrayItemType != nil { + newProps = append(newProps, extractNodes(k, arrayItemType.Properties, refs, ownFields)...) + } + // Adds node with definition for the type of the Map value. Example: + // targets: + // : + if mapValueType != nil { + newProps = append(newProps, extractNodes(getMapKeyPrefix(k), mapValueType.Properties, refs, ownFields)...) + } + + rootProps = append(rootProps, newProps...) + } + } + + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].Title < nodes[j].Title + }) + return nodes +} + +func getMapValueType(v *jsonschema.Schema, refs map[string]*jsonschema.Schema) *jsonschema.Schema { + additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema) + if ok { + return resolveRefs(additionalProps, refs) + } + return nil +} + +func getMapKeyPrefix(s string) string { + return s + "." +} + +func removePluralForm(s string) string { + if strings.HasSuffix(s, "s") { + return strings.TrimSuffix(s, "s") + } + return s +} + +func getHumanReadableType(t jsonschema.Type) string { + typesMapping := map[string]string{ + "string": "String", + "integer": "Integer", + "boolean": "Boolean", + "array": "Sequence", + "object": "Map", + } + return typesMapping[string(t)] +} + +func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[string]bool, prefix string, circular bool) []attributeNode { + attributes := []attributeNode{} + for k, v := range props { + v = resolveRefs(v, refs) + typeString := getHumanReadableType(v.Type) + if typeString == "" { + typeString = "Any" + } + var reference string + if isReferenceType(v, refs, ownFields) && !circular { + reference = prefix + "." + k + } + attributes = append(attributes, attributeNode{ + Title: k, + Type: typeString, + Description: getDescription(v, true), + Link: reference, + }) + } + sort.Slice(attributes, func(i, j int) bool { + return attributes[i].Title < attributes[j].Title + }) + return attributes +} + +func getDescription(s *jsonschema.Schema, allowMarkdown bool) string { + if allowMarkdown && s.MarkdownDescription != "" { + return s.MarkdownDescription + } + return s.Description +} + +func shouldExtract(ref string, ownFields map[string]bool) bool { + if i := strings.Index(ref, "github.com"); i >= 0 { + ref = ref[i:] + } + _, isCustomField := ownFields[ref] + return isCustomField +} + +// extractNodes returns a list of rootProp items for all properties of the json-schema node that should be extracted based on context +// E.g. we extract all propert +func extractNodes(prefix string, props, refs map[string]*jsonschema.Schema, ownFields map[string]bool) []rootProp { + nodes := []rootProp{} + for k, v := range props { + if v.Reference != nil && !shouldExtract(*v.Reference, ownFields) { + continue + } + v = resolveRefs(v, refs) + if v.Type == "object" || v.Type == "array" { + nodes = append(nodes, rootProp{prefix + "." + k, v, false, isCycleField(k)}) + } + } + return nodes +} + +func isCycleField(field string) bool { + return field == "for_each_task" +} + +func getExample(v *jsonschema.Schema) string { + examples := v.Examples + if len(examples) == 0 { + return "" + } + return examples[0].(string) +} diff --git a/bundle/docsgen/nodes_test.go b/bundle/docsgen/nodes_test.go new file mode 100644 index 000000000..bdb2ce9db --- /dev/null +++ b/bundle/docsgen/nodes_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "testing" + + "github.com/databricks/cli/libs/jsonschema" + "github.com/stretchr/testify/assert" +) + +func TestBuildNodes_ChildExpansion(t *testing.T) { + tests := []struct { + name string + schema jsonschema.Schema + refs map[string]*jsonschema.Schema + ownFields map[string]bool + wantNodes []rootNode + }{ + { + name: "array expansion", + schema: jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "list": { + Type: "array", + Items: &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "listSub": {Reference: strPtr("#/$defs/github.com/listSub")}, + }, + }, + }, + }, + }, + refs: map[string]*jsonschema.Schema{ + "github.com/listSub": {Type: "array", Items: &jsonschema.Schema{Type: "object", Properties: map[string]*jsonschema.Schema{"subField": {Type: "string"}}}}, + }, + ownFields: map[string]bool{"github.com/listSub": true}, + wantNodes: []rootNode{ + { + Title: "list", + TopLevel: true, + Type: "Sequence", + ArrayItemAttributes: []attributeNode{ + {Title: "listSub", Type: "Sequence", Link: "list.listSub"}, + }, + }, + { + Title: "list.listSub", + Type: "Sequence", + ArrayItemAttributes: []attributeNode{ + {Title: "subField", Type: "String"}, + }, + }, + }, + }, + { + name: "map expansion", + schema: jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "myMap": { + Type: "object", + AdditionalProperties: &jsonschema.Schema{ + Reference: strPtr("#/$defs/github.com/myMap"), + Properties: map[string]*jsonschema.Schema{ + "mapSub": {Type: "object", Reference: strPtr("#/$defs/github.com/mapSub")}, + }, + }, + }, + }, + }, + refs: map[string]*jsonschema.Schema{ + "github.com/myMap": { + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "mapSub": {Type: "boolean", Reference: strPtr("#/$defs/github.com/mapSub")}, + }, + }, + "github.com/mapSub": { + Type: "object", + Properties: map[string]*jsonschema.Schema{ + "deepSub": {Type: "boolean"}, + }, + }, + }, + ownFields: map[string]bool{ + "github.com/myMap": true, + "github.com/mapSub": true, + }, + wantNodes: []rootNode{ + { + Title: "myMap", + TopLevel: true, + Type: "Map", + ObjectKeyAttributes: []attributeNode{ + {Title: "mapSub", Type: "Map", Link: "myMap..mapSub"}, + }, + }, + { + Title: "myMap..mapSub", + Type: "Map", + Attributes: []attributeNode{ + {Title: "deepSub", Type: "Boolean"}, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildNodes(tt.schema, tt.refs, tt.ownFields) + assert.Equal(t, tt.wantNodes, got) + }) + } +} + +func strPtr(s string) *string { + return &s +} diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md new file mode 100644 index 000000000..a2241d017 --- /dev/null +++ b/bundle/docsgen/output/reference.md @@ -0,0 +1,1336 @@ + +--- +description: Configuration reference for databricks.yml +--- + +# Configuration reference + +This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). + +For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples). + + +## artifacts + +**`Type: Map`** + +Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [_](/dev-tools/bundles/settings.md#artifacts). + +Artifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [_](/dev-tools/bundles/artifact-overrides.md). + +```yaml +artifacts: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `build` + - String + - An optional set of non-default build commands to run locally before deployment. + + * - `executable` + - String + - The executable type. Valid values are `bash`, `sh`, and `cmd`. + + * - `files` + - Sequence + - The source files for the artifact. See [_](#artifacts..files). + + * - `path` + - String + - The location where the built artifact will be saved. + + * - `type` + - String + - Required. The type of the artifact. Valid values are `whl`. + + +**Example** + +```yaml +artifacts: + default: + type: whl + build: poetry build + path: . +``` + +### artifacts..files + +**`Type: Sequence`** + +The source files for the artifact. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `source` + - String + - Required. The path of the files used to build the artifact. + + +## bundle + +**`Type: Map`** + +The bundle attributes when deploying to this target, + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `cluster_id` + - String + - The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id). + + * - `compute_id` + - String + - + + * - `databricks_cli_version` + - String + - The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version). + + * - `deployment` + - Map + - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundle.deployment). + + * - `git` + - Map + - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundle.git). + + * - `name` + - String + - The name of the bundle. + + * - `uuid` + - String + - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command). + + +### bundle.deployment + +**`Type: Map`** + +The definition of the bundle deployment + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `fail_on_active_runs` + - Boolean + - Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted. + + * - `lock` + - Map + - The deployment lock attributes. See [_](#bundle.deployment.lock). + + +### bundle.deployment.lock + +**`Type: Map`** + +The deployment lock attributes. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - Whether this lock is enabled. + + * - `force` + - Boolean + - Whether to force this lock if it is enabled. + + +### bundle.git + +**`Type: Map`** + +The Git version control details that are associated with your bundle. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `branch` + - String + - The Git branch name. See [_](/dev-tools/bundles/settings.md#git). + + * - `origin_url` + - String + - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git). + + +## experimental + +**`Type: Map`** + +Defines attributes for experimental features. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `pydabs` + - Map + - The PyDABs configuration. See [_](#experimental.pydabs). + + * - `python` + - Map + - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimental.python). + + * - `python_wheel_wrapper` + - Boolean + - Whether to use a Python wheel wrapper. + + * - `scripts` + - Map + - The commands to run. + + * - `use_legacy_run_as` + - Boolean + - Whether to use the legacy run_as behavior. + + +### experimental.pydabs + +**`Type: Map`** + +The PyDABs configuration. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - Whether or not PyDABs (Private Preview) is enabled + + * - `import` + - Sequence + - The PyDABs project to import to discover resources, resource generator and mutators + + * - `venv_path` + - String + - The Python virtual environment path + + +### experimental.python + +**`Type: Map`** + +Configures loading of Python code defined with 'databricks-bundles' package. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `mutators` + - Sequence + - Mutators contains a list of fully qualified function paths to mutator functions. Example: ["my_project.mutators:add_default_cluster"] + + * - `resources` + - Sequence + - Resources contains a list of fully qualified function paths to load resources defined in Python code. Example: ["my_project.resources:load_resources"] + + * - `venv_path` + - String + - VEnvPath is path to the virtual environment. If enabled, Python code will execute within this environment. If disabled, it defaults to using the Python interpreter available in the current shell. + + +## include + +**`Type: Sequence`** + +Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include) + + +## permissions + +**`Type: Sequence`** + +A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity. + +See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +**Example** + +```yaml +permissions: + - level: CAN_VIEW + group_name: test-group + - level: CAN_MANAGE + user_name: someone@example.com + - level: CAN_RUN + service_principal_name: 123456-abcdef +``` + +## presets + +**`Type: Map`** + +Defines bundle deployment presets. See [_](/dev-tools/bundles/deployment-modes.md#presets). + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jobs_max_concurrent_runs` + - Integer + - The maximum concurrent runs for a job. + + * - `name_prefix` + - String + - The prefix for job runs of the bundle. + + * - `pipelines_development` + - Boolean + - Whether pipeline deployments should be locked in development mode. + + * - `source_linked_deployment` + - Boolean + - Whether to link the deployment to the bundle source. + + * - `tags` + - Map + - The tags for the bundle deployment. + + * - `trigger_pause_status` + - String + - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED. + + +## resources + +**`Type: Map`** + +A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about supported resources, and resource definition reference, see [_](/dev-tools/bundles/resources.md). + +```yaml +resources: + : + : + : +``` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apps` + - Map + - + + * - `clusters` + - Map + - The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters) + + * - `dashboards` + - Map + - The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards) + + * - `experiments` + - Map + - The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments) + + * - `jobs` + - Map + - The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs) + + * - `model_serving_endpoints` + - Map + - The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints) + + * - `models` + - Map + - The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models) + + * - `pipelines` + - Map + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines) + + * - `quality_monitors` + - Map + - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors) + + * - `registered_models` + - Map + - The registered model definitions for the bundle, where each key is the name of the registered model. See [_](/dev-tools/bundles/resources.md#registered_models) + + * - `schemas` + - Map + - The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas) + + * - `volumes` + - Map + - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes) + + +## run_as + +**`Type: Map`** + +The identity to use when running workflows. See [_](/dev-tools/bundles/run-as.md). + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `service_principal_name` + - String + - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + + * - `user_name` + - String + - The email of an active workspace user. Non-admin users can only set this field to their own email. + + +## sync + +**`Type: Map`** + +The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/settings.md#sync). + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `exclude` + - Sequence + - A list of files or folders to exclude from the bundle. + + * - `include` + - Sequence + - A list of files or folders to include in the bundle. + + * - `paths` + - Sequence + - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed. + + +## targets + +**`Type: Map`** + +Defines deployment targets for the bundle. See [_](/dev-tools/bundles/settings.md#targets) + +```yaml +targets: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `artifacts` + - Map + - The artifacts to include in the target deployment. See [_](#targets..artifacts). + + * - `bundle` + - Map + - The bundle attributes when deploying to this target. See [_](#targets..bundle). + + * - `cluster_id` + - String + - The ID of the cluster to use for this target. + + * - `compute_id` + - String + - Deprecated. The ID of the compute to use for this target. + + * - `default` + - Boolean + - Whether this target is the default target. + + * - `git` + - Map + - The Git version control settings for the target. See [_](#targets..git). + + * - `mode` + - String + - The deployment mode for the target. Valid values are `development` or `production`. See [_](/dev-tools/bundles/deployment-modes.md). + + * - `permissions` + - Sequence + - The permissions for deploying and running the bundle in the target. See [_](#targets..permissions). + + * - `presets` + - Map + - The deployment presets for the target. See [_](#targets..presets). + + * - `resources` + - Map + - The resource definitions for the target. See [_](#targets..resources). + + * - `run_as` + - Map + - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targets..run_as). + + * - `sync` + - Map + - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targets..sync). + + * - `variables` + - Map + - The custom variable definitions for the target. See [_](#targets..variables). + + * - `workspace` + - Map + - The Databricks workspace for the target. See [_](#targets..workspace). + + +### targets..artifacts + +**`Type: Map`** + +The artifacts to include in the target deployment. + +```yaml +artifacts: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `build` + - String + - An optional set of non-default build commands to run locally before deployment. + + * - `executable` + - String + - The executable type. Valid values are `bash`, `sh`, and `cmd`. + + * - `files` + - Sequence + - The source files for the artifact. See [_](#targets..artifacts..files). + + * - `path` + - String + - The location where the built artifact will be saved. + + * - `type` + - String + - Required. The type of the artifact. Valid values are `whl`. + + +### targets..artifacts..files + +**`Type: Sequence`** + +The source files for the artifact. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `source` + - String + - Required. The path of the files used to build the artifact. + + +### targets..bundle + +**`Type: Map`** + +The bundle attributes when deploying to this target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `cluster_id` + - String + - The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id). + + * - `compute_id` + - String + - + + * - `databricks_cli_version` + - String + - The Databricks CLI version to use for the bundle. See [_](/dev-tools/bundles/settings.md#databricks_cli_version). + + * - `deployment` + - Map + - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targets..bundle.deployment). + + * - `git` + - Map + - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targets..bundle.git). + + * - `name` + - String + - The name of the bundle. + + * - `uuid` + - String + - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command). + + +### targets..bundle.deployment + +**`Type: Map`** + +The definition of the bundle deployment + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `fail_on_active_runs` + - Boolean + - Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted. + + * - `lock` + - Map + - The deployment lock attributes. See [_](#targets..bundle.deployment.lock). + + +### targets..bundle.deployment.lock + +**`Type: Map`** + +The deployment lock attributes. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - Whether this lock is enabled. + + * - `force` + - Boolean + - Whether to force this lock if it is enabled. + + +### targets..bundle.git + +**`Type: Map`** + +The Git version control details that are associated with your bundle. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `branch` + - String + - The Git branch name. See [_](/dev-tools/bundles/settings.md#git). + + * - `origin_url` + - String + - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git). + + +### targets..git + +**`Type: Map`** + +The Git version control settings for the target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `branch` + - String + - The Git branch name. See [_](/dev-tools/bundles/settings.md#git). + + * - `origin_url` + - String + - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git). + + +### targets..permissions + +**`Type: Sequence`** + +The permissions for deploying and running the bundle in the target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### targets..presets + +**`Type: Map`** + +The deployment presets for the target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jobs_max_concurrent_runs` + - Integer + - The maximum concurrent runs for a job. + + * - `name_prefix` + - String + - The prefix for job runs of the bundle. + + * - `pipelines_development` + - Boolean + - Whether pipeline deployments should be locked in development mode. + + * - `source_linked_deployment` + - Boolean + - Whether to link the deployment to the bundle source. + + * - `tags` + - Map + - The tags for the bundle deployment. + + * - `trigger_pause_status` + - String + - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED. + + +### targets..resources + +**`Type: Map`** + +The resource definitions for the target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apps` + - Map + - + + * - `clusters` + - Map + - The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters) + + * - `dashboards` + - Map + - The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards) + + * - `experiments` + - Map + - The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments) + + * - `jobs` + - Map + - The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs) + + * - `model_serving_endpoints` + - Map + - The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints) + + * - `models` + - Map + - The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models) + + * - `pipelines` + - Map + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines) + + * - `quality_monitors` + - Map + - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors) + + * - `registered_models` + - Map + - The registered model definitions for the bundle, where each key is the name of the registered model. See [_](/dev-tools/bundles/resources.md#registered_models) + + * - `schemas` + - Map + - The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas) + + * - `volumes` + - Map + - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes) + + +### targets..run_as + +**`Type: Map`** + +The identity to use to run the bundle. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `service_principal_name` + - String + - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + + * - `user_name` + - String + - The email of an active workspace user. Non-admin users can only set this field to their own email. + + +### targets..sync + +**`Type: Map`** + +The local paths to sync to the target workspace when a bundle is run or deployed. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `exclude` + - Sequence + - A list of files or folders to exclude from the bundle. + + * - `include` + - Sequence + - A list of files or folders to include in the bundle. + + * - `paths` + - Sequence + - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed. + + +### targets..variables + +**`Type: Map`** + +The custom variable definitions for the target. + +```yaml +variables: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `default` + - Any + - + + * - `description` + - String + - The description of the variable. + + * - `lookup` + - Map + - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targets..variables..lookup). + + * - `type` + - String + - The type of the variable. + + +### targets..variables..lookup + +**`Type: Map`** + +The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alert` + - String + - + + * - `cluster` + - String + - + + * - `cluster_policy` + - String + - + + * - `dashboard` + - String + - + + * - `instance_pool` + - String + - + + * - `job` + - String + - + + * - `metastore` + - String + - + + * - `notification_destination` + - String + - + + * - `pipeline` + - String + - + + * - `query` + - String + - + + * - `service_principal` + - String + - + + * - `warehouse` + - String + - + + +### targets..workspace + +**`Type: Map`** + +The Databricks workspace for the target. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `artifact_path` + - String + - The artifact path to use within the workspace for both deployments and workflow runs + + * - `auth_type` + - String + - The authentication type. + + * - `azure_client_id` + - String + - The Azure client ID + + * - `azure_environment` + - String + - The Azure environment + + * - `azure_login_app_id` + - String + - The Azure login app ID + + * - `azure_tenant_id` + - String + - The Azure tenant ID + + * - `azure_use_msi` + - Boolean + - Whether to use MSI for Azure + + * - `azure_workspace_resource_id` + - String + - The Azure workspace resource ID + + * - `client_id` + - String + - The client ID for the workspace + + * - `file_path` + - String + - The file path to use within the workspace for both deployments and workflow runs + + * - `google_service_account` + - String + - The Google service account name + + * - `host` + - String + - The Databricks workspace host URL + + * - `profile` + - String + - The Databricks workspace profile name + + * - `resource_path` + - String + - The workspace resource path + + * - `root_path` + - String + - The Databricks workspace root path + + * - `state_path` + - String + - The workspace state path + + +## variables + +**`Type: Map`** + +Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables). + +```yaml +variables: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `default` + - Any + - + + * - `description` + - String + - The description of the variable + + * - `lookup` + - Map + - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variables..lookup). + + * - `type` + - String + - The type of the variable. + + +### variables..lookup + +**`Type: Map`** + +The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alert` + - String + - + + * - `cluster` + - String + - + + * - `cluster_policy` + - String + - + + * - `dashboard` + - String + - + + * - `instance_pool` + - String + - + + * - `job` + - String + - + + * - `metastore` + - String + - + + * - `notification_destination` + - String + - + + * - `pipeline` + - String + - + + * - `query` + - String + - + + * - `service_principal` + - String + - + + * - `warehouse` + - String + - + + +## workspace + +**`Type: Map`** + +Defines the Databricks workspace for the bundle. See [_](/dev-tools/bundles/settings.md#workspace). + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `artifact_path` + - String + - The artifact path to use within the workspace for both deployments and workflow runs + + * - `auth_type` + - String + - The authentication type. + + * - `azure_client_id` + - String + - The Azure client ID + + * - `azure_environment` + - String + - The Azure environment + + * - `azure_login_app_id` + - String + - The Azure login app ID + + * - `azure_tenant_id` + - String + - The Azure tenant ID + + * - `azure_use_msi` + - Boolean + - Whether to use MSI for Azure + + * - `azure_workspace_resource_id` + - String + - The Azure workspace resource ID + + * - `client_id` + - String + - The client ID for the workspace + + * - `file_path` + - String + - The file path to use within the workspace for both deployments and workflow runs + + * - `google_service_account` + - String + - The Google service account name + + * - `host` + - String + - The Databricks workspace host URL + + * - `profile` + - String + - The Databricks workspace profile name + + * - `resource_path` + - String + - The workspace resource path + + * - `root_path` + - String + - The Databricks workspace root path + + * - `state_path` + - String + - The workspace state path + \ No newline at end of file diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md new file mode 100644 index 000000000..ff80ee635 --- /dev/null +++ b/bundle/docsgen/output/resources.md @@ -0,0 +1,8189 @@ + +--- +description: Learn about resources supported by Databricks Asset Bundles and how to configure them. +--- + +# resources + + allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). + +This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md). + +## Supported resources + +The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle. + +Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table. + +.. tip:: The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files. + + +.. list-table:: + :header-rows: 1 + + * - Resource + - Create support + - Corresponding REST API object + + * - [cluster](#cluster) + - ✓ + - [Cluster object](/api/workspace/clusters/create) + + * - [dashboard](#dashboard) + - + - [Dashboard object](/api/workspace/lakeview/create) + + * - [experiment](#experiment) + - ✓ + - [Experiment object](/api/workspace/experiments/createexperiment) + + * - [job](#job) + - ✓ + - [Job object](/api/workspace/jobs/create) + + * - [model (legacy)](#model-legacy) + - ✓ + - [Model (legacy) object](/api/workspace/modelregistry/createmodel) + + * - [model_serving_endpoint](#model-serving-endpoint) + - ✓ + - [Model serving endpoint object](/api/workspace/servingendpoints/create) + + * - [pipeline](#pipeline) + - ✓ + - [Pipeline object](/api/workspace/pipelines/create) + + * - [quality_monitor](#quality-monitor) + - ✓ + - [Quality monitor object](/api/workspace/qualitymonitors/create) + + * - [registered_model](#registered-model) () + - ✓ + - [Registered model object](/api/workspace/registeredmodels/create) + + * - [schema](#schema) () + - ✓ + - [Schema object](/api/workspace/schemas/create) + + * - [volume](#volume) () + - ✓ + - [Volume object](/api/workspace/volumes/create) + + +## apps + +**`Type: Map`** + + + +```yaml +apps: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `active_deployment` + - Map + - See [_](#apps..active_deployment). + + * - `app_status` + - Map + - See [_](#apps..app_status). + + * - `compute_status` + - Map + - See [_](#apps..compute_status). + + * - `config` + - Map + - + + * - `create_time` + - String + - + + * - `creator` + - String + - + + * - `default_source_code_path` + - String + - + + * - `description` + - String + - + + * - `name` + - String + - + + * - `pending_deployment` + - Map + - See [_](#apps..pending_deployment). + + * - `permissions` + - Sequence + - See [_](#apps..permissions). + + * - `resources` + - Sequence + - See [_](#apps..resources). + + * - `service_principal_client_id` + - String + - + + * - `service_principal_id` + - Integer + - + + * - `service_principal_name` + - String + - + + * - `source_code_path` + - String + - + + * - `update_time` + - String + - + + * - `updater` + - String + - + + * - `url` + - String + - + + +### apps..active_deployment + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `create_time` + - String + - + + * - `creator` + - String + - + + * - `deployment_artifacts` + - Map + - See [_](#apps..active_deployment.deployment_artifacts). + + * - `deployment_id` + - String + - + + * - `mode` + - String + - + + * - `source_code_path` + - String + - + + * - `status` + - Map + - See [_](#apps..active_deployment.status). + + * - `update_time` + - String + - + + +### apps..active_deployment.deployment_artifacts + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `source_code_path` + - String + - + + +### apps..active_deployment.status + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `message` + - String + - + + * - `state` + - String + - + + +### apps..app_status + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `message` + - String + - + + * - `state` + - String + - + + +### apps..compute_status + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `message` + - String + - + + * - `state` + - String + - State of the app compute. + + +### apps..pending_deployment + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `create_time` + - String + - + + * - `creator` + - String + - + + * - `deployment_artifacts` + - Map + - See [_](#apps..pending_deployment.deployment_artifacts). + + * - `deployment_id` + - String + - + + * - `mode` + - String + - + + * - `source_code_path` + - String + - + + * - `status` + - Map + - See [_](#apps..pending_deployment.status). + + * - `update_time` + - String + - + + +### apps..pending_deployment.deployment_artifacts + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `source_code_path` + - String + - + + +### apps..pending_deployment.status + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `message` + - String + - + + * - `state` + - String + - + + +### apps..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### apps..resources + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `description` + - String + - Description of the App Resource. + + * - `job` + - Map + - See [_](#apps..resources.job). + + * - `name` + - String + - Name of the App Resource. + + * - `secret` + - Map + - See [_](#apps..resources.secret). + + * - `serving_endpoint` + - Map + - See [_](#apps..resources.serving_endpoint). + + * - `sql_warehouse` + - Map + - See [_](#apps..resources.sql_warehouse). + + +### apps..resources.job + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + * - `permission` + - String + - + + +### apps..resources.secret + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `key` + - String + - + + * - `permission` + - String + - Permission to grant on the secret scope. Supported permissions are: "READ", "WRITE", "MANAGE". + + * - `scope` + - String + - + + +### apps..resources.serving_endpoint + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `name` + - String + - + + * - `permission` + - String + - + + +### apps..resources.sql_warehouse + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + * - `permission` + - String + - + + +## clusters + +**`Type: Map`** + +The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). + +```yaml +clusters: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apply_policy_default_values` + - Boolean + - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied. + + * - `autoscale` + - Map + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clusters..autoscale). + + * - `autotermination_minutes` + - Integer + - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination. + + * - `aws_attributes` + - Map + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..aws_attributes). + + * - `azure_attributes` + - Map + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..azure_attributes). + + * - `cluster_log_conf` + - Map + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clusters..cluster_log_conf). + + * - `cluster_name` + - String + - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. + + * - `custom_tags` + - Map + - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags + + * - `data_security_mode` + - String + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + + * - `docker_image` + - Map + - See [_](#clusters..docker_image). + + * - `driver_instance_pool_id` + - String + - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. + + * - `driver_node_type_id` + - String + - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. + + * - `enable_elastic_disk` + - Boolean + - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details. + + * - `enable_local_disk_encryption` + - Boolean + - Whether to enable LUKS on cluster VMs' local disks + + * - `gcp_attributes` + - Map + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..gcp_attributes). + + * - `init_scripts` + - Sequence + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#clusters..init_scripts). + + * - `instance_pool_id` + - String + - The optional ID of the instance pool to which the cluster belongs. + + * - `is_single_node` + - Boolean + - This field can only be used with `kind`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` + + * - `kind` + - String + - + + * - `node_type_id` + - String + - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. + + * - `num_workers` + - Integer + - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. + + * - `permissions` + - Sequence + - See [_](#clusters..permissions). + + * - `policy_id` + - String + - The ID of the cluster policy used to create the cluster if applicable. + + * - `runtime_engine` + - String + - Determines the cluster's runtime engine, either standard or Photon. This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`. If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. + + * - `single_user_name` + - String + - Single user name if data_security_mode is `SINGLE_USER` + + * - `spark_conf` + - Map + - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. + + * - `spark_env_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well. Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` + + * - `spark_version` + - String + - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. + + * - `ssh_public_keys` + - Sequence + - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + + * - `use_ml_runtime` + - Boolean + - This field can only be used with `kind`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. + + * - `workload_type` + - Map + - See [_](#clusters..workload_type). + + +**Example** + +The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: + +```yaml +bundle: + name: clusters + +resources: + clusters: + my_cluster: + num_workers: 2 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 2 + max_workers: 7 + spark_version: "13.3.x-scala2.12" + spark_conf: + "spark.executor.memory": "2g" + + jobs: + my_job: + tasks: + - task_key: test_task + notebook_task: + notebook_path: "./src/my_notebook.py" +``` + +### clusters..autoscale + +**`Type: Map`** + +Parameters needed in order to automatically scale clusters up and down based on load. +Note: autoscaling works best with DB runtime versions 3.0 or later. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `max_workers` + - Integer + - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`. + + * - `min_workers` + - Integer + - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. + + +### clusters..aws_attributes + +**`Type: Map`** + +Attributes related to clusters running on Amazon Web Services. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. + + * - `ebs_volume_count` + - Integer + - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail. These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc. If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes. Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden. + + * - `ebs_volume_iops` + - Integer + - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_size` + - Integer + - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096. + + * - `ebs_volume_throughput` + - Integer + - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_type` + - String + - The type of EBS volumes that will be launched with this cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `instance_profile_arn` + - String + - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator. This feature may only be available to certain customer plans. If this field is ommitted, we will pull in the default from the conf if it exists. + + * - `spot_bid_price_percent` + - Integer + - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000. The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent. + + * - `zone_id` + - String + - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. + + +### clusters..azure_attributes + +**`Type: Map`** + +Attributes related to clusters running on Microsoft Azure. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `log_analytics_info` + - Map + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clusters..azure_attributes.log_analytics_info). + + * - `spot_bid_max_price` + - Any + - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. + + +### clusters..azure_attributes.log_analytics_info + +**`Type: Map`** + +Defines values necessary to configure and run Azure Log Analytics agent + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `log_analytics_primary_key` + - String + - + + * - `log_analytics_workspace_id` + - String + - + + +### clusters..cluster_log_conf + +**`Type: Map`** + +The configuration for delivering spark logs to a long-term storage destination. +Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified +for one cluster. If the conf is given, the logs will be delivered to the destination every +`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while +the destination of executor logs is `$destination/$clusterId/executor`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters..cluster_log_conf.dbfs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters..cluster_log_conf.s3). + + +### clusters..cluster_log_conf.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### clusters..cluster_log_conf.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### clusters..docker_image + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `basic_auth` + - Map + - See [_](#clusters..docker_image.basic_auth). + + * - `url` + - String + - URL of the docker image. + + +### clusters..docker_image.basic_auth + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `password` + - String + - Password of the user + + * - `username` + - String + - Name of the user + + +### clusters..gcp_attributes + +**`Type: Map`** + +Attributes related to clusters running on Google Cloud Platform. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable. + + * - `boot_disk_size` + - Integer + - boot disk size in GB + + * - `google_service_account` + - String + - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator. + + * - `local_ssd_count` + - Integer + - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type. + + * - `use_preemptible_executors` + - Boolean + - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead. + + * - `zone_id` + - String + - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. + + +### clusters..init_scripts + +**`Type: Sequence`** + +The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `abfss` + - Map + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#clusters..init_scripts.abfss). + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters..init_scripts.dbfs). + + * - `file` + - Map + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clusters..init_scripts.file). + + * - `gcs` + - Map + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clusters..init_scripts.gcs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters..init_scripts.s3). + + * - `volumes` + - Map + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clusters..init_scripts.volumes). + + * - `workspace` + - Map + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clusters..init_scripts.workspace). + + +### clusters..init_scripts.abfss + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } } + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. + + +### clusters..init_scripts.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### clusters..init_scripts.file + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "file" : { "destination" : "file:/my/local/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - local file destination, e.g. `file:/my/local/file.sh` + + +### clusters..init_scripts.gcs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` + + +### clusters..init_scripts.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### clusters..init_scripts.volumes + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` + + +### clusters..init_scripts.workspace + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` + + +### clusters..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### clusters..workload_type + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `clients` + - Map + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clusters..workload_type.clients). + + +### clusters..workload_type.clients + +**`Type: Map`** + + defined what type of clients can use the cluster. E.g. Notebooks, Jobs + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jobs` + - Boolean + - With jobs set, the cluster can be used for jobs + + * - `notebooks` + - Boolean + - With notebooks set, this cluster can be used for notebooks + + +## dashboards + +**`Type: Map`** + +The dashboard resource allows you to manage [AI/BI dashboards](/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [_](/dashboards/index.md). + +```yaml +dashboards: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `create_time` + - String + - The timestamp of when the dashboard was created. + + * - `dashboard_id` + - String + - UUID identifying the dashboard. + + * - `display_name` + - String + - The display name of the dashboard. + + * - `embed_credentials` + - Boolean + - + + * - `etag` + - String + - The etag for the dashboard. Can be optionally provided on updates to ensure that the dashboard has not been modified since the last read. This field is excluded in List Dashboards responses. + + * - `file_path` + - String + - + + * - `lifecycle_state` + - String + - The state of the dashboard resource. Used for tracking trashed status. + + * - `parent_path` + - String + - The workspace path of the folder containing the dashboard. Includes leading slash and no trailing slash. This field is excluded in List Dashboards responses. + + * - `path` + - String + - The workspace path of the dashboard asset, including the file name. Exported dashboards always have the file extension `.lvdash.json`. This field is excluded in List Dashboards responses. + + * - `permissions` + - Sequence + - See [_](#dashboards..permissions). + + * - `serialized_dashboard` + - Any + - The contents of the dashboard in serialized string form. This field is excluded in List Dashboards responses. Use the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get) to retrieve an example response, which includes the `serialized_dashboard` field. This field provides the structure of the JSON string that represents the dashboard's layout and components. + + * - `update_time` + - String + - The timestamp of when the dashboard was last updated by the user. This field is excluded in List Dashboards responses. + + * - `warehouse_id` + - String + - The warehouse ID used to run the dashboard. + + +**Example** + +The following example includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to the Databricks workspace. + +``` yaml +resources: + dashboards: + nyc_taxi_trip_analysis: + display_name: "NYC Taxi Trip Analysis" + file_path: ../src/nyc_taxi_trip_analysis.lvdash.json + warehouse_id: ${var.warehouse_id} +``` +If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). + +In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). + +### dashboards..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +## experiments + +**`Type: Map`** + +The experiment resource allows you to define [MLflow experiments](/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [_](/mlflow/experiments.md). + +```yaml +experiments: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `artifact_location` + - String + - Location where artifacts for the experiment are stored. + + * - `creation_time` + - Integer + - Creation time + + * - `experiment_id` + - String + - Unique identifier for the experiment. + + * - `last_update_time` + - Integer + - Last update time + + * - `lifecycle_stage` + - String + - Current life cycle stage of the experiment: "active" or "deleted". Deleted experiments are not returned by APIs. + + * - `name` + - String + - Human readable name that identifies the experiment. + + * - `permissions` + - Sequence + - See [_](#experiments..permissions). + + * - `tags` + - Sequence + - Tags: Additional metadata key-value pairs. See [_](#experiments..tags). + + +**Example** + +The following example defines an experiment that all users can view: + +```yaml +resources: + experiments: + experiment: + name: my_ml_experiment + permissions: + - level: CAN_READ + group_name: users + description: MLflow experiment used to track runs +``` + +### experiments..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### experiments..tags + +**`Type: Sequence`** + +Tags: Additional metadata key-value pairs. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `key` + - String + - The tag key. + + * - `value` + - String + - The tag value. + + +## jobs + +**`Type: Map`** + +The job resource allows you to define [jobs and their corresponding tasks](/api/workspace/jobs/create) in your bundle. For information about jobs, see [_](/jobs/index.md). For a tutorial that uses a template to create a job, see [_](/dev-tools/bundles/jobs-tutorial.md). + +```yaml +jobs: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `budget_policy_id` + - String + - The id of the user specified budget policy to use for this job. If not specified, a default budget policy may be applied when creating or modifying the job. See `effective_budget_policy_id` for the budget policy used by this workload. + + * - `continuous` + - Map + - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobs..continuous). + + * - `deployment` + - Map + - Deployment information for jobs managed by external sources. See [_](#jobs..deployment). + + * - `description` + - String + - An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding. + + * - `edit_mode` + - String + - Edit mode of the job. * `UI_LOCKED`: The job is in a locked UI state and cannot be modified. * `EDITABLE`: The job is in an editable state and can be modified. + + * - `email_notifications` + - Map + - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobs..email_notifications). + + * - `environments` + - Sequence + - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobs..environments). + + * - `format` + - String + - Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `"MULTI_TASK"`. + + * - `git_source` + - Map + - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobs..git_source). + + * - `health` + - Map + - An optional set of health rules that can be defined for this job. See [_](#jobs..health). + + * - `job_clusters` + - Sequence + - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobs..job_clusters). + + * - `max_concurrent_runs` + - Integer + - An optional maximum allowed number of concurrent runs of the job. Set this value if you want to be able to execute multiple runs of the same job concurrently. This is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters. This setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs. However, from then on, new runs are skipped unless there are fewer than 3 active runs. This value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped. + + * - `name` + - String + - An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding. + + * - `notification_settings` + - Map + - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobs..notification_settings). + + * - `parameters` + - Sequence + - Job-level parameter definitions. See [_](#jobs..parameters). + + * - `permissions` + - Sequence + - See [_](#jobs..permissions). + + * - `queue` + - Map + - The queue settings of the job. See [_](#jobs..queue). + + * - `run_as` + - Map + - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job. Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobs..run_as). + + * - `schedule` + - Map + - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs..schedule). + + * - `tags` + - Map + - A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job. + + * - `tasks` + - Sequence + - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobs..tasks). + + * - `timeout_seconds` + - Integer + - An optional timeout applied to each run of this job. A value of `0` means no timeout. + + * - `trigger` + - Map + - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs..trigger). + + * - `webhook_notifications` + - Map + - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobs..webhook_notifications). + + +**Example** + +The following example defines a job with the resource key `hello-job` with one notebook task: + +```yaml +resources: + jobs: + hello-job: + name: hello-job + tasks: + - task_key: hello-task + notebook_task: + notebook_path: ./hello.py +``` + +For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). + +### jobs..continuous + +**`Type: Map`** + +An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `pause_status` + - String + - Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED. + + +### jobs..deployment + +**`Type: Map`** + +Deployment information for jobs managed by external sources. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `kind` + - String + - The kind of deployment that manages the job. * `BUNDLE`: The job is managed by Databricks Asset Bundle. + + * - `metadata_file_path` + - String + - Path of the file that contains deployment metadata. + + +### jobs..email_notifications + +**`Type: Map`** + +An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `no_alert_for_skipped_runs` + - Boolean + - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + + * - `on_duration_warning_threshold_exceeded` + - Sequence + - A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. + + * - `on_failure` + - Sequence + - A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent. + + * - `on_start` + - Sequence + - A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + + * - `on_streaming_backlog_exceeded` + - Sequence + - A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. + + * - `on_success` + - Sequence + - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + + +### jobs..environments + +**`Type: Sequence`** + +A list of task execution environment specifications that can be referenced by serverless tasks of this job. +An environment is required to be present for serverless tasks. +For serverless notebook tasks, the environment is accessible in the notebook environment panel. +For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `environment_key` + - String + - The key of an environment. It has to be unique within a job. + + * - `spec` + - Map + - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobs..environments.spec). + + +### jobs..environments.spec + +**`Type: Map`** + +The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. +In this minimal environment spec, only pip dependencies are supported. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `client` + - String + - Client version used by the environment The client is the user-facing environment of the runtime. Each client comes with a specific set of pre-installed libraries. The version is a string, consisting of the major client version. + + * - `dependencies` + - Sequence + - List of pip dependencies, as supported by the version of pip in this environment. Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be , , (WSFS or Volumes in Databricks), E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"] + + +### jobs..git_source + +**`Type: Map`** + +An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. + +If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. + +Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `git_branch` + - String + - Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit. + + * - `git_commit` + - String + - Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag. + + * - `git_provider` + - String + - Unique identifier of the service used to host the Git repository. The value is case insensitive. + + * - `git_snapshot` + - Map + - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobs..git_source.git_snapshot). + + * - `git_tag` + - String + - Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit. + + * - `git_url` + - String + - URL of the repository to be cloned by this job. + + * - `job_source` + - Map + - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobs..git_source.job_source). + + +### jobs..git_source.git_snapshot + +**`Type: Map`** + +Read-only state of the remote repository at the time the job was run. This field is only included on job runs. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `used_commit` + - String + - Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to. + + +### jobs..git_source.job_source + +**`Type: Map`** + +The source of the job specification in the remote repository when the job is source controlled. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dirty_state` + - String + - Dirty state indicates the job is not fully synced with the job specification in the remote repository. Possible values are: * `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced. * `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced. + + * - `import_from_git_branch` + - String + - Name of the branch which the job is imported from. + + * - `job_config_path` + - String + - Path of the job YAML file that contains the job specification. + + +### jobs..health + +**`Type: Map`** + +An optional set of health rules that can be defined for this job. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `rules` + - Sequence + - See [_](#jobs..health.rules). + + +### jobs..health.rules + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `metric` + - String + - Specifies the health metric that is being evaluated for a particular health rule. * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview. + + * - `op` + - String + - Specifies the operator used to compare the health metric value with the specified threshold. + + * - `value` + - Integer + - Specifies the threshold value that the health metric should obey to satisfy the health rule. + + +### jobs..job_clusters + +**`Type: Sequence`** + +A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. +If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `job_cluster_key` + - String + - A unique name for the job cluster. This field is required and must be unique within the job. `JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution. + + * - `new_cluster` + - Map + - If new_cluster, a description of a cluster that is created for each task. See [_](#jobs..job_clusters.new_cluster). + + +### jobs..job_clusters.new_cluster + +**`Type: Map`** + +If new_cluster, a description of a cluster that is created for each task. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apply_policy_default_values` + - Boolean + - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied. + + * - `autoscale` + - Map + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs..job_clusters.new_cluster.autoscale). + + * - `autotermination_minutes` + - Integer + - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination. + + * - `aws_attributes` + - Map + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.aws_attributes). + + * - `azure_attributes` + - Map + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.azure_attributes). + + * - `cluster_log_conf` + - Map + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf). + + * - `cluster_name` + - String + - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. + + * - `custom_tags` + - Map + - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags + + * - `data_security_mode` + - String + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + + * - `docker_image` + - Map + - See [_](#jobs..job_clusters.new_cluster.docker_image). + + * - `driver_instance_pool_id` + - String + - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. + + * - `driver_node_type_id` + - String + - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. + + * - `enable_elastic_disk` + - Boolean + - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details. + + * - `enable_local_disk_encryption` + - Boolean + - Whether to enable LUKS on cluster VMs' local disks + + * - `gcp_attributes` + - Map + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.gcp_attributes). + + * - `init_scripts` + - Sequence + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobs..job_clusters.new_cluster.init_scripts). + + * - `instance_pool_id` + - String + - The optional ID of the instance pool to which the cluster belongs. + + * - `is_single_node` + - Boolean + - This field can only be used with `kind`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` + + * - `kind` + - String + - + + * - `node_type_id` + - String + - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. + + * - `num_workers` + - Integer + - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. + + * - `policy_id` + - String + - The ID of the cluster policy used to create the cluster if applicable. + + * - `runtime_engine` + - String + - Determines the cluster's runtime engine, either standard or Photon. This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`. If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. + + * - `single_user_name` + - String + - Single user name if data_security_mode is `SINGLE_USER` + + * - `spark_conf` + - Map + - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. + + * - `spark_env_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well. Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` + + * - `spark_version` + - String + - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. + + * - `ssh_public_keys` + - Sequence + - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + + * - `use_ml_runtime` + - Boolean + - This field can only be used with `kind`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. + + * - `workload_type` + - Map + - See [_](#jobs..job_clusters.new_cluster.workload_type). + + +### jobs..job_clusters.new_cluster.autoscale + +**`Type: Map`** + +Parameters needed in order to automatically scale clusters up and down based on load. +Note: autoscaling works best with DB runtime versions 3.0 or later. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `max_workers` + - Integer + - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`. + + * - `min_workers` + - Integer + - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. + + +### jobs..job_clusters.new_cluster.aws_attributes + +**`Type: Map`** + +Attributes related to clusters running on Amazon Web Services. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. + + * - `ebs_volume_count` + - Integer + - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail. These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc. If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes. Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden. + + * - `ebs_volume_iops` + - Integer + - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_size` + - Integer + - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096. + + * - `ebs_volume_throughput` + - Integer + - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_type` + - String + - The type of EBS volumes that will be launched with this cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `instance_profile_arn` + - String + - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator. This feature may only be available to certain customer plans. If this field is ommitted, we will pull in the default from the conf if it exists. + + * - `spot_bid_price_percent` + - Integer + - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000. The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent. + + * - `zone_id` + - String + - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. + + +### jobs..job_clusters.new_cluster.azure_attributes + +**`Type: Map`** + +Attributes related to clusters running on Microsoft Azure. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `log_analytics_info` + - Map + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs..job_clusters.new_cluster.azure_attributes.log_analytics_info). + + * - `spot_bid_max_price` + - Any + - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. + + +### jobs..job_clusters.new_cluster.azure_attributes.log_analytics_info + +**`Type: Map`** + +Defines values necessary to configure and run Azure Log Analytics agent + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `log_analytics_primary_key` + - String + - + + * - `log_analytics_workspace_id` + - String + - + + +### jobs..job_clusters.new_cluster.cluster_log_conf + +**`Type: Map`** + +The configuration for delivering spark logs to a long-term storage destination. +Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified +for one cluster. If the conf is given, the logs will be delivered to the destination every +`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while +the destination of executor logs is `$destination/$clusterId/executor`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf.dbfs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf.s3). + + +### jobs..job_clusters.new_cluster.cluster_log_conf.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### jobs..job_clusters.new_cluster.cluster_log_conf.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### jobs..job_clusters.new_cluster.docker_image + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `basic_auth` + - Map + - See [_](#jobs..job_clusters.new_cluster.docker_image.basic_auth). + + * - `url` + - String + - URL of the docker image. + + +### jobs..job_clusters.new_cluster.docker_image.basic_auth + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `password` + - String + - Password of the user + + * - `username` + - String + - Name of the user + + +### jobs..job_clusters.new_cluster.gcp_attributes + +**`Type: Map`** + +Attributes related to clusters running on Google Cloud Platform. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable. + + * - `boot_disk_size` + - Integer + - boot disk size in GB + + * - `google_service_account` + - String + - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator. + + * - `local_ssd_count` + - Integer + - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type. + + * - `use_preemptible_executors` + - Boolean + - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead. + + * - `zone_id` + - String + - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. + + +### jobs..job_clusters.new_cluster.init_scripts + +**`Type: Sequence`** + +The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `abfss` + - Map + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobs..job_clusters.new_cluster.init_scripts.abfss). + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.dbfs). + + * - `file` + - Map + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.file). + + * - `gcs` + - Map + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.gcs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..job_clusters.new_cluster.init_scripts.s3). + + * - `volumes` + - Map + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.volumes). + + * - `workspace` + - Map + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.workspace). + + +### jobs..job_clusters.new_cluster.init_scripts.abfss + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } } + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. + + +### jobs..job_clusters.new_cluster.init_scripts.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### jobs..job_clusters.new_cluster.init_scripts.file + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "file" : { "destination" : "file:/my/local/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - local file destination, e.g. `file:/my/local/file.sh` + + +### jobs..job_clusters.new_cluster.init_scripts.gcs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` + + +### jobs..job_clusters.new_cluster.init_scripts.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### jobs..job_clusters.new_cluster.init_scripts.volumes + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` + + +### jobs..job_clusters.new_cluster.init_scripts.workspace + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` + + +### jobs..job_clusters.new_cluster.workload_type + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `clients` + - Map + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs..job_clusters.new_cluster.workload_type.clients). + + +### jobs..job_clusters.new_cluster.workload_type.clients + +**`Type: Map`** + + defined what type of clients can use the cluster. E.g. Notebooks, Jobs + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jobs` + - Boolean + - With jobs set, the cluster can be used for jobs + + * - `notebooks` + - Boolean + - With notebooks set, this cluster can be used for notebooks + + +### jobs..notification_settings + +**`Type: Map`** + +Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `no_alert_for_canceled_runs` + - Boolean + - If true, do not send notifications to recipients specified in `on_failure` if the run is canceled. + + * - `no_alert_for_skipped_runs` + - Boolean + - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. + + +### jobs..parameters + +**`Type: Sequence`** + +Job-level parameter definitions + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `default` + - String + - Default value of the parameter. + + * - `name` + - String + - The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.` + + +### jobs..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### jobs..queue + +**`Type: Map`** + +The queue settings of the job. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - If true, enable queueing for the job. This is a required field. + + +### jobs..run_as + +**`Type: Map`** + +Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job. + +Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `service_principal_name` + - String + - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + + * - `user_name` + - String + - The email of an active workspace user. Non-admin users can only set this field to their own email. + + +### jobs..schedule + +**`Type: Map`** + +An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `pause_status` + - String + - Indicate whether this schedule is paused or not. + + * - `quartz_cron_expression` + - String + - A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required. + + * - `timezone_id` + - String + - A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required. + + +### jobs..tasks + +**`Type: Sequence`** + +A list of task specifications to be executed by this job. +If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `clean_rooms_notebook_task` + - Map + - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobs..tasks.clean_rooms_notebook_task). + + * - `condition_task` + - Map + - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobs..tasks.condition_task). + + * - `dbt_task` + - Map + - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobs..tasks.dbt_task). + + * - `depends_on` + - Sequence + - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobs..tasks.depends_on). + + * - `description` + - String + - An optional description for this task. + + * - `disable_auto_optimization` + - Boolean + - An option to disable auto optimization in serverless + + * - `email_notifications` + - Map + - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobs..tasks.email_notifications). + + * - `environment_key` + - String + - The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute. + + * - `existing_cluster_id` + - String + - If existing_cluster_id, the ID of an existing cluster that is used for all runs. When running jobs or tasks on an existing cluster, you may need to manually restart the cluster if it stops responding. We suggest running jobs and tasks on new clusters for greater reliability + + * - `for_each_task` + - Map + - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobs..tasks.for_each_task). + + * - `health` + - Map + - An optional set of health rules that can be defined for this job. See [_](#jobs..tasks.health). + + * - `job_cluster_key` + - String + - If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`. + + * - `libraries` + - Sequence + - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobs..tasks.libraries). + + * - `max_retries` + - Integer + - An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry. + + * - `min_retry_interval_millis` + - Integer + - An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried. + + * - `new_cluster` + - Map + - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobs..tasks.new_cluster). + + * - `notebook_task` + - Map + - The task runs a notebook when the `notebook_task` field is present. See [_](#jobs..tasks.notebook_task). + + * - `notification_settings` + - Map + - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobs..tasks.notification_settings). + + * - `pipeline_task` + - Map + - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobs..tasks.pipeline_task). + + * - `python_wheel_task` + - Map + - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobs..tasks.python_wheel_task). + + * - `retry_on_timeout` + - Boolean + - An optional policy to specify whether to retry a job when it times out. The default behavior is to not retry on timeout. + + * - `run_if` + - String + - An optional value specifying the condition determining whether the task is run once its dependencies have been completed. * `ALL_SUCCESS`: All dependencies have executed and succeeded * `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded * `NONE_FAILED`: None of the dependencies have failed and at least one was executed * `ALL_DONE`: All dependencies have been completed * `AT_LEAST_ONE_FAILED`: At least one dependency failed * `ALL_FAILED`: ALl dependencies have failed + + * - `run_job_task` + - Map + - The task triggers another job when the `run_job_task` field is present. See [_](#jobs..tasks.run_job_task). + + * - `spark_jar_task` + - Map + - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobs..tasks.spark_jar_task). + + * - `spark_python_task` + - Map + - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobs..tasks.spark_python_task). + + * - `spark_submit_task` + - Map + - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute. In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations. `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters. By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage. The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobs..tasks.spark_submit_task). + + * - `sql_task` + - Map + - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobs..tasks.sql_task). + + * - `task_key` + - String + - A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset. + + * - `timeout_seconds` + - Integer + - An optional timeout applied to each run of this job task. A value of `0` means no timeout. + + * - `webhook_notifications` + - Map + - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobs..tasks.webhook_notifications). + + +### jobs..tasks.clean_rooms_notebook_task + +**`Type: Map`** + +The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook +when the `clean_rooms_notebook_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `clean_room_name` + - String + - The clean room that the notebook belongs to. + + * - `etag` + - String + - Checksum to validate the freshness of the notebook resource (i.e. the notebook being run is the latest version). It can be fetched by calling the :method:cleanroomassets/get API. + + * - `notebook_base_parameters` + - Map + - Base parameters to be used for the clean room notebook job. + + * - `notebook_name` + - String + - Name of the notebook being run. + + +### jobs..tasks.condition_task + +**`Type: Map`** + +The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. +The condition task does not require a cluster to execute and does not support retries or notifications. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `left` + - String + - The left operand of the condition task. Can be either a string value or a job state or parameter reference. + + * - `op` + - String + - * `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`. * `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” >= “12”` will evaluate to `true`, `“10.0” >= “12”` will evaluate to `false`. The boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison. + + * - `right` + - String + - The right operand of the condition task. Can be either a string value or a job state or parameter reference. + + +### jobs..tasks.dbt_task + +**`Type: Map`** + +The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog` + - String + - Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks >= 1.1.1. + + * - `commands` + - Sequence + - A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided. + + * - `profiles_directory` + - String + - Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used. + + * - `project_directory` + - String + - Path to the project directory. Optional for Git sourced tasks, in which case if no value is provided, the root of the Git repository is used. + + * - `schema` + - String + - Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used. + + * - `source` + - String + - Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud Git provider. + + * - `warehouse_id` + - String + - ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument. + + +### jobs..tasks.depends_on + +**`Type: Sequence`** + +An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. +The key is `task_key`, and the value is the name assigned to the dependent task. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `outcome` + - String + - Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run. + + * - `task_key` + - String + - The name of the task this task depends on. + + +### jobs..tasks.email_notifications + +**`Type: Map`** + +An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `no_alert_for_skipped_runs` + - Boolean + - If true, do not send email to recipients specified in `on_failure` if the run is skipped. This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field. + + * - `on_duration_warning_threshold_exceeded` + - Sequence + - A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent. + + * - `on_failure` + - Sequence + - A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent. + + * - `on_start` + - Sequence + - A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + + * - `on_streaming_backlog_exceeded` + - Sequence + - A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. + + * - `on_success` + - Sequence + - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. + + +### jobs..tasks.for_each_task + +**`Type: Map`** + +The task executes a nested task for every input provided when the `for_each_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `concurrency` + - Integer + - An optional maximum allowed number of concurrent runs of the task. Set this value if you want to be able to execute multiple runs of the task concurrently. + + * - `inputs` + - String + - Array for task to iterate on. This can be a JSON string or a reference to an array parameter. + + * - `task` + - Map + - Configuration for the task that will be run for each element in the array + + +### jobs..tasks.health + +**`Type: Map`** + +An optional set of health rules that can be defined for this job. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `rules` + - Sequence + - See [_](#jobs..tasks.health.rules). + + +### jobs..tasks.health.rules + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `metric` + - String + - Specifies the health metric that is being evaluated for a particular health rule. * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview. + + * - `op` + - String + - Specifies the operator used to compare the health metric value with the specified threshold. + + * - `value` + - Integer + - Specifies the threshold value that the health metric should obey to satisfy the health rule. + + +### jobs..tasks.libraries + +**`Type: Sequence`** + +An optional list of libraries to be installed on the cluster. +The default value is an empty list. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `cran` + - Map + - Specification of a CRAN library to be installed as part of the library. See [_](#jobs..tasks.libraries.cran). + + * - `egg` + - String + - Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above. + + * - `jar` + - String + - URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "jar": "/Workspace/path/to/library.jar" }`, `{ "jar" : "/Volumes/path/to/library.jar" }` or `{ "jar": "s3://my-bucket/library.jar" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI. + + * - `maven` + - Map + - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobs..tasks.libraries.maven). + + * - `pypi` + - Map + - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobs..tasks.libraries.pypi). + + * - `requirements` + - String + - URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported. For example: `{ "requirements": "/Workspace/path/to/requirements.txt" }` or `{ "requirements" : "/Volumes/path/to/requirements.txt" }` + + * - `whl` + - String + - URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "whl": "/Workspace/path/to/library.whl" }`, `{ "whl" : "/Volumes/path/to/library.whl" }` or `{ "whl": "s3://my-bucket/library.whl" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI. + + +### jobs..tasks.libraries.cran + +**`Type: Map`** + +Specification of a CRAN library to be installed as part of the library + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `package` + - String + - The name of the CRAN package to install. + + * - `repo` + - String + - The repository where the package can be found. If not specified, the default CRAN repo is used. + + +### jobs..tasks.libraries.maven + +**`Type: Map`** + +Specification of a maven library to be installed. For example: +`{ "coordinates": "org.jsoup:jsoup:1.7.2" }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `coordinates` + - String + - Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2". + + * - `exclusions` + - Sequence + - List of dependences to exclude. For example: `["slf4j:slf4j", "*:hadoop-client"]`. Maven dependency exclusions: https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html. + + * - `repo` + - String + - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched. + + +### jobs..tasks.libraries.pypi + +**`Type: Map`** + +Specification of a PyPi library to be installed. For example: +`{ "package": "simplejson" }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `package` + - String + - The name of the pypi package to install. An optional exact version specification is also supported. Examples: "simplejson" and "simplejson==3.8.0". + + * - `repo` + - String + - The repository where the package can be found. If not specified, the default pip index is used. + + +### jobs..tasks.new_cluster + +**`Type: Map`** + +If new_cluster, a description of a new cluster that is created for each run. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apply_policy_default_values` + - Boolean + - When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied. + + * - `autoscale` + - Map + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs..tasks.new_cluster.autoscale). + + * - `autotermination_minutes` + - Integer + - Automatically terminates the cluster after it is inactive for this time in minutes. If not set, this cluster will not be automatically terminated. If specified, the threshold must be between 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic termination. + + * - `aws_attributes` + - Map + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.aws_attributes). + + * - `azure_attributes` + - Map + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.azure_attributes). + + * - `cluster_log_conf` + - Map + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs..tasks.new_cluster.cluster_log_conf). + + * - `cluster_name` + - String + - Cluster name requested by the user. This doesn't have to be unique. If not specified at creation, the cluster name will be an empty string. + + * - `custom_tags` + - Map + - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags + + * - `data_security_mode` + - String + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + + * - `docker_image` + - Map + - See [_](#jobs..tasks.new_cluster.docker_image). + + * - `driver_instance_pool_id` + - String + - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. + + * - `driver_node_type_id` + - String + - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. + + * - `enable_elastic_disk` + - Boolean + - Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS permissions to function correctly - refer to the User Guide for more details. + + * - `enable_local_disk_encryption` + - Boolean + - Whether to enable LUKS on cluster VMs' local disks + + * - `gcp_attributes` + - Map + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.gcp_attributes). + + * - `init_scripts` + - Sequence + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobs..tasks.new_cluster.init_scripts). + + * - `instance_pool_id` + - String + - The optional ID of the instance pool to which the cluster belongs. + + * - `is_single_node` + - Boolean + - This field can only be used with `kind`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` + + * - `kind` + - String + - + + * - `node_type_id` + - String + - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. + + * - `num_workers` + - Integer + - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. + + * - `policy_id` + - String + - The ID of the cluster policy used to create the cluster if applicable. + + * - `runtime_engine` + - String + - Determines the cluster's runtime engine, either standard or Photon. This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`. If left unspecified, the runtime engine defaults to standard unless the spark_version contains -photon-, in which case Photon will be used. + + * - `single_user_name` + - String + - Single user name if data_security_mode is `SINGLE_USER` + + * - `spark_conf` + - Map + - An object containing a set of optional, user-specified Spark configuration key-value pairs. Users can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. + + * - `spark_env_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well. Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` + + * - `spark_version` + - String + - The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. + + * - `ssh_public_keys` + - Sequence + - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + + * - `use_ml_runtime` + - Boolean + - This field can only be used with `kind`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. + + * - `workload_type` + - Map + - See [_](#jobs..tasks.new_cluster.workload_type). + + +### jobs..tasks.new_cluster.autoscale + +**`Type: Map`** + +Parameters needed in order to automatically scale clusters up and down based on load. +Note: autoscaling works best with DB runtime versions 3.0 or later. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `max_workers` + - Integer + - The maximum number of workers to which the cluster can scale up when overloaded. Note that `max_workers` must be strictly greater than `min_workers`. + + * - `min_workers` + - Integer + - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. + + +### jobs..tasks.new_cluster.aws_attributes + +**`Type: Map`** + +Attributes related to clusters running on Amazon Web Services. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. + + * - `ebs_volume_count` + - Integer + - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail. These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc. If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes. Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden. + + * - `ebs_volume_iops` + - Integer + - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_size` + - Integer + - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096. + + * - `ebs_volume_throughput` + - Integer + - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_type` + - String + - The type of EBS volumes that will be launched with this cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `instance_profile_arn` + - String + - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator. This feature may only be available to certain customer plans. If this field is ommitted, we will pull in the default from the conf if it exists. + + * - `spot_bid_price_percent` + - Integer + - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000. The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent. + + * - `zone_id` + - String + - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. + + +### jobs..tasks.new_cluster.azure_attributes + +**`Type: Map`** + +Attributes related to clusters running on Microsoft Azure. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `log_analytics_info` + - Map + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs..tasks.new_cluster.azure_attributes.log_analytics_info). + + * - `spot_bid_max_price` + - Any + - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. + + +### jobs..tasks.new_cluster.azure_attributes.log_analytics_info + +**`Type: Map`** + +Defines values necessary to configure and run Azure Log Analytics agent + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `log_analytics_primary_key` + - String + - + + * - `log_analytics_workspace_id` + - String + - + + +### jobs..tasks.new_cluster.cluster_log_conf + +**`Type: Map`** + +The configuration for delivering spark logs to a long-term storage destination. +Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified +for one cluster. If the conf is given, the logs will be delivered to the destination every +`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while +the destination of executor logs is `$destination/$clusterId/executor`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..tasks.new_cluster.cluster_log_conf.dbfs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..tasks.new_cluster.cluster_log_conf.s3). + + +### jobs..tasks.new_cluster.cluster_log_conf.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### jobs..tasks.new_cluster.cluster_log_conf.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### jobs..tasks.new_cluster.docker_image + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `basic_auth` + - Map + - See [_](#jobs..tasks.new_cluster.docker_image.basic_auth). + + * - `url` + - String + - URL of the docker image. + + +### jobs..tasks.new_cluster.docker_image.basic_auth + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `password` + - String + - Password of the user + + * - `username` + - String + - Name of the user + + +### jobs..tasks.new_cluster.gcp_attributes + +**`Type: Map`** + +Attributes related to clusters running on Google Cloud Platform. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable. + + * - `boot_disk_size` + - Integer + - boot disk size in GB + + * - `google_service_account` + - String + - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator. + + * - `local_ssd_count` + - Integer + - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type. + + * - `use_preemptible_executors` + - Boolean + - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead. + + * - `zone_id` + - String + - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. + + +### jobs..tasks.new_cluster.init_scripts + +**`Type: Sequence`** + +The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `abfss` + - Map + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobs..tasks.new_cluster.init_scripts.abfss). + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.dbfs). + + * - `file` + - Map + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.file). + + * - `gcs` + - Map + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.gcs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..tasks.new_cluster.init_scripts.s3). + + * - `volumes` + - Map + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.volumes). + + * - `workspace` + - Map + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.workspace). + + +### jobs..tasks.new_cluster.init_scripts.abfss + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } } + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. + + +### jobs..tasks.new_cluster.init_scripts.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### jobs..tasks.new_cluster.init_scripts.file + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "file" : { "destination" : "file:/my/local/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - local file destination, e.g. `file:/my/local/file.sh` + + +### jobs..tasks.new_cluster.init_scripts.gcs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` + + +### jobs..tasks.new_cluster.init_scripts.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### jobs..tasks.new_cluster.init_scripts.volumes + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` + + +### jobs..tasks.new_cluster.init_scripts.workspace + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` + + +### jobs..tasks.new_cluster.workload_type + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `clients` + - Map + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs..tasks.new_cluster.workload_type.clients). + + +### jobs..tasks.new_cluster.workload_type.clients + +**`Type: Map`** + + defined what type of clients can use the cluster. E.g. Notebooks, Jobs + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jobs` + - Boolean + - With jobs set, the cluster can be used for jobs + + * - `notebooks` + - Boolean + - With notebooks set, this cluster can be used for notebooks + + +### jobs..tasks.notebook_task + +**`Type: Map`** + +The task runs a notebook when the `notebook_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `base_parameters` + - Map + - Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run Now with parameters specified, the two parameters maps are merged. If the same key is specified in `base_parameters` and in `run-now`, the value from `run-now` is used. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. If the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters, the default value from the notebook is used. Retrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets). The JSON representation of this field cannot exceed 1MB. + + * - `notebook_path` + - String + - The path of the notebook to be run in the Databricks workspace or remote repository. For notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash. For notebooks stored in a remote repository, the path must be relative. This field is required. + + * - `source` + - String + - Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: Notebook is located in Databricks workspace. * `GIT`: Notebook is located in cloud Git provider. + + * - `warehouse_id` + - String + - Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses. Note that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail. + + +### jobs..tasks.notification_settings + +**`Type: Map`** + +Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alert_on_last_attempt` + - Boolean + - If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run. + + * - `no_alert_for_canceled_runs` + - Boolean + - If true, do not send notifications to recipients specified in `on_failure` if the run is canceled. + + * - `no_alert_for_skipped_runs` + - Boolean + - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. + + +### jobs..tasks.pipeline_task + +**`Type: Map`** + +The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `full_refresh` + - Boolean + - If true, triggers a full refresh on the delta live table. + + * - `pipeline_id` + - String + - The full name of the pipeline task to execute. + + +### jobs..tasks.python_wheel_task + +**`Type: Map`** + +The task runs a Python wheel when the `python_wheel_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `entry_point` + - String + - Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()` + + * - `named_parameters` + - Map + - Command-line parameters passed to Python wheel task in the form of `["--name=task", "--data=dbfs:/path/to/data.json"]`. Leave it empty if `parameters` is not null. + + * - `package_name` + - String + - Name of the package to execute + + * - `parameters` + - Sequence + - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null. + + +### jobs..tasks.run_job_task + +**`Type: Map`** + +The task triggers another job when the `run_job_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dbt_commands` + - Sequence + - An array of commands to execute for jobs with the dbt task, for example `"dbt_commands": ["dbt deps", "dbt seed", "dbt deps", "dbt seed", "dbt run"]` + + * - `jar_params` + - Sequence + - A list of parameters for jobs with Spark JAR tasks, for example `"jar_params": ["john doe", "35"]`. The parameters are used to invoke the main function of the main class specified in the Spark JAR task. If not specified upon `run-now`, it defaults to an empty list. jar_params cannot be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. + + * - `job_id` + - Integer + - ID of the job to trigger. + + * - `job_parameters` + - Map + - Job-level parameters used to trigger the job. + + * - `notebook_params` + - Map + - A map from keys to values for jobs with notebook task, for example `"notebook_params": {"name": "john doe", "age": "35"}`. The map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function. If not specified upon `run-now`, the triggered run uses the job’s base parameters. notebook_params cannot be specified in conjunction with jar_params. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. + + * - `pipeline_params` + - Map + - Controls whether the pipeline should perform a full refresh. See [_](#jobs..tasks.run_job_task.pipeline_params). + + * - `python_named_params` + - Map + - + + * - `python_params` + - Sequence + - A list of parameters for jobs with Python tasks, for example `"python_params": ["john doe", "35"]`. The parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. Important These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. + + * - `spark_submit_params` + - Sequence + - A list of parameters for jobs with spark submit task, for example `"spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"]`. The parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs Important These parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. + + * - `sql_params` + - Map + - A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. + + +### jobs..tasks.run_job_task.pipeline_params + +**`Type: Map`** + +Controls whether the pipeline should perform a full refresh + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `full_refresh` + - Boolean + - If true, triggers a full refresh on the delta live table. + + +### jobs..tasks.spark_jar_task + +**`Type: Map`** + +The task runs a JAR when the `spark_jar_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `jar_uri` + - String + - Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create. + + * - `main_class_name` + - String + - The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library. The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail. + + * - `parameters` + - Sequence + - Parameters passed to the main method. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. + + * - `run_as_repl` + - Boolean + - Deprecated. A value of `false` is no longer supported. + + +### jobs..tasks.spark_python_task + +**`Type: Map`** + +The task runs a Python file when the `spark_python_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `parameters` + - Sequence + - Command line parameters passed to the Python file. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. + + * - `python_file` + - String + - The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required. + + * - `source` + - String + - Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local Databricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`, the Python file will be retrieved from a Git repository defined in `git_source`. * `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI. * `GIT`: The Python file is located in a remote Git repository. + + +### jobs..tasks.spark_submit_task + +**`Type: Map`** + +(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute. + +In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations. + +`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters. + +By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage. + +The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `parameters` + - Sequence + - Command-line parameters passed to spark submit. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. + + +### jobs..tasks.sql_task + +**`Type: Map`** + +The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alert` + - Map + - If alert, indicates that this job must refresh a SQL alert. See [_](#jobs..tasks.sql_task.alert). + + * - `dashboard` + - Map + - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobs..tasks.sql_task.dashboard). + + * - `file` + - Map + - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobs..tasks.sql_task.file). + + * - `parameters` + - Map + - Parameters to be used for each run of this job. The SQL alert task does not support custom parameters. + + * - `query` + - Map + - If query, indicates that this job must execute a SQL query. See [_](#jobs..tasks.sql_task.query). + + * - `warehouse_id` + - String + - The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs. + + +### jobs..tasks.sql_task.alert + +**`Type: Map`** + +If alert, indicates that this job must refresh a SQL alert. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alert_id` + - String + - The canonical identifier of the SQL alert. + + * - `pause_subscriptions` + - Boolean + - If true, the alert notifications are not sent to subscribers. + + * - `subscriptions` + - Sequence + - If specified, alert notifications are sent to subscribers. See [_](#jobs..tasks.sql_task.alert.subscriptions). + + +### jobs..tasks.sql_task.alert.subscriptions + +**`Type: Sequence`** + +If specified, alert notifications are sent to subscribers. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination_id` + - String + - The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications. + + * - `user_name` + - String + - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications. + + +### jobs..tasks.sql_task.dashboard + +**`Type: Map`** + +If dashboard, indicates that this job must refresh a SQL dashboard. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `custom_subject` + - String + - Subject of the email sent to subscribers of this task. + + * - `dashboard_id` + - String + - The canonical identifier of the SQL dashboard. + + * - `pause_subscriptions` + - Boolean + - If true, the dashboard snapshot is not taken, and emails are not sent to subscribers. + + * - `subscriptions` + - Sequence + - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobs..tasks.sql_task.dashboard.subscriptions). + + +### jobs..tasks.sql_task.dashboard.subscriptions + +**`Type: Sequence`** + +If specified, dashboard snapshots are sent to subscriptions. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination_id` + - String + - The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications. + + * - `user_name` + - String + - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications. + + +### jobs..tasks.sql_task.file + +**`Type: Map`** + +If file, indicates that this job runs a SQL file in a remote Git repository. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `path` + - String + - Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths. + + * - `source` + - String + - Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider. + + +### jobs..tasks.sql_task.query + +**`Type: Map`** + +If query, indicates that this job must execute a SQL query. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `query_id` + - String + - The canonical identifier of the SQL query. + + +### jobs..tasks.webhook_notifications + +**`Type: Map`** + +A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `on_duration_warning_threshold_exceeded` + - Sequence + - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs..tasks.webhook_notifications.on_duration_warning_threshold_exceeded). + + * - `on_failure` + - Sequence + - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs..tasks.webhook_notifications.on_failure). + + * - `on_start` + - Sequence + - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs..tasks.webhook_notifications.on_start). + + * - `on_streaming_backlog_exceeded` + - Sequence + - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs..tasks.webhook_notifications.on_streaming_backlog_exceeded). + + * - `on_success` + - Sequence + - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs..tasks.webhook_notifications.on_success). + + +### jobs..tasks.webhook_notifications.on_duration_warning_threshold_exceeded + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..tasks.webhook_notifications.on_failure + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..tasks.webhook_notifications.on_start + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..tasks.webhook_notifications.on_streaming_backlog_exceeded + +**`Type: Sequence`** + +An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. +Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. +Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. +A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..tasks.webhook_notifications.on_success + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..trigger + +**`Type: Map`** + +A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `file_arrival` + - Map + - File arrival trigger settings. See [_](#jobs..trigger.file_arrival). + + * - `pause_status` + - String + - Whether this trigger is paused or not. + + * - `periodic` + - Map + - Periodic trigger settings. See [_](#jobs..trigger.periodic). + + * - `table` + - Map + - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobs..trigger.table). + + * - `table_update` + - Map + - See [_](#jobs..trigger.table_update). + + +### jobs..trigger.file_arrival + +**`Type: Map`** + +File arrival trigger settings. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `min_time_between_triggers_seconds` + - Integer + - If set, the trigger starts a run only after the specified amount of time passed since the last time the trigger fired. The minimum allowed value is 60 seconds + + * - `url` + - String + - URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location. + + * - `wait_after_last_change_seconds` + - Integer + - If set, the trigger starts a run only after no file activity has occurred for the specified amount of time. This makes it possible to wait for a batch of incoming files to arrive before triggering a run. The minimum allowed value is 60 seconds. + + +### jobs..trigger.periodic + +**`Type: Map`** + +Periodic trigger settings. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `interval` + - Integer + - The interval at which the trigger should run. + + * - `unit` + - String + - The unit of time for the interval. + + +### jobs..trigger.table + +**`Type: Map`** + +Old table trigger settings name. Deprecated in favor of `table_update`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `condition` + - String + - The table(s) condition based on which to trigger a job run. + + * - `min_time_between_triggers_seconds` + - Integer + - If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds. + + * - `table_names` + - Sequence + - A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`. + + * - `wait_after_last_change_seconds` + - Integer + - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. + + +### jobs..trigger.table_update + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `condition` + - String + - The table(s) condition based on which to trigger a job run. + + * - `min_time_between_triggers_seconds` + - Integer + - If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds. + + * - `table_names` + - Sequence + - A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`. + + * - `wait_after_last_change_seconds` + - Integer + - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. + + +### jobs..webhook_notifications + +**`Type: Map`** + +A collection of system notification IDs to notify when runs of this job begin or complete. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `on_duration_warning_threshold_exceeded` + - Sequence + - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs..webhook_notifications.on_duration_warning_threshold_exceeded). + + * - `on_failure` + - Sequence + - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs..webhook_notifications.on_failure). + + * - `on_start` + - Sequence + - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs..webhook_notifications.on_start). + + * - `on_streaming_backlog_exceeded` + - Sequence + - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs..webhook_notifications.on_streaming_backlog_exceeded). + + * - `on_success` + - Sequence + - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs..webhook_notifications.on_success). + + +### jobs..webhook_notifications.on_duration_warning_threshold_exceeded + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..webhook_notifications.on_failure + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..webhook_notifications.on_start + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..webhook_notifications.on_streaming_backlog_exceeded + +**`Type: Sequence`** + +An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. +Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. +Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. +A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +### jobs..webhook_notifications.on_success + +**`Type: Sequence`** + +An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `id` + - String + - + + +## model_serving_endpoints + +**`Type: Map`** + +The model_serving_endpoint resource allows you to define [model serving endpoints](/api/workspace/servingendpoints/create). See [_](/machine-learning/model-serving/manage-serving-endpoints.md). + +```yaml +model_serving_endpoints: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `ai_gateway` + - Map + - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpoints..ai_gateway). + + * - `config` + - Map + - The core config of the serving endpoint. See [_](#model_serving_endpoints..config). + + * - `name` + - String + - The name of the serving endpoint. This field is required and must be unique across a Databricks workspace. An endpoint name can consist of alphanumeric characters, dashes, and underscores. + + * - `permissions` + - Sequence + - See [_](#model_serving_endpoints..permissions). + + * - `rate_limits` + - Sequence + - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpoints..rate_limits). + + * - `route_optimized` + - Boolean + - Enable route optimization for the serving endpoint. + + * - `tags` + - Sequence + - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpoints..tags). + + +**Example** + +The following example defines a model serving endpoint: + +```yaml +resources: + model_serving_endpoints: + uc_model_serving_endpoint: + name: "uc-model-endpoint" + config: + served_entities: + - entity_name: "myCatalog.mySchema.my-ads-model" + entity_version: "10" + workload_size: "Small" + scale_to_zero_enabled: "true" + traffic_config: + routes: + - served_model_name: "my-ads-model-10" + traffic_percentage: "100" + tags: + - key: "team" + value: "data science" +``` + +### model_serving_endpoints..ai_gateway + +**`Type: Map`** + +The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `guardrails` + - Map + - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpoints..ai_gateway.guardrails). + + * - `inference_table_config` + - Map + - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpoints..ai_gateway.inference_table_config). + + * - `rate_limits` + - Sequence + - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpoints..ai_gateway.rate_limits). + + * - `usage_tracking_config` + - Map + - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpoints..ai_gateway.usage_tracking_config). + + +### model_serving_endpoints..ai_gateway.guardrails + +**`Type: Map`** + +Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `input` + - Map + - Configuration for input guardrail filters. See [_](#model_serving_endpoints..ai_gateway.guardrails.input). + + * - `output` + - Map + - Configuration for output guardrail filters. See [_](#model_serving_endpoints..ai_gateway.guardrails.output). + + +### model_serving_endpoints..ai_gateway.guardrails.input + +**`Type: Map`** + +Configuration for input guardrail filters. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `invalid_keywords` + - Sequence + - List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content. + + * - `pii` + - Map + - Configuration for guardrail PII filter. See [_](#model_serving_endpoints..ai_gateway.guardrails.input.pii). + + * - `safety` + - Boolean + - Indicates whether the safety filter is enabled. + + * - `valid_topics` + - Sequence + - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. + + +### model_serving_endpoints..ai_gateway.guardrails.input.pii + +**`Type: Map`** + +Configuration for guardrail PII filter. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `behavior` + - String + - Configuration for input guardrail filters. + + +### model_serving_endpoints..ai_gateway.guardrails.output + +**`Type: Map`** + +Configuration for output guardrail filters. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `invalid_keywords` + - Sequence + - List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content. + + * - `pii` + - Map + - Configuration for guardrail PII filter. See [_](#model_serving_endpoints..ai_gateway.guardrails.output.pii). + + * - `safety` + - Boolean + - Indicates whether the safety filter is enabled. + + * - `valid_topics` + - Sequence + - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. + + +### model_serving_endpoints..ai_gateway.guardrails.output.pii + +**`Type: Map`** + +Configuration for guardrail PII filter. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `behavior` + - String + - Configuration for input guardrail filters. + + +### model_serving_endpoints..ai_gateway.inference_table_config + +**`Type: Map`** + +Configuration for payload logging using inference tables. +Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog_name` + - String + - The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name. + + * - `enabled` + - Boolean + - Indicates whether the inference table is enabled. + + * - `schema_name` + - String + - The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name. + + * - `table_name_prefix` + - String + - The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name. + + +### model_serving_endpoints..ai_gateway.rate_limits + +**`Type: Sequence`** + +Configuration for rate limits which can be set to limit endpoint traffic. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `calls` + - Integer + - Used to specify how many calls are allowed for a key within the renewal_period. + + * - `key` + - String + - Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + + * - `renewal_period` + - String + - Renewal period field for a rate limit. Currently, only 'minute' is supported. + + +### model_serving_endpoints..ai_gateway.usage_tracking_config + +**`Type: Map`** + +Configuration to enable usage tracking using system tables. +These tables allow you to monitor operational usage on endpoints and their associated costs. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - Whether to enable usage tracking. + + +### model_serving_endpoints..config + +**`Type: Map`** + +The core config of the serving endpoint. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `auto_capture_config` + - Map + - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpoints..config.auto_capture_config). + + * - `served_entities` + - Sequence + - The list of served entities under the serving endpoint config. See [_](#model_serving_endpoints..config.served_entities). + + * - `served_models` + - Sequence + - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpoints..config.served_models). + + * - `traffic_config` + - Map + - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpoints..config.traffic_config). + + +### model_serving_endpoints..config.auto_capture_config + +**`Type: Map`** + +Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. +Note: this field is deprecated for creating new provisioned throughput endpoints, +or updating existing provisioned throughput endpoints that never have inference table configured; +in these cases please use AI Gateway to manage inference tables. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog_name` + - String + - The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled. + + * - `enabled` + - Boolean + - Indicates whether the inference table is enabled. + + * - `schema_name` + - String + - The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled. + + * - `table_name_prefix` + - String + - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. + + +### model_serving_endpoints..config.served_entities + +**`Type: Sequence`** + +The list of served entities under the serving endpoint config. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `entity_name` + - String + - The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**. + + * - `entity_version` + - String + - + + * - `environment_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}` + + * - `external_model` + - Map + - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpoints..config.served_entities.external_model). + + * - `instance_profile_arn` + - String + - ARN of the instance profile that the served entity uses to access AWS resources. + + * - `max_provisioned_throughput` + - Integer + - The maximum tokens per second that the endpoint can scale up to. + + * - `min_provisioned_throughput` + - Integer + - The minimum tokens per second that the endpoint can scale down to. + + * - `name` + - String + - The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version. + + * - `scale_to_zero_enabled` + - Boolean + - Whether the compute resources for the served entity should scale down to zero. + + * - `workload_size` + - String + - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + + * - `workload_type` + - String + - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). + + +### model_serving_endpoints..config.served_entities.external_model + +**`Type: Map`** + +The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `ai21labs_config` + - Map + - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpoints..config.served_entities.external_model.ai21labs_config). + + * - `amazon_bedrock_config` + - Map + - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpoints..config.served_entities.external_model.amazon_bedrock_config). + + * - `anthropic_config` + - Map + - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpoints..config.served_entities.external_model.anthropic_config). + + * - `cohere_config` + - Map + - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpoints..config.served_entities.external_model.cohere_config). + + * - `databricks_model_serving_config` + - Map + - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpoints..config.served_entities.external_model.databricks_model_serving_config). + + * - `google_cloud_vertex_ai_config` + - Map + - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpoints..config.served_entities.external_model.google_cloud_vertex_ai_config). + + * - `name` + - String + - The name of the external model. + + * - `openai_config` + - Map + - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpoints..config.served_entities.external_model.openai_config). + + * - `palm_config` + - Map + - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpoints..config.served_entities.external_model.palm_config). + + * - `provider` + - String + - The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'. + + * - `task` + - String + - The task type of the external model. + + +### model_serving_endpoints..config.served_entities.external_model.ai21labs_config + +**`Type: Map`** + +AI21Labs Config. Only required if the provider is 'ai21labs'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `ai21labs_api_key` + - String + - The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. + + * - `ai21labs_api_key_plaintext` + - String + - An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. + + +### model_serving_endpoints..config.served_entities.external_model.amazon_bedrock_config + +**`Type: Map`** + +Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `aws_access_key_id` + - String + - The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id_plaintext`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + + * - `aws_access_key_id_plaintext` + - String + - An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`. + + * - `aws_region` + - String + - The AWS region to use. Bedrock has to be enabled there. + + * - `aws_secret_access_key` + - String + - The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + + * - `aws_secret_access_key_plaintext` + - String + - An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`. + + * - `bedrock_provider` + - String + - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. + + +### model_serving_endpoints..config.served_entities.external_model.anthropic_config + +**`Type: Map`** + +Anthropic Config. Only required if the provider is 'anthropic'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `anthropic_api_key` + - String + - The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. + + * - `anthropic_api_key_plaintext` + - String + - The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. + + +### model_serving_endpoints..config.served_entities.external_model.cohere_config + +**`Type: Map`** + +Cohere Config. Only required if the provider is 'cohere'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `cohere_api_base` + - String + - This is an optional field to provide a customized base URL for the Cohere API. If left unspecified, the standard Cohere base URL is used. + + * - `cohere_api_key` + - String + - The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. + + * - `cohere_api_key_plaintext` + - String + - The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. + + +### model_serving_endpoints..config.served_entities.external_model.databricks_model_serving_config + +**`Type: Map`** + +Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `databricks_api_token` + - String + - The Databricks secret key reference for a Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model. If you prefer to paste your API key directly, see `databricks_api_token_plaintext`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. + + * - `databricks_api_token_plaintext` + - String + - The Databricks API token that corresponds to a user or service principal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `databricks_api_token`. You must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`. + + * - `databricks_workspace_url` + - String + - The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model. + + +### model_serving_endpoints..config.served_entities.external_model.google_cloud_vertex_ai_config + +**`Type: Map`** + +Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `private_key` + - String + - The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys]. If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext` [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys + + * - `private_key_plaintext` + - String + - The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys]. If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`. [Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys + + * - `project_id` + - String + - This is the Google Cloud project id that the service account is associated with. + + * - `region` + - String + - This is the region for the Google Cloud Vertex AI Service. See [supported regions] for more details. Some models are only available in specific regions. [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations + + +### model_serving_endpoints..config.served_entities.external_model.openai_config + +**`Type: Map`** + +OpenAI Config. Only required if the provider is 'openai'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `microsoft_entra_client_id` + - String + - This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID. + + * - `microsoft_entra_client_secret` + - String + - The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication. If you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`. You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + + * - `microsoft_entra_client_secret_plaintext` + - String + - The client secret used for Microsoft Entra ID authentication provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`. You must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`. + + * - `microsoft_entra_tenant_id` + - String + - This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID. + + * - `openai_api_base` + - String + - This is a field to provide a customized base URl for the OpenAI API. For Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service provided by Azure. For other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used. + + * - `openai_api_key` + - String + - The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + + * - `openai_api_key_plaintext` + - String + - The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`. + + * - `openai_api_type` + - String + - This is an optional field to specify the type of OpenAI API to use. For Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security access validation protocol. For access token validation, use azure. For authentication using Azure Active Directory (Azure AD) use, azuread. + + * - `openai_api_version` + - String + - This is an optional field to specify the OpenAI API version. For Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to utilize, specified by a date. + + * - `openai_deployment_name` + - String + - This field is only required for Azure OpenAI and is the name of the deployment resource for the Azure OpenAI service. + + * - `openai_organization` + - String + - This is an optional field to specify the organization in OpenAI or Azure OpenAI. + + +### model_serving_endpoints..config.served_entities.external_model.palm_config + +**`Type: Map`** + +PaLM Config. Only required if the provider is 'palm'. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `palm_api_key` + - String + - The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. + + * - `palm_api_key_plaintext` + - String + - The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. + + +### model_serving_endpoints..config.served_models + +**`Type: Sequence`** + +(Deprecated, use served_entities instead) The list of served models under the serving endpoint config. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `environment_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{"OPENAI_API_KEY": "{{secrets/my_scope/my_key}}", "DATABRICKS_TOKEN": "{{secrets/my_scope2/my_key2}}"}` + + * - `instance_profile_arn` + - String + - ARN of the instance profile that the served entity uses to access AWS resources. + + * - `max_provisioned_throughput` + - Integer + - The maximum tokens per second that the endpoint can scale up to. + + * - `min_provisioned_throughput` + - Integer + - The minimum tokens per second that the endpoint can scale down to. + + * - `model_name` + - String + - + + * - `model_version` + - String + - + + * - `name` + - String + - The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version. + + * - `scale_to_zero_enabled` + - Boolean + - Whether the compute resources for the served entity should scale down to zero. + + * - `workload_size` + - String + - The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are "Small" (4 - 4 provisioned concurrency), "Medium" (8 - 16 provisioned concurrency), and "Large" (16 - 64 provisioned concurrency). If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. + + * - `workload_type` + - String + - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). + + +### model_serving_endpoints..config.traffic_config + +**`Type: Map`** + +The traffic configuration associated with the serving endpoint config. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `routes` + - Sequence + - The list of routes that define traffic to each served entity. See [_](#model_serving_endpoints..config.traffic_config.routes). + + +### model_serving_endpoints..config.traffic_config.routes + +**`Type: Sequence`** + +The list of routes that define traffic to each served entity. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `served_model_name` + - String + - The name of the served model this route configures traffic for. + + * - `traffic_percentage` + - Integer + - The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive. + + +### model_serving_endpoints..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### model_serving_endpoints..rate_limits + +**`Type: Sequence`** + +Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `calls` + - Integer + - Used to specify how many calls are allowed for a key within the renewal_period. + + * - `key` + - String + - Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified. + + * - `renewal_period` + - String + - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. + + +### model_serving_endpoints..tags + +**`Type: Sequence`** + +Tags to be attached to the serving endpoint and automatically propagated to billing logs. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `key` + - String + - Key field for a serving endpoint tag. + + * - `value` + - String + - Optional value field for a serving endpoint tag. + + +## models + +**`Type: Map`** + +The model resource allows you to define [legacy models](/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use [registered models](#registered-model) instead. + +```yaml +models: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `creation_timestamp` + - Integer + - Timestamp recorded when this `registered_model` was created. + + * - `description` + - String + - Description of this `registered_model`. + + * - `last_updated_timestamp` + - Integer + - Timestamp recorded when metadata for this `registered_model` was last updated. + + * - `latest_versions` + - Sequence + - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#models..latest_versions). + + * - `name` + - String + - Unique name for the model. + + * - `permissions` + - Sequence + - See [_](#models..permissions). + + * - `tags` + - Sequence + - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#models..tags). + + * - `user_id` + - String + - User that created this `registered_model` + + +### models..latest_versions + +**`Type: Sequence`** + +Collection of latest model versions for each stage. +Only contains models with current `READY` status. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `creation_timestamp` + - Integer + - Timestamp recorded when this `model_version` was created. + + * - `current_stage` + - String + - Current stage for this `model_version`. + + * - `description` + - String + - Description of this `model_version`. + + * - `last_updated_timestamp` + - Integer + - Timestamp recorded when metadata for this `model_version` was last updated. + + * - `name` + - String + - Unique name of the model + + * - `run_id` + - String + - MLflow run ID used when creating `model_version`, if `source` was generated by an experiment run stored in MLflow tracking server. + + * - `run_link` + - String + - Run Link: Direct link to the run that generated this version + + * - `source` + - String + - URI indicating the location of the source model artifacts, used when creating `model_version` + + * - `status` + - String + - Current status of `model_version` + + * - `status_message` + - String + - Details on current `status`, if it is pending or failed. + + * - `tags` + - Sequence + - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#models..latest_versions.tags). + + * - `user_id` + - String + - User that created this `model_version`. + + * - `version` + - String + - Model's version number. + + +### models..latest_versions.tags + +**`Type: Sequence`** + +Tags: Additional metadata key-value pairs for this `model_version`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `key` + - String + - The tag key. + + * - `value` + - String + - The tag value. + + +### models..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### models..tags + +**`Type: Sequence`** + +Tags: Additional metadata key-value pairs for this `registered_model`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `key` + - String + - The tag key. + + * - `value` + - String + - The tag value. + + +## pipelines + +**`Type: Map`** + +The pipeline resource allows you to create [pipelines](/api/workspace/pipelines/create). For information about pipelines, see [_](/delta-live-tables/index.md). For a tutorial that uses the template to create a pipeline, see [_](/dev-tools/bundles/pipelines-tutorial.md). + +```yaml +pipelines: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `budget_policy_id` + - String + - Budget policy of this pipeline. + + * - `catalog` + - String + - A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog. + + * - `channel` + - String + - DLT Release Channel that specifies which version to use. + + * - `clusters` + - Sequence + - Cluster settings for this pipeline deployment. See [_](#pipelines..clusters). + + * - `configuration` + - Map + - String-String configuration for this pipeline execution. + + * - `continuous` + - Boolean + - Whether the pipeline is continuous or triggered. This replaces `trigger`. + + * - `deployment` + - Map + - Deployment type of this pipeline. See [_](#pipelines..deployment). + + * - `development` + - Boolean + - Whether the pipeline is in Development mode. Defaults to false. + + * - `edition` + - String + - Pipeline product edition. + + * - `filters` + - Map + - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelines..filters). + + * - `gateway_definition` + - Map + - The definition of a gateway pipeline to support change data capture. See [_](#pipelines..gateway_definition). + + * - `id` + - String + - Unique identifier for this pipeline. + + * - `ingestion_definition` + - Map + - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelines..ingestion_definition). + + * - `libraries` + - Sequence + - Libraries or code needed by this deployment. See [_](#pipelines..libraries). + + * - `name` + - String + - Friendly identifier for this pipeline. + + * - `notifications` + - Sequence + - List of notification settings for this pipeline. See [_](#pipelines..notifications). + + * - `permissions` + - Sequence + - See [_](#pipelines..permissions). + + * - `photon` + - Boolean + - Whether Photon is enabled for this pipeline. + + * - `restart_window` + - Map + - Restart window of this pipeline. See [_](#pipelines..restart_window). + + * - `schema` + - String + - The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode. + + * - `serverless` + - Boolean + - Whether serverless compute is enabled for this pipeline. + + * - `storage` + - String + - DBFS root directory for storing checkpoints and tables. + + * - `target` + - String + - Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`. + + * - `trigger` + - Map + - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelines..trigger). + + +**Example** + +The following example defines a pipeline with the resource key `hello-pipeline`: + +```yaml +resources: + pipelines: + hello-pipeline: + name: hello-pipeline + clusters: + - label: default + num_workers: 1 + development: true + continuous: false + channel: CURRENT + edition: CORE + photon: false + libraries: + - notebook: + path: ./pipeline.py +``` + +### pipelines..clusters + +**`Type: Sequence`** + +Cluster settings for this pipeline deployment. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `apply_policy_default_values` + - Boolean + - Note: This field won't be persisted. Only API users will check this field. + + * - `autoscale` + - Map + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelines..clusters.autoscale). + + * - `aws_attributes` + - Map + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.aws_attributes). + + * - `azure_attributes` + - Map + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.azure_attributes). + + * - `cluster_log_conf` + - Map + - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelines..clusters.cluster_log_conf). + + * - `custom_tags` + - Map + - Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`. Notes: - Currently, Databricks allows at most 45 custom tags - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags + + * - `driver_instance_pool_id` + - String + - The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned. + + * - `driver_node_type_id` + - String + - The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above. + + * - `enable_local_disk_encryption` + - Boolean + - Whether to enable local disk encryption for the cluster. + + * - `gcp_attributes` + - Map + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.gcp_attributes). + + * - `init_scripts` + - Sequence + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#pipelines..clusters.init_scripts). + + * - `instance_pool_id` + - String + - The optional ID of the instance pool to which the cluster belongs. + + * - `label` + - String + - A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`. + + * - `node_type_id` + - String + - This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute intensive workloads. A list of available node types can be retrieved by using the :method:clusters/listNodeTypes API call. + + * - `num_workers` + - Integer + - Number of worker nodes that this cluster should have. A cluster has one Spark Driver and `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. Note: When reading the properties of a cluster, this field reflects the desired number of workers rather than the actual current number of workers. For instance, if a cluster is resized from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the new nodes are provisioned. + + * - `policy_id` + - String + - The ID of the cluster policy used to create the cluster if applicable. + + * - `spark_conf` + - Map + - An object containing a set of optional, user-specified Spark configuration key-value pairs. See :method:clusters/create for more details. + + * - `spark_env_vars` + - Map + - An object containing a set of optional, user-specified environment variable key-value pairs. Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) while launching the driver and workers. In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default databricks managed environmental variables are included as well. Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true"}` + + * - `ssh_public_keys` + - Sequence + - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + + +### pipelines..clusters.autoscale + +**`Type: Map`** + +Parameters needed in order to automatically scale clusters up and down based on load. +Note: autoscaling works best with DB runtime versions 3.0 or later. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `max_workers` + - Integer + - The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`. + + * - `min_workers` + - Integer + - The minimum number of workers the cluster can scale down to when underutilized. It is also the initial number of workers the cluster will have after creation. + + * - `mode` + - String + - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. + + +### pipelines..clusters.aws_attributes + +**`Type: Map`** + +Attributes related to clusters running on Amazon Web Services. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero, this availability type will be used for the entire cluster. + + * - `ebs_volume_count` + - Integer + - The number of volumes launched for each instance. Users can choose up to 10 volumes. This feature is only enabled for supported node types. Legacy node types cannot specify custom EBS volumes. For node types with no instance store, at least one EBS volume needs to be specified; otherwise, cluster creation will fail. These EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc. Instance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc. If EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for scratch storage because heterogenously sized scratch devices can lead to inefficient disk utilization. If no EBS volumes are attached, Databricks will configure Spark to use instance store volumes. Please note that if EBS volumes are specified, then the Spark configuration `spark.local.dir` will be overridden. + + * - `ebs_volume_iops` + - Integer + - If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_size` + - Integer + - The size of each EBS volume (in GiB) launched for each instance. For general purpose SSD, this value must be within the range 100 - 4096. For throughput optimized HDD, this value must be within the range 500 - 4096. + + * - `ebs_volume_throughput` + - Integer + - If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used. + + * - `ebs_volume_type` + - String + - The type of EBS volumes that will be launched with this cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. If this value is greater than 0, the cluster driver node in particular will be placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `instance_profile_arn` + - String + - Nodes for this cluster will only be placed on AWS instances with this instance profile. If ommitted, nodes will be placed on instances without an IAM instance profile. The instance profile must have previously been added to the Databricks environment by an account administrator. This feature may only be available to certain customer plans. If this field is ommitted, we will pull in the default from the conf if it exists. + + * - `spot_bid_price_percent` + - Integer + - The bid price for AWS spot instances, as a percentage of the corresponding instance type's on-demand price. For example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot instance, then the bid price is half of the price of on-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice the price of on-demand `r3.xlarge` instances. If not specified, the default value is 100. When spot instances are requested for this cluster, only spot instances whose bid price percentage matches this field will be considered. Note that, for safety, we enforce this field to be no more than 10000. The default value and documentation here should be kept consistent with CommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent. + + * - `zone_id` + - String + - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. + + +### pipelines..clusters.azure_attributes + +**`Type: Map`** + +Attributes related to clusters running on Microsoft Azure. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - Availability type used for all subsequent nodes past the `first_on_demand` ones. Note: If `first_on_demand` is zero (which only happens on pool clusters), this availability type will be used for the entire cluster. + + * - `first_on_demand` + - Integer + - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. + + * - `log_analytics_info` + - Map + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelines..clusters.azure_attributes.log_analytics_info). + + * - `spot_bid_max_price` + - Any + - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. + + +### pipelines..clusters.azure_attributes.log_analytics_info + +**`Type: Map`** + +Defines values necessary to configure and run Azure Log Analytics agent + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `log_analytics_primary_key` + - String + - + + * - `log_analytics_workspace_id` + - String + - + + +### pipelines..clusters.cluster_log_conf + +**`Type: Map`** + +The configuration for delivering spark logs to a long-term storage destination. +Only dbfs destinations are supported. Only one destination can be specified +for one cluster. If the conf is given, the logs will be delivered to the destination every +`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while +the destination of executor logs is `$destination/$clusterId/executor`. + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines..clusters.cluster_log_conf.dbfs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines..clusters.cluster_log_conf.s3). + + +### pipelines..clusters.cluster_log_conf.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### pipelines..clusters.cluster_log_conf.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### pipelines..clusters.gcp_attributes + +**`Type: Map`** + +Attributes related to clusters running on Google Cloud Platform. +If not specified at cluster creation, a set of default values will be used. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `availability` + - String + - This field determines whether the instance pool will contain preemptible VMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable. + + * - `boot_disk_size` + - Integer + - boot disk size in GB + + * - `google_service_account` + - String + - If provided, the cluster will impersonate the google service account when accessing gcloud services (like GCS). The google service account must have previously been added to the Databricks environment by an account administrator. + + * - `local_ssd_count` + - Integer + - If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type. + + * - `use_preemptible_executors` + - Boolean + - This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default). Note: Soon to be deprecated, use the availability field instead. + + * - `zone_id` + - String + - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. + + +### pipelines..clusters.init_scripts + +**`Type: Sequence`** + +The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `abfss` + - Map + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#pipelines..clusters.init_scripts.abfss). + + * - `dbfs` + - Map + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines..clusters.init_scripts.dbfs). + + * - `file` + - Map + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelines..clusters.init_scripts.file). + + * - `gcs` + - Map + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelines..clusters.init_scripts.gcs). + + * - `s3` + - Map + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines..clusters.init_scripts.s3). + + * - `volumes` + - Map + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelines..clusters.init_scripts.volumes). + + * - `workspace` + - Map + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelines..clusters.init_scripts.workspace). + + +### pipelines..clusters.init_scripts.abfss + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } } + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. + + +### pipelines..clusters.init_scripts.dbfs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - dbfs destination, e.g. `dbfs:/my/path` + + +### pipelines..clusters.init_scripts.file + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "file" : { "destination" : "file:/my/local/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - local file destination, e.g. `file:/my/local/file.sh` + + +### pipelines..clusters.init_scripts.gcs + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "gcs": { "destination": "gs://my-bucket/file.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` + + +### pipelines..clusters.init_scripts.s3 + +**`Type: Map`** + +destination and either the region or endpoint need to be provided. e.g. +`{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` +Cluster iam role is used to access s3, please make sure the cluster iam role in +`instance_profile_arn` has permission to write data to the s3 destination. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `canned_acl` + - String + - (Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`. If `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on the destination bucket and prefix. The full list of possible canned acl can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl. Please also note that by default only the object owner gets full controls. If you are using cross account role for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to read the logs. + + * - `destination` + - String + - S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using cluster iam role, please make sure you set cluster iam role and the role has write access to the destination. Please also note that you cannot use AWS keys to deliver logs. + + * - `enable_encryption` + - Boolean + - (Optional) Flag to enable server side encryption, `false` by default. + + * - `encryption_type` + - String + - (Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when encryption is enabled and the default type is `sse-s3`. + + * - `endpoint` + - String + - S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + * - `kms_key` + - String + - (Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`. + + * - `region` + - String + - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. + + +### pipelines..clusters.init_scripts.volumes + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` + + +### pipelines..clusters.init_scripts.workspace + +**`Type: Map`** + +destination needs to be provided. e.g. +`{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }` + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination` + - String + - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` + + +### pipelines..deployment + +**`Type: Map`** + +Deployment type of this pipeline. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `kind` + - String + - The deployment method that manages the pipeline. + + * - `metadata_file_path` + - String + - The path to the file containing metadata about the deployment. + + +### pipelines..filters + +**`Type: Map`** + +Filters on which Pipeline packages to include in the deployed graph. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `exclude` + - Sequence + - Paths to exclude. + + * - `include` + - Sequence + - Paths to include. + + +### pipelines..gateway_definition + +**`Type: Map`** + +The definition of a gateway pipeline to support change data capture. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `connection_id` + - String + - [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + + * - `connection_name` + - String + - Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source. + + * - `gateway_storage_catalog` + - String + - Required, Immutable. The name of the catalog for the gateway pipeline's storage location. + + * - `gateway_storage_name` + - String + - Optional. The Unity Catalog-compatible name for the gateway storage location. This is the destination to use for the data that is extracted by the gateway. Delta Live Tables system will automatically create the storage location under the catalog and schema. + + * - `gateway_storage_schema` + - String + - Required, Immutable. The name of the schema for the gateway pipelines's storage location. + + +### pipelines..ingestion_definition + +**`Type: Map`** + +The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `connection_name` + - String + - Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on. + + * - `ingestion_gateway_id` + - String + - Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server. + + * - `objects` + - Sequence + - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelines..ingestion_definition.objects). + + * - `table_configuration` + - Map + - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelines..ingestion_definition.table_configuration). + + +### pipelines..ingestion_definition.objects + +**`Type: Sequence`** + +Required. Settings specifying tables to replicate and the destination for the replicated tables. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `report` + - Map + - Select a specific source report. See [_](#pipelines..ingestion_definition.objects.report). + + * - `schema` + - Map + - Select all tables from a specific source schema. See [_](#pipelines..ingestion_definition.objects.schema). + + * - `table` + - Map + - Select a specific source table. See [_](#pipelines..ingestion_definition.objects.table). + + +### pipelines..ingestion_definition.objects.report + +**`Type: Map`** + +Select a specific source report. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination_catalog` + - String + - Required. Destination catalog to store table. + + * - `destination_schema` + - String + - Required. Destination schema to store table. + + * - `destination_table` + - String + - Required. Destination table name. The pipeline fails if a table with that name already exists. + + * - `source_url` + - String + - Required. Report URL in the source system. + + * - `table_configuration` + - Map + - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines..ingestion_definition.objects.report.table_configuration). + + +### pipelines..ingestion_definition.objects.report.table_configuration + +**`Type: Map`** + +Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `primary_keys` + - Sequence + - The primary key of the table used to apply changes. + + * - `salesforce_include_formula_fields` + - Boolean + - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector + + * - `scd_type` + - String + - The SCD type to use to ingest the table. + + * - `sequence_by` + - Sequence + - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + + +### pipelines..ingestion_definition.objects.schema + +**`Type: Map`** + +Select all tables from a specific source schema. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination_catalog` + - String + - Required. Destination catalog to store tables. + + * - `destination_schema` + - String + - Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists. + + * - `source_catalog` + - String + - The source catalog name. Might be optional depending on the type of source. + + * - `source_schema` + - String + - Required. Schema name in the source database. + + * - `table_configuration` + - Map + - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines..ingestion_definition.objects.schema.table_configuration). + + +### pipelines..ingestion_definition.objects.schema.table_configuration + +**`Type: Map`** + +Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `primary_keys` + - Sequence + - The primary key of the table used to apply changes. + + * - `salesforce_include_formula_fields` + - Boolean + - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector + + * - `scd_type` + - String + - The SCD type to use to ingest the table. + + * - `sequence_by` + - Sequence + - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + + +### pipelines..ingestion_definition.objects.table + +**`Type: Map`** + +Select a specific source table. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `destination_catalog` + - String + - Required. Destination catalog to store table. + + * - `destination_schema` + - String + - Required. Destination schema to store table. + + * - `destination_table` + - String + - Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used. + + * - `source_catalog` + - String + - Source catalog name. Might be optional depending on the type of source. + + * - `source_schema` + - String + - Schema name in the source database. Might be optional depending on the type of source. + + * - `source_table` + - String + - Required. Table name in the source database. + + * - `table_configuration` + - Map + - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelines..ingestion_definition.objects.table.table_configuration). + + +### pipelines..ingestion_definition.objects.table.table_configuration + +**`Type: Map`** + +Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `primary_keys` + - Sequence + - The primary key of the table used to apply changes. + + * - `salesforce_include_formula_fields` + - Boolean + - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector + + * - `scd_type` + - String + - The SCD type to use to ingest the table. + + * - `sequence_by` + - Sequence + - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + + +### pipelines..ingestion_definition.table_configuration + +**`Type: Map`** + +Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `primary_keys` + - Sequence + - The primary key of the table used to apply changes. + + * - `salesforce_include_formula_fields` + - Boolean + - If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector + + * - `scd_type` + - String + - The SCD type to use to ingest the table. + + * - `sequence_by` + - Sequence + - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. + + +### pipelines..libraries + +**`Type: Sequence`** + +Libraries or code needed by this deployment. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `file` + - Map + - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelines..libraries.file). + + * - `jar` + - String + - URI of the jar to be installed. Currently only DBFS is supported. + + * - `maven` + - Map + - Specification of a maven library to be installed. . See [_](#pipelines..libraries.maven). + + * - `notebook` + - Map + - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelines..libraries.notebook). + + * - `whl` + - String + - URI of the whl to be installed. + + +### pipelines..libraries.file + +**`Type: Map`** + +The path to a file that defines a pipeline and is stored in the Databricks Repos. + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `path` + - String + - The absolute path of the file. + + +### pipelines..libraries.maven + +**`Type: Map`** + +Specification of a maven library to be installed. + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `coordinates` + - String + - Gradle-style maven coordinates. For example: "org.jsoup:jsoup:1.7.2". + + * - `exclusions` + - Sequence + - List of dependences to exclude. For example: `["slf4j:slf4j", "*:hadoop-client"]`. Maven dependency exclusions: https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html. + + * - `repo` + - String + - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched. + + +### pipelines..libraries.notebook + +**`Type: Map`** + +The path to a notebook that defines a pipeline and is stored in the Databricks workspace. + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `path` + - String + - The absolute path of the notebook. + + +### pipelines..notifications + +**`Type: Sequence`** + +List of notification settings for this pipeline. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `alerts` + - Sequence + - A list of alerts that trigger the sending of notifications to the configured destinations. The supported alerts are: * `on-update-success`: A pipeline update completes successfully. * `on-update-failure`: Each time a pipeline update fails. * `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error. * `on-flow-failure`: A single data flow fails. + + * - `email_recipients` + - Sequence + - A list of email addresses notified when a configured alert is triggered. + + +### pipelines..permissions + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `group_name` + - String + - The name of the group that has the permission set in level. + + * - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + + * - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + + * - `user_name` + - String + - The name of the user that has the permission set in level. + + +### pipelines..restart_window + +**`Type: Map`** + +Restart window of this pipeline. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `days_of_week` + - Sequence + - Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). If not specified all days of the week will be used. + + * - `start_hour` + - Integer + - An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day. Continuous pipeline restart is triggered only within a five-hour window starting at this hour. + + * - `time_zone_id` + - String + - Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. + + +### pipelines..restart_window.days_of_week + +**`Type: Sequence`** + +Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). +If not specified all days of the week will be used. + + +### pipelines..trigger + +**`Type: Map`** + +Which pipeline trigger to use. Deprecated: Use `continuous` instead. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `cron` + - Map + - See [_](#pipelines..trigger.cron). + + * - `manual` + - Map + - + + +### pipelines..trigger.cron + +**`Type: Map`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `quartz_cron_schedule` + - String + - + + * - `timezone_id` + - String + - + + +### pipelines..trigger.manual + +**`Type: Map`** + + + + +## quality_monitors + +**`Type: Map`** + +The quality_monitor resource allows you to define a [table monitor](/api/workspace/qualitymonitors/create). For information about monitors, see [_](/machine-learning/model-serving/monitor-diagnose-endpoints.md). + +```yaml +quality_monitors: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `assets_dir` + - String + - The directory to store monitoring assets (e.g. dashboard, metric tables). + + * - `baseline_table_name` + - String + - Name of the baseline table from which drift metrics are computed from. Columns in the monitored table should also be present in the baseline table. + + * - `custom_metrics` + - Sequence + - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitors..custom_metrics). + + * - `data_classification_config` + - Map + - The data classification config for the monitor. See [_](#quality_monitors..data_classification_config). + + * - `inference_log` + - Map + - Configuration for monitoring inference logs. See [_](#quality_monitors..inference_log). + + * - `notifications` + - Map + - The notification settings for the monitor. See [_](#quality_monitors..notifications). + + * - `output_schema_name` + - String + - Schema where output metric tables are created. + + * - `schedule` + - Map + - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitors..schedule). + + * - `skip_builtin_dashboard` + - Boolean + - Whether to skip creating a default dashboard summarizing data quality metrics. + + * - `slicing_exprs` + - Sequence + - List of column expressions to slice data with for targeted analysis. The data is grouped by each expression independently, resulting in a separate slice for each predicate and its complements. For high-cardinality columns, only the top 100 unique values by frequency will generate slices. + + * - `snapshot` + - Map + - Configuration for monitoring snapshot tables. + + * - `table_name` + - String + - + + * - `time_series` + - Map + - Configuration for monitoring time series tables. See [_](#quality_monitors..time_series). + + * - `warehouse_id` + - String + - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used. + + +**Example** + +The following example defines a quality monitor: + +```yaml +resources: + quality_monitors: + my_quality_monitor: + table_name: dev.mlops_schema.predictions + output_schema_name: ${bundle.target}.mlops_schema + assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + label_col: price + problem_type: PROBLEM_TYPE_REGRESSION + timestamp_col: timestamp + schedule: + quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am + timezone_id: UTC +``` + +### quality_monitors..custom_metrics + +**`Type: Sequence`** + +Custom metrics to compute on the monitored table. These can be aggregate metrics, derived +metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time +windows). + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `definition` + - String + - Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition). + + * - `input_columns` + - Sequence + - A list of column names in the input table the metric should be computed for. Can use ``":table"`` to indicate that the metric needs information from multiple columns. + + * - `name` + - String + - Name of the metric in the output tables. + + * - `output_data_type` + - String + - The output type of the custom metric. + + * - `type` + - String + - Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics + + +### quality_monitors..data_classification_config + +**`Type: Map`** + +The data classification config for the monitor. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `enabled` + - Boolean + - Whether data classification is enabled. + + +### quality_monitors..inference_log + +**`Type: Map`** + +Configuration for monitoring inference logs. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `granularities` + - Sequence + - Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``" week(s)"``, ``"1 month"``, ``"1 year"``}. + + * - `label_col` + - String + - Optional column that contains the ground truth for the prediction. + + * - `model_id_col` + - String + - Column that contains the id of the model generating the predictions. Metrics will be computed per model id by default, and also across all model ids. + + * - `prediction_col` + - String + - Column that contains the output/prediction from the model. + + * - `prediction_proba_col` + - String + - Optional column that contains the prediction probabilities for each class in a classification problem type. The values in this column should be a map, mapping each class label to the prediction probability for a given sample. The map should be of PySpark MapType(). + + * - `problem_type` + - String + - Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed. + + * - `timestamp_col` + - String + - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). + + +### quality_monitors..notifications + +**`Type: Map`** + +The notification settings for the monitor. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `on_failure` + - Map + - Who to send notifications to on monitor failure. See [_](#quality_monitors..notifications.on_failure). + + * - `on_new_classification_tag_detected` + - Map + - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitors..notifications.on_new_classification_tag_detected). + + +### quality_monitors..notifications.on_failure + +**`Type: Map`** + +Who to send notifications to on monitor failure. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `email_addresses` + - Sequence + - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. + + +### quality_monitors..notifications.on_new_classification_tag_detected + +**`Type: Map`** + +Who to send notifications to when new data classification tags are detected. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `email_addresses` + - Sequence + - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. + + +### quality_monitors..schedule + +**`Type: Map`** + +The schedule for automatically updating and refreshing metric tables. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `pause_status` + - String + - Read only field that indicates whether a schedule is paused or not. + + * - `quartz_cron_expression` + - String + - The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). + + * - `timezone_id` + - String + - The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression. + + +### quality_monitors..snapshot + +**`Type: Map`** + +Configuration for monitoring snapshot tables. + + +### quality_monitors..time_series + +**`Type: Map`** + +Configuration for monitoring time series tables. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `granularities` + - Sequence + - Granularities for aggregating data into time windows based on their timestamp. Currently the following static granularities are supported: {``"5 minutes"``, ``"30 minutes"``, ``"1 hour"``, ``"1 day"``, ``" week(s)"``, ``"1 month"``, ``"1 year"``}. + + * - `timestamp_col` + - String + - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). + + +## registered_models + +**`Type: Map`** + +The registered model resource allows you to define models in . For information about [registered models](/api/workspace/registeredmodels/create), see [_](/machine-learning/manage-model-lifecycle/index.md). + +```yaml +registered_models: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog_name` + - String + - The name of the catalog where the schema and the registered model reside + + * - `comment` + - String + - The comment attached to the registered model + + * - `grants` + - Sequence + - See [_](#registered_models..grants). + + * - `name` + - String + - The name of the registered model + + * - `schema_name` + - String + - The name of the schema where the registered model resides + + * - `storage_location` + - String + - The storage location on the cloud under which model version data files are stored + + +**Example** + +The following example defines a registered model in : + +```yaml +resources: + registered_models: + model: + name: my_model + catalog_name: ${bundle.target} + schema_name: mlops_schema + comment: Registered model in Unity Catalog for ${bundle.target} deployment target + grants: + - privileges: + - EXECUTE + principal: account users +``` + +### registered_models..grants + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `principal` + - String + - The name of the principal that will be granted privileges + + * - `privileges` + - Sequence + - The privileges to grant to the specified entity + + +## schemas + +**`Type: Map`** + +The schema resource type allows you to define [schemas](/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations: + +- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema. +- Only fields supported by the corresponding [Schemas object create API](/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](/api/workspace/schemas/update). + +```yaml +schemas: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog_name` + - String + - Name of parent catalog. + + * - `comment` + - String + - User-provided free-form text description. + + * - `grants` + - Sequence + - See [_](#schemas..grants). + + * - `name` + - String + - Name of schema, relative to parent catalog. + + * - `properties` + - Map + - + + * - `storage_root` + - String + - Storage root URL for managed tables within schema. + + +**Example** + +The following example defines a pipeline with the resource key `my_pipeline` that creates a schema with the key `my_schema` as the target: + +```yaml +resources: + pipelines: + my_pipeline: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + target: ${resources.schemas.my_schema.id} + + schemas: + my_schema: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created by DABs. +``` + +A top-level grants mapping is not supported by , so if you want to set grants for a schema, define the grants for the schema within the `schemas` mapping. For more information about grants, see [_](/data-governance/unity-catalog/manage-privileges/index.md#grant). + +The following example defines a schema with grants: + +```yaml +resources: + schemas: + my_schema: + name: test-schema + grants: + - principal: users + privileges: + - CAN_MANAGE + - principal: my_team + privileges: + - CAN_READ + catalog_name: main + ``` + +### schemas..grants + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `principal` + - String + - The name of the principal that will be granted privileges + + * - `privileges` + - Sequence + - The privileges to grant to the specified entity + + +## volumes + +**`Type: Map`** + +The volume resource type allows you to define and create [volumes](/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that: + +- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. + +- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). + +```yaml +volumes: + : + : +``` + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `catalog_name` + - String + - The name of the catalog where the schema and the volume are + + * - `comment` + - String + - The comment attached to the volume + + * - `grants` + - Sequence + - See [_](#volumes..grants). + + * - `name` + - String + - The name of the volume + + * - `schema_name` + - String + - The name of the schema where the volume is + + * - `storage_location` + - String + - The storage location on the cloud + + * - `volume_type` + - String + - + + +**Example** + +The following example creates a volume with the key `my_volume`: + +```yaml +resources: + volumes: + my_volume: + catalog_name: main + name: my_volume + schema_name: my_schema +``` + +For an example bundle that runs a job that writes to a file in volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume). + +### volumes..grants + +**`Type: Sequence`** + + + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `principal` + - String + - The name of the principal that will be granted privileges + + * - `privileges` + - Sequence + - The privileges to grant to the specified entity + \ No newline at end of file diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go new file mode 100644 index 000000000..ca45e6ab2 --- /dev/null +++ b/bundle/docsgen/refs.go @@ -0,0 +1,97 @@ +package main + +import ( + "log" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +func isReferenceType(v *jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFields map[string]bool) bool { + if v.Type != "object" && v.Type != "array" { + return false + } + if len(v.Properties) > 0 { + return true + } + if v.Items != nil { + items := resolveRefs(v.Items, refs) + if items != nil && items.Type == "object" { + return true + } + } + props := resolveAdditionalProperties(v) + if !isInOwnFields(props, ownFields) { + return false + } + if props != nil { + propsResolved := resolveRefs(props, refs) + return propsResolved.Type == "object" + } + + return false +} + +func isInOwnFields(node *jsonschema.Schema, ownFields map[string]bool) bool { + if node != nil && node.Reference != nil { + return ownFields[getRefType(node)] + } + return true +} + +func resolveAdditionalProperties(v *jsonschema.Schema) *jsonschema.Schema { + if v.AdditionalProperties == nil { + return nil + } + additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema) + if !ok { + return nil + } + return additionalProps +} + +func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *jsonschema.Schema { + if s == nil { + return nil + } + + node := s + description := s.Description + markdownDescription := s.MarkdownDescription + examples := s.Examples + + for node.Reference != nil { + ref := getRefType(node) + newNode, ok := schemas[ref] + if !ok { + log.Printf("schema %s not found", ref) + break + } + + if description == "" { + description = newNode.Description + } + if markdownDescription == "" { + markdownDescription = newNode.MarkdownDescription + } + if len(examples) == 0 { + examples = newNode.Examples + } + + node = newNode + } + + newNode := *node + newNode.Description = description + newNode.MarkdownDescription = markdownDescription + newNode.Examples = examples + + return &newNode +} + +func getRefType(node *jsonschema.Schema) string { + if node.Reference == nil { + return "" + } + return strings.TrimPrefix(*node.Reference, "#/$defs/") +} diff --git a/bundle/docsgen/renderer.go b/bundle/docsgen/renderer.go new file mode 100644 index 000000000..5f6c77258 --- /dev/null +++ b/bundle/docsgen/renderer.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "runtime" + "strings" +) + +type markdownRenderer struct { + nodes []string +} + +func newMardownRenderer() *markdownRenderer { + return &markdownRenderer{} +} + +func (m *markdownRenderer) add(s string) *markdownRenderer { + m.nodes = append(m.nodes, s) + return m +} + +func (m *markdownRenderer) PlainText(s string) *markdownRenderer { + return m.add(s) +} + +func (m *markdownRenderer) LF() *markdownRenderer { + return m.add(" ") +} + +func (m *markdownRenderer) H2(s string) *markdownRenderer { + return m.add("## " + s) +} + +func (m *markdownRenderer) H3(s string) *markdownRenderer { + return m.add("### " + s) +} + +func (m *markdownRenderer) CodeBlocks(lang, s string) *markdownRenderer { + return m.add(fmt.Sprintf("```%s%s%s%s```", lang, lineFeed(), s, lineFeed())) +} + +func (m *markdownRenderer) String() string { + return strings.Join(m.nodes, lineFeed()) +} + +func lineFeed() string { + if runtime.GOOS == "windows" { + return "\r\n" + } + return "\n" +} diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md new file mode 100644 index 000000000..a17d53315 --- /dev/null +++ b/bundle/docsgen/templates/reference.md @@ -0,0 +1,10 @@ + +--- +description: Configuration reference for databricks.yml +--- + +# Configuration reference + +This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). + +For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples). diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md new file mode 100644 index 000000000..fccfac47d --- /dev/null +++ b/bundle/docsgen/templates/resources.md @@ -0,0 +1,70 @@ + +--- +description: Learn about resources supported by Databricks Asset Bundles and how to configure them. +--- + +# resources + + allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). + +This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md). + +## Supported resources + +The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle. + +Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table. + +.. tip:: The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files. + + +.. list-table:: + :header-rows: 1 + + * - Resource + - Create support + - Corresponding REST API object + + * - [cluster](#cluster) + - ✓ + - [Cluster object](/api/workspace/clusters/create) + + * - [dashboard](#dashboard) + - + - [Dashboard object](/api/workspace/lakeview/create) + + * - [experiment](#experiment) + - ✓ + - [Experiment object](/api/workspace/experiments/createexperiment) + + * - [job](#job) + - ✓ + - [Job object](/api/workspace/jobs/create) + + * - [model (legacy)](#model-legacy) + - ✓ + - [Model (legacy) object](/api/workspace/modelregistry/createmodel) + + * - [model_serving_endpoint](#model-serving-endpoint) + - ✓ + - [Model serving endpoint object](/api/workspace/servingendpoints/create) + + * - [pipeline](#pipeline) + - ✓ + - [Pipeline object](/api/workspace/pipelines/create) + + * - [quality_monitor](#quality-monitor) + - ✓ + - [Quality monitor object](/api/workspace/qualitymonitors/create) + + * - [registered_model](#registered-model) () + - ✓ + - [Registered model object](/api/workspace/registeredmodels/create) + + * - [schema](#schema) () + - ✓ + - [Schema object](/api/workspace/schemas/create) + + * - [volume](#volume) () + - ✓ + - [Volume object](/api/workspace/volumes/create) diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go new file mode 100644 index 000000000..26c1a0b06 --- /dev/null +++ b/bundle/internal/annotation/descriptor.go @@ -0,0 +1,12 @@ +package annotation + +type Descriptor struct { + Description string `json:"description,omitempty"` + MarkdownDescription string `json:"markdown_description,omitempty"` + Title string `json:"title,omitempty"` + Default any `json:"default,omitempty"` + Enum []any `json:"enum,omitempty"` + MarkdownExamples string `json:"markdown_examples,omitempty"` +} + +const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/annotation/file.go b/bundle/internal/annotation/file.go new file mode 100644 index 000000000..0317f441a --- /dev/null +++ b/bundle/internal/annotation/file.go @@ -0,0 +1,44 @@ +package annotation + +import ( + "bytes" + "os" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/cli/libs/dyn/yamlloader" +) + +// Parsed file with annotations, expected format: +// github.com/databricks/cli/bundle/config.Bundle: +// +// cluster_id: +// description: "Description" +type File map[string]map[string]Descriptor + +func LoadAndMerge(sources []string) (File, error) { + prev := dyn.NilValue + for _, path := range sources { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) + if err != nil { + return nil, err + } + prev, err = merge.Merge(prev, generated) + if err != nil { + return nil, err + } + } + + var data File + + err := convert.ToTyped(&data, prev) + if err != nil { + return nil, err + } + return data, nil +} diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 91aaa4555..ee3c25ca1 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -11,6 +11,7 @@ import ( yaml3 "gopkg.in/yaml.v3" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/merge" @@ -19,60 +20,23 @@ import ( "github.com/databricks/cli/libs/jsonschema" ) -type annotation struct { - Description string `json:"description,omitempty"` - MarkdownDescription string `json:"markdown_description,omitempty"` - Title string `json:"title,omitempty"` - Default any `json:"default,omitempty"` - Enum []any `json:"enum,omitempty"` -} - type annotationHandler struct { // Annotations read from all annotation files including all overrides - parsedAnnotations annotationFile + parsedAnnotations annotation.File // Missing annotations for fields that are found in config that need to be added to the annotation file - missingAnnotations annotationFile + missingAnnotations annotation.File } -/** - * Parsed file with annotations, expected format: - * github.com/databricks/cli/bundle/config.Bundle: - * cluster_id: - * description: "Description" - */ -type annotationFile map[string]map[string]annotation - -const Placeholder = "PLACEHOLDER" - // Adds annotations to the JSON schema reading from the annotation files. // More details https://json-schema.org/understanding-json-schema/reference/annotations func newAnnotationHandler(sources []string) (*annotationHandler, error) { - prev := dyn.NilValue - for _, path := range sources { - b, err := os.ReadFile(path) - if err != nil { - return nil, err - } - generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b)) - if err != nil { - return nil, err - } - prev, err = merge.Merge(prev, generated) - if err != nil { - return nil, err - } - } - - var data annotationFile - - err := convert.ToTyped(&data, prev) + data, err := annotation.LoadAndMerge(sources) if err != nil { return nil, err } - d := &annotationHandler{} d.parsedAnnotations = data - d.missingAnnotations = annotationFile{} + d.missingAnnotations = annotation.File{} return d, nil } @@ -85,7 +49,7 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema annotations := d.parsedAnnotations[refPath] if annotations == nil { - annotations = map[string]annotation{} + annotations = map[string]annotation.Descriptor{} } rootTypeAnnotation, ok := annotations[RootTypeKey] @@ -96,11 +60,11 @@ func (d *annotationHandler) addAnnotations(typ reflect.Type, s jsonschema.Schema for k, v := range s.Properties { item := annotations[k] if item.Description == "" { - item.Description = Placeholder + item.Description = annotation.Placeholder emptyAnnotations := d.missingAnnotations[refPath] if emptyAnnotations == nil { - emptyAnnotations = map[string]annotation{} + emptyAnnotations = map[string]annotation.Descriptor{} d.missingAnnotations[refPath] = emptyAnnotations } emptyAnnotations[k] = item @@ -124,7 +88,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error for k := range d.missingAnnotations { if !isCliPath(k) { delete(d.missingAnnotations, k) - fmt.Printf("Missing annotations for `%s` that are not in CLI package, try to fetch latest OpenAPI spec and regenerate annotations", k) + fmt.Printf("Missing annotations for `%s` that are not in CLI package, try to fetch latest OpenAPI spec and regenerate annotations\n", k) } } @@ -138,7 +102,7 @@ func (d *annotationHandler) syncWithMissingAnnotations(outputPath string) error return err } - var outputTyped annotationFile + var outputTyped annotation.File err = convert.ToTyped(&outputTyped, output) if err != nil { return err @@ -155,8 +119,8 @@ func getPath(typ reflect.Type) string { return typ.PkgPath() + "." + typ.Name() } -func assignAnnotation(s *jsonschema.Schema, a annotation) { - if a.Description != Placeholder { +func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { + if a.Description != annotation.Placeholder { s.Description = a.Description } @@ -168,7 +132,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation) { s.Enum = a.Enum } -func saveYamlWithStyle(outputPath string, annotations annotationFile) error { +func saveYamlWithStyle(outputPath string, annotations annotation.File) error { annotationOrder := yamlsaver.NewOrder([]string{"description", "markdown_description", "title", "default", "enum"}) style := map[string]yaml3.Style{} @@ -220,15 +184,17 @@ func convertLinksToAbsoluteUrl(s string) string { referencePage := "/dev-tools/bundles/reference.html" // Regular expression to match Markdown-style links like [_](link) - re := regexp.MustCompile(`\[_\]\(([^)]+)\)`) + re := regexp.MustCompile(`\[(.*?)\]\((.*?)\)`) result := re.ReplaceAllStringFunc(s, func(match string) string { matches := re.FindStringSubmatch(match) if len(matches) < 2 { return match } - link := matches[1] - var text, absoluteURL string + originalText := matches[1] + link := matches[2] + + var text, absoluteURL string if strings.HasPrefix(link, "#") { text = strings.TrimPrefix(link, "#") absoluteURL = fmt.Sprintf("%s%s%s", base, referencePage, link) @@ -246,6 +212,10 @@ func convertLinksToAbsoluteUrl(s string) string { return match } + if originalText != "_" { + text = originalText + } + return fmt.Sprintf("[%s](%s)", text, absoluteURL) }) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e18de7896..c10f43b04 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1,31 +1,25 @@ github.com/databricks/cli/bundle/config.Artifact: "build": "description": |- - An optional set of non-default build commands that you want to run locally before deployment. - - For Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment. - - To specify multiple build commands, separate each command with double-ampersand (&&) characters. + An optional set of non-default build commands to run locally before deployment. "executable": "description": |- - The executable type. + The executable type. Valid values are `bash`, `sh`, and `cmd`. "files": "description": |- The source files for the artifact. - "markdown_description": |- - The source files for the artifact, defined as an [_](#artifact_file). "path": "description": |- The location where the built artifact will be saved. "type": "description": |- - The type of the artifact. + Required. The type of the artifact. "markdown_description": |- - The type of the artifact. Valid values are `wheel` or `jar` + Required. The type of the artifact. Valid values are `whl`. github.com/databricks/cli/bundle/config.ArtifactFile: "source": "description": |- - The path of the files used to build the artifact. + Required. The path of the files used to build the artifact. github.com/databricks/cli/bundle/config.Bundle: "cluster_id": "description": |- @@ -44,12 +38,12 @@ github.com/databricks/cli/bundle/config.Bundle: "description": |- The definition of the bundle deployment "markdown_description": |- - The definition of the bundle deployment. For supported attributes, see [_](#deployment) and [_](/dev-tools/bundles/deployment-modes.md). + The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). "git": "description": |- The Git version control details that are associated with your bundle. "markdown_description": |- - The Git version control details that are associated with your bundle. For supported attributes, see [_](#git) and [_](/dev-tools/bundles/settings.md#git). + The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). "name": "description": |- The name of the bundle. @@ -63,8 +57,6 @@ github.com/databricks/cli/bundle/config.Deployment: "lock": "description": |- The deployment lock attributes. - "markdown_description": |- - The deployment lock attributes. See [_](#lock). github.com/databricks/cli/bundle/config.Experimental: "pydabs": "description": |- @@ -74,13 +66,13 @@ github.com/databricks/cli/bundle/config.Experimental: Configures loading of Python code defined with 'databricks-bundles' package. "python_wheel_wrapper": "description": |- - Whether to use a Python wheel wrapper + Whether to use a Python wheel wrapper. "scripts": "description": |- - The commands to run + The commands to run. "use_legacy_run_as": "description": |- - Whether to use the legacy run_as behavior + Whether to use the legacy run_as behavior. github.com/databricks/cli/bundle/config.Git: "branch": "description": |- @@ -152,66 +144,80 @@ github.com/databricks/cli/bundle/config.Resources: PLACEHOLDER "clusters": "description": |- - The cluster definitions for the bundle. + The cluster definitions for the bundle, where each key is the name of a cluster. "markdown_description": |- - The cluster definitions for the bundle. See [_](/dev-tools/bundles/resources.md#cluster) + The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters) "dashboards": "description": |- - The dashboard definitions for the bundle. + The dashboard definitions for the bundle, where each key is the name of the dashboard. "markdown_description": |- - The dashboard definitions for the bundle. See [_](/dev-tools/bundles/resources.md#dashboard) + The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards) "experiments": "description": |- - The experiment definitions for the bundle. + The experiment definitions for the bundle, where each key is the name of the experiment. "markdown_description": |- - The experiment definitions for the bundle. See [_](/dev-tools/bundles/resources.md#experiment) + The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments) "jobs": "description": |- - The job definitions for the bundle. + The job definitions for the bundle, where each key is the name of the job. "markdown_description": |- - The job definitions for the bundle. See [_](/dev-tools/bundles/resources.md#job) + The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs) "model_serving_endpoints": "description": |- - The model serving endpoint definitions for the bundle. + The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. "markdown_description": |- - The model serving endpoint definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model_serving_endpoint) + The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints) "models": "description": |- - The model definitions for the bundle. + The model definitions for the bundle, where each key is the name of the model. "markdown_description": |- - The model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#model) + The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models) "pipelines": "description": |- - The pipeline definitions for the bundle. + The pipeline definitions for the bundle, where each key is the name of the pipeline. "markdown_description": |- - The pipeline definitions for the bundle. See [_](/dev-tools/bundles/resources.md#pipeline) + The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines) "quality_monitors": "description": |- - The quality monitor definitions for the bundle. + The quality monitor definitions for the bundle, where each key is the name of the quality monitor. "markdown_description": |- - The quality monitor definitions for the bundle. See [_](/dev-tools/bundles/resources.md#quality_monitor) + The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors) "registered_models": "description": |- - The registered model definitions for the bundle. + The registered model definitions for the bundle, where each key is the name of the registered model. "markdown_description": |- - The registered model definitions for the bundle. See [_](/dev-tools/bundles/resources.md#registered_model) + The registered model definitions for the bundle, where each key is the name of the registered model. See [_](/dev-tools/bundles/resources.md#registered_models) "schemas": "description": |- - The schema definitions for the bundle. + The schema definitions for the bundle, where each key is the name of the schema. "markdown_description": |- - The schema definitions for the bundle. See [_](/dev-tools/bundles/resources.md#schema) + The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas) "volumes": "description": |- - PLACEHOLDER + The volume definitions for the bundle, where each key is the name of the volume. + "markdown_description": |- + The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes) github.com/databricks/cli/bundle/config.Root: "artifacts": "description": |- Defines the attributes to build an artifact + "markdown_description": |- + Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [_](/dev-tools/bundles/settings.md#artifacts). + + Artifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [_](/dev-tools/bundles/artifact-overrides.md). + "markdown_examples": |- + ```yaml + artifacts: + default: + type: whl + build: poetry build + path: . + ``` "bundle": "description": |- - The attributes of the bundle. + The bundle attributes when deploying to this target. "markdown_description": |- - The attributes of the bundle. See [_](/dev-tools/bundles/settings.md#bundle) + The bundle attributes when deploying to this target, "experimental": "description": |- Defines attributes for experimental features. @@ -222,9 +228,21 @@ github.com/databricks/cli/bundle/config.Root: Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include) "permissions": "description": |- - Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle + Defines a permission for a specific entity. "markdown_description": |- - Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). + A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity. + + See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). + "markdown_examples": |- + ```yaml + permissions: + - level: CAN_VIEW + group_name: test-group + - level: CAN_MANAGE + user_name: someone@example.com + - level: CAN_RUN + service_principal_name: 123456-abcdef + ``` "presets": "description": |- Defines bundle deployment presets. @@ -232,26 +250,39 @@ github.com/databricks/cli/bundle/config.Root: Defines bundle deployment presets. See [_](/dev-tools/bundles/deployment-modes.md#presets). "resources": "description": |- - Specifies information about the Databricks resources used by the bundle + A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. "markdown_description": |- - Specifies information about the Databricks resources used by the bundle. See [_](/dev-tools/bundles/resources.md). + A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about supported resources, and resource definition reference, see [_](/dev-tools/bundles/resources.md). + + ```yaml + resources: + : + : + : + ``` "run_as": "description": |- - The identity to use to run the bundle. + The identity to use when running workflows. + "markdown_description": |- + The identity to use when running workflows. See [_](/dev-tools/bundles/run-as.md). "sync": "description": |- The files and file paths to include or exclude in the bundle. "markdown_description": |- - The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/) + The files and file paths to include or exclude in the bundle. See [_](/dev-tools/bundles/settings.md#sync). "targets": "description": |- Defines deployment targets for the bundle. + "markdown_description": |- + Defines deployment targets for the bundle. See [_](/dev-tools/bundles/settings.md#targets) "variables": "description": |- A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable. "workspace": "description": |- Defines the Databricks workspace for the bundle. + "markdown_description": |- + Defines the Databricks workspace for the bundle. See [_](/dev-tools/bundles/settings.md#workspace). github.com/databricks/cli/bundle/config.Sync: "exclude": "description": |- @@ -266,11 +297,9 @@ github.com/databricks/cli/bundle/config.Target: "artifacts": "description": |- The artifacts to include in the target deployment. - "markdown_description": |- - The artifacts to include in the target deployment. See [_](#artifact) "bundle": "description": |- - The name of the bundle when deploying to this target. + The bundle attributes when deploying to this target. "cluster_id": "description": |- The ID of the cluster to use for this target. @@ -283,8 +312,6 @@ github.com/databricks/cli/bundle/config.Target: "git": "description": |- The Git version control settings for the target. - "markdown_description": |- - The Git version control settings for the target. See [_](#git). "mode": "description": |- The deployment mode for the target. @@ -293,38 +320,26 @@ github.com/databricks/cli/bundle/config.Target: "permissions": "description": |- The permissions for deploying and running the bundle in the target. - "markdown_description": |- - The permissions for deploying and running the bundle in the target. See [_](#permission). "presets": "description": |- The deployment presets for the target. - "markdown_description": |- - The deployment presets for the target. See [_](#preset). "resources": "description": |- The resource definitions for the target. - "markdown_description": |- - The resource definitions for the target. See [_](#resources). "run_as": "description": |- The identity to use to run the bundle. "markdown_description": |- - The identity to use to run the bundle. See [_](#job_run_as) and [_](/dev-tools/bundles/run_as.md). + The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). "sync": "description": |- The local paths to sync to the target workspace when a bundle is run or deployed. - "markdown_description": |- - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#sync). "variables": "description": |- The custom variable definitions for the target. - "markdown_description": |- - The custom variable definitions for the target. See [_](/dev-tools/bundles/settings.md#variables) and [_](/dev-tools/bundles/variables.md). "workspace": "description": |- The Databricks workspace for the target. - "markdown_description": |- - The Databricks workspace for the target. [_](#workspace) github.com/databricks/cli/bundle/config.Workspace: "artifact_path": "description": |- @@ -374,64 +389,6 @@ github.com/databricks/cli/bundle/config.Workspace: "state_path": "description": |- The workspace state path -github.com/databricks/cli/bundle/config/resources.App: - "active_deployment": - "description": |- - PLACEHOLDER - "app_status": - "description": |- - PLACEHOLDER - "compute_status": - "description": |- - PLACEHOLDER - "config": - "description": |- - PLACEHOLDER - "create_time": - "description": |- - PLACEHOLDER - "creator": - "description": |- - PLACEHOLDER - "default_source_code_path": - "description": |- - PLACEHOLDER - "description": - "description": |- - PLACEHOLDER - "name": - "description": |- - PLACEHOLDER - "pending_deployment": - "description": |- - PLACEHOLDER - "permissions": - "description": |- - PLACEHOLDER - "resources": - "description": |- - PLACEHOLDER - "service_principal_client_id": - "description": |- - PLACEHOLDER - "service_principal_id": - "description": |- - PLACEHOLDER - "service_principal_name": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER - "updater": - "description": |- - PLACEHOLDER - "url": - "description": |- - PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Grant: "principal": "description": |- @@ -440,6 +397,11 @@ github.com/databricks/cli/bundle/config/resources.Grant: "description": |- The privileges to grant to the specified entity github.com/databricks/cli/bundle/config/resources.Permission: + "-": + "description": |- + Defines a permission for a specific entity. + "markdown_description": |- + Defines a permission for a specific entity. See [_](/dev-tools/bundles/settings.md#permissions) and [_](/dev-tools/bundles/permissions.md). "group_name": "description": |- The name of the group that has the permission set in level. @@ -506,6 +468,11 @@ github.com/databricks/cli/bundle/config/variable.TargetVariable: "description": |- The type of the variable. github.com/databricks/cli/bundle/config/variable.Variable: + "_": + "description": |- + Defines a custom variable for the bundle. + "markdown_description": |- + Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables). "default": "description": |- PLACEHOLDER @@ -516,107 +483,14 @@ github.com/databricks/cli/bundle/config/variable.Variable: "description": |- The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. "markdown_description": |- - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID." + The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. "type": "description": |- The type of the variable. -github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: - "create_time": +github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs: + "service_principal_name": "description": |- - PLACEHOLDER - "creator": + The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": "description": |- - PLACEHOLDER - "deployment_artifacts": - "description": |- - PLACEHOLDER - "deployment_id": - "description": |- - PLACEHOLDER - "mode": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "status": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: - "source_code_path": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResource: - "description": - "description": |- - PLACEHOLDER - "job": - "description": |- - PLACEHOLDER - "name": - "description": |- - PLACEHOLDER - "secret": - "description": |- - PLACEHOLDER - "serving_endpoint": - "description": |- - PLACEHOLDER - "sql_warehouse": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: - "key": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER - "scope": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: - "name": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER + The email of an active workspace user. Non-admin users can only set this field to their own email. diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 323432fa3..912a4fda0 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -1,4 +1,7 @@ github.com/databricks/cli/bundle/config/resources.App: + "active_deployment": + "description": |- + PLACEHOLDER "app_status": "description": |- PLACEHOLDER @@ -8,9 +11,30 @@ github.com/databricks/cli/bundle/config/resources.App: "config": "description": |- PLACEHOLDER + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "default_source_code_path": + "description": |- + PLACEHOLDER + "description": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER + "pending_deployment": + "description": |- + PLACEHOLDER "permissions": "description": |- PLACEHOLDER + "resources": + "description": |- + PLACEHOLDER "service_principal_client_id": "description": |- PLACEHOLDER @@ -23,7 +47,46 @@ github.com/databricks/cli/bundle/config/resources.App: "source_code_path": "description": |- PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER + "updater": + "description": |- + PLACEHOLDER + "url": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Cluster: + "_": + "markdown_description": |- + The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). + + "markdown_examples": |- + The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: + + ```yaml + bundle: + name: clusters + + resources: + clusters: + my_cluster: + num_workers: 2 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 2 + max_workers: 7 + spark_version: "13.3.x-scala2.12" + spark_conf: + "spark.executor.memory": "2g" + + jobs: + my_job: + tasks: + - task_key: test_task + notebook_task: + notebook_path: "./src/my_notebook.py" + ``` "data_security_mode": "description": |- PLACEHOLDER @@ -43,6 +106,24 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Dashboard: + "_": + "markdown_description": |- + The dashboard resource allows you to manage [AI/BI dashboards](/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [_](/dashboards/index.md). + "markdown_examples": |- + The following example includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to the Databricks workspace. + + ``` yaml + resources: + dashboards: + nyc_taxi_trip_analysis: + display_name: "NYC Taxi Trip Analysis" + file_path: ../src/nyc_taxi_trip_analysis.lvdash.json + warehouse_id: ${var.warehouse_id} + ``` + If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). + + In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). + "embed_credentials": "description": |- PLACEHOLDER @@ -53,6 +134,24 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Job: + "_": + "markdown_description": |- + The job resource allows you to define [jobs and their corresponding tasks](/api/workspace/jobs/create) in your bundle. For information about jobs, see [_](/jobs/index.md). For a tutorial that uses a template to create a job, see [_](/dev-tools/bundles/jobs-tutorial.md). + "markdown_examples": |- + The following example defines a job with the resource key `hello-job` with one notebook task: + + ```yaml + resources: + jobs: + hello-job: + name: hello-job + tasks: + - task_key: hello-task + notebook_task: + notebook_path: ./hello.py + ``` + + For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). "health": "description": |- PLACEHOLDER @@ -63,30 +162,186 @@ github.com/databricks/cli/bundle/config/resources.Job: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.MlflowExperiment: + "_": + "markdown_description": |- + The experiment resource allows you to define [MLflow experiments](/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [_](/mlflow/experiments.md). + "markdown_examples": |- + The following example defines an experiment that all users can view: + + ```yaml + resources: + experiments: + experiment: + name: my_ml_experiment + permissions: + - level: CAN_READ + group_name: users + description: MLflow experiment used to track runs + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.MlflowModel: + "_": + "markdown_description": |- + The model resource allows you to define [legacy models](/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use [registered models](#registered-model) instead. "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: + "_": + "markdown_description": |- + The model_serving_endpoint resource allows you to define [model serving endpoints](/api/workspace/servingendpoints/create). See [_](/machine-learning/model-serving/manage-serving-endpoints.md). + "markdown_examples": |- + The following example defines a model serving endpoint: + + ```yaml + resources: + model_serving_endpoints: + uc_model_serving_endpoint: + name: "uc-model-endpoint" + config: + served_entities: + - entity_name: "myCatalog.mySchema.my-ads-model" + entity_version: "10" + workload_size: "Small" + scale_to_zero_enabled: "true" + traffic_config: + routes: + - served_model_name: "my-ads-model-10" + traffic_percentage: "100" + tags: + - key: "team" + value: "data science" + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Pipeline: + "_": + "markdown_description": |- + The pipeline resource allows you to create [pipelines](/api/workspace/pipelines/create). For information about pipelines, see [_](/delta-live-tables/index.md). For a tutorial that uses the template to create a pipeline, see [_](/dev-tools/bundles/pipelines-tutorial.md). + "markdown_examples": |- + The following example defines a pipeline with the resource key `hello-pipeline`: + + ```yaml + resources: + pipelines: + hello-pipeline: + name: hello-pipeline + clusters: + - label: default + num_workers: 1 + development: true + continuous: false + channel: CURRENT + edition: CORE + photon: false + libraries: + - notebook: + path: ./pipeline.py + ``` "permissions": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.QualityMonitor: + "_": + "markdown_description": |- + The quality_monitor resource allows you to define a [table monitor](/api/workspace/qualitymonitors/create). For information about monitors, see [_](/machine-learning/model-serving/monitor-diagnose-endpoints.md). + "markdown_examples": |- + The following example defines a quality monitor: + + ```yaml + resources: + quality_monitors: + my_quality_monitor: + table_name: dev.mlops_schema.predictions + output_schema_name: ${bundle.target}.mlops_schema + assets_dir: /Users/${workspace.current_user.userName}/databricks_lakehouse_monitoring + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + label_col: price + problem_type: PROBLEM_TYPE_REGRESSION + timestamp_col: timestamp + schedule: + quartz_cron_expression: 0 0 8 * * ? # Run Every day at 8am + timezone_id: UTC + ``` "table_name": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.RegisteredModel: + "_": + "markdown_description": |- + The registered model resource allows you to define models in . For information about [registered models](/api/workspace/registeredmodels/create), see [_](/machine-learning/manage-model-lifecycle/index.md). + "markdown_examples": |- + The following example defines a registered model in : + + ```yaml + resources: + registered_models: + model: + name: my_model + catalog_name: ${bundle.target} + schema_name: mlops_schema + comment: Registered model in Unity Catalog for ${bundle.target} deployment target + grants: + - privileges: + - EXECUTE + principal: account users + ``` "grants": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Schema: + "_": + "markdown_description": |- + The schema resource type allows you to define [schemas](/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations: + + - The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema. + - Only fields supported by the corresponding [Schemas object create API](/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](/api/workspace/schemas/update). + "markdown_examples": |- + The following example defines a pipeline with the resource key `my_pipeline` that creates a schema with the key `my_schema` as the target: + + ```yaml + resources: + pipelines: + my_pipeline: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + target: ${resources.schemas.my_schema.id} + + schemas: + my_schema: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created by DABs. + ``` + + A top-level grants mapping is not supported by , so if you want to set grants for a schema, define the grants for the schema within the `schemas` mapping. For more information about grants, see [_](/data-governance/unity-catalog/manage-privileges/index.md#grant). + + The following example defines a schema with grants: + + ```yaml + resources: + schemas: + my_schema: + name: test-schema + grants: + - principal: users + privileges: + - CAN_MANAGE + - principal: my_team + privileges: + - CAN_READ + catalog_name: main + ``` "grants": "description": |- PLACEHOLDER @@ -94,6 +349,27 @@ github.com/databricks/cli/bundle/config/resources.Schema: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Volume: + "_": + "markdown_description": |- + The volume resource type allows you to define and create [volumes](/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that: + + - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. + + - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). + + "markdown_examples": |- + The following example creates a volume with the key `my_volume`: + + ```yaml + resources: + volumes: + my_volume: + catalog_name: main + name: my_volume + schema_name: my_schema + ``` + + For an example bundle that runs a job that writes to a file in volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume). "grants": "description": |- PLACEHOLDER @@ -197,6 +473,85 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "deployment_artifacts": + "description": |- + PLACEHOLDER + "deployment_id": + "description": |- + PLACEHOLDER + "mode": + "description": |- + PLACEHOLDER + "source_code_path": + "description": |- + PLACEHOLDER + "status": + "description": |- + PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: + "source_code_path": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: + "key": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER + "scope": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: + "name": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: + "message": + "description": |- + PLACEHOLDER + "state": github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/internal/schema/annotations_test.go b/bundle/internal/schema/annotations_test.go index d7e2fea7c..0e1593359 100644 --- a/bundle/internal/schema/annotations_test.go +++ b/bundle/internal/schema/annotations_test.go @@ -33,6 +33,10 @@ func TestConvertLinksToAbsoluteUrl(t *testing.T) { input: "This is a link to [external](https://external.com)", expected: "This is a link to [external](https://external.com)", }, + { + input: "This is a link to [one](/relative), [two](/relative-2)", + expected: "This is a link to [one](https://docs.databricks.com/relative), [two](https://docs.databricks.com/relative-2)", + }, } for _, test := range tests { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 06e89c856..051243c4d 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" @@ -113,13 +114,13 @@ func TestNoDetachedAnnotations(t *testing.T) { assert.Empty(t, types, "Detached annotations found, regenerate schema and check for package path changes") } -func getAnnotations(path string) (annotationFile, error) { +func getAnnotations(path string) (annotation.File, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } - var data annotationFile + var data annotation.File err = yaml.Unmarshal(b, &data) return data, err } diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 919908429..50e69e7c8 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/databricks/cli/bundle/internal/annotation" "github.com/databricks/cli/libs/jsonschema" "gopkg.in/yaml.v3" ) @@ -114,8 +115,8 @@ func mapIncorrectTypNames(ref string) string { // Use the OpenAPI spec to load descriptions for the given type. func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overridesPath string) error { - annotations := annotationFile{} - overrides := annotationFile{} + annotations := annotation.File{} + overrides := annotation.File{} b, err := os.ReadFile(overridesPath) if err != nil { @@ -126,7 +127,7 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid return err } if overrides == nil { - overrides = annotationFile{} + overrides = annotation.File{} } _, err = jsonschema.FromType(typ, []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ @@ -137,16 +138,16 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid } basePath := getPath(typ) - pkg := map[string]annotation{} + pkg := map[string]annotation.Descriptor{} annotations[basePath] = pkg if ref.Description != "" || ref.Enum != nil { - pkg[RootTypeKey] = annotation{Description: ref.Description, Enum: ref.Enum} + pkg[RootTypeKey] = annotation.Descriptor{Description: ref.Description, Enum: ref.Enum} } for k := range s.Properties { if refProp, ok := ref.Properties[k]; ok { - pkg[k] = annotation{Description: refProp.Description, Enum: refProp.Enum} + pkg[k] = annotation.Descriptor{Description: refProp.Description, Enum: refProp.Enum} if refProp.Description == "" { addEmptyOverride(k, basePath, overrides) } @@ -195,22 +196,22 @@ func prependCommentToFile(outputPath, comment string) error { return err } -func addEmptyOverride(key, pkg string, overridesFile annotationFile) { +func addEmptyOverride(key, pkg string, overridesFile annotation.File) { if overridesFile[pkg] == nil { - overridesFile[pkg] = map[string]annotation{} + overridesFile[pkg] = map[string]annotation.Descriptor{} } overrides := overridesFile[pkg] if overrides[key].Description == "" { - overrides[key] = annotation{Description: Placeholder} + overrides[key] = annotation.Descriptor{Description: annotation.Placeholder} } a, ok := overrides[key] if !ok { - a = annotation{} + a = annotation.Descriptor{} } if a.Description == "" { - a.Description = Placeholder + a.Description = annotation.Placeholder } overrides[key] = a } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 17a621ba0..7c72c440e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -258,7 +258,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The cluster resource defines an [all-purpose cluster](https://docs.databricks.com/api/workspace/clusters/create)." }, { "type": "string", @@ -321,7 +322,8 @@ "$ref": "#/$defs/string" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The dashboard resource allows you to manage [AI/BI dashboards](https://docs.databricks.com/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [link](https://docs.databricks.com/dashboards/index.html)." }, { "type": "string", @@ -442,7 +444,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The job resource allows you to define [jobs and their corresponding tasks](https://docs.databricks.com/api/workspace/jobs/create) in your bundle. For information about jobs, see [link](https://docs.databricks.com/jobs/index.html). For a tutorial that uses a \u003cDABS\u003e template to create a job, see [link](https://docs.databricks.com/dev-tools/bundles/jobs-tutorial.html)." }, { "type": "string", @@ -487,7 +490,8 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The experiment resource allows you to define [MLflow experiments](https://docs.databricks.com/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [link](https://docs.databricks.com/mlflow/experiments.html)." }, { "type": "string", @@ -532,7 +536,8 @@ "$ref": "#/$defs/string" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The model resource allows you to define [legacy models](https://docs.databricks.com/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use \u003cUC\u003e [registered models](https://docs.databricks.com/dev-tools/bundles/reference.html#registered-model) instead." }, { "type": "string", @@ -576,7 +581,8 @@ "additionalProperties": false, "required": [ "name" - ] + ], + "markdownDescription": "The model_serving_endpoint resource allows you to define [model serving endpoints](https://docs.databricks.com/api/workspace/servingendpoints/create). See [link](https://docs.databricks.com/machine-learning/model-serving/manage-serving-endpoints.html)." }, { "type": "string", @@ -718,7 +724,8 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "The pipeline resource allows you to create \u003cDLT\u003e [pipelines](https://docs.databricks.com/api/workspace/pipelines/create). For information about pipelines, see [link](https://docs.databricks.com/delta-live-tables/index.html). For a tutorial that uses the \u003cDABS\u003e template to create a pipeline, see [link](https://docs.databricks.com/dev-tools/bundles/pipelines-tutorial.html)." }, { "type": "string", @@ -792,7 +799,8 @@ "table_name", "assets_dir", "output_schema_name" - ] + ], + "markdownDescription": "The quality_monitor resource allows you to define a \u003cUC\u003e [table monitor](https://docs.databricks.com/api/workspace/qualitymonitors/create). For information about monitors, see [link](https://docs.databricks.com/machine-learning/model-serving/monitor-diagnose-endpoints.html)." }, { "type": "string", @@ -834,7 +842,8 @@ "catalog_name", "name", "schema_name" - ] + ], + "markdownDescription": "The registered model resource allows you to define models in \u003cUC\u003e. For information about \u003cUC\u003e [registered models](https://docs.databricks.com/api/workspace/registeredmodels/create), see [link](https://docs.databricks.com/machine-learning/manage-model-lifecycle/index.html)." }, { "type": "string", @@ -874,7 +883,8 @@ "required": [ "catalog_name", "name" - ] + ], + "markdownDescription": "The schema resource type allows you to define \u003cUC\u003e [schemas](https://docs.databricks.com/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:\n\n- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.\n- Only fields supported by the corresponding [Schemas object create API](https://docs.databricks.com/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](https://docs.databricks.com/api/workspace/schemas/update)." }, { "type": "string", @@ -919,7 +929,8 @@ "catalog_name", "name", "schema_name" - ] + ], + "markdownDescription": "The volume resource type allows you to define and create \u003cUC\u003e [volumes](https://docs.databricks.com/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:\n\n- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use \u003cDABS\u003e to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.\n\n- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [custom-presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#custom-presets)." }, { "type": "string", @@ -1005,6 +1016,7 @@ }, "variable.Variable": { "type": "object", + "description": "Defines a custom variable for the bundle.", "properties": { "default": { "$ref": "#/$defs/interface" @@ -1016,14 +1028,15 @@ "lookup": { "description": "The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup", - "markdownDescription": "The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID.\"" + "markdownDescription": "The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID." }, "type": { "description": "The type of the variable.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" } }, - "additionalProperties": false + "additionalProperties": false, + "markdownDescription": "Defines a custom variable for the bundle. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables)." }, "variable.VariableType": { "type": "string" @@ -1035,26 +1048,25 @@ "type": "object", "properties": { "build": { - "description": "An optional set of non-default build commands that you want to run locally before deployment.\n\nFor Python wheel builds, the Databricks CLI assumes that it can find a local install of the Python wheel package to run builds, and it runs the command python setup.py bdist_wheel by default during each bundle deployment.\n\nTo specify multiple build commands, separate each command with double-ampersand (\u0026\u0026) characters.", + "description": "An optional set of non-default build commands to run locally before deployment.", "$ref": "#/$defs/string" }, "executable": { - "description": "The executable type.", + "description": "The executable type. Valid values are `bash`, `sh`, and `cmd`.", "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" }, "files": { "description": "The source files for the artifact.", - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile", - "markdownDescription": "The source files for the artifact, defined as an [artifact_file](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact_file)." + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" }, "path": { "description": "The location where the built artifact will be saved.", "$ref": "#/$defs/string" }, "type": { - "description": "The type of the artifact.", + "description": "Required. The type of the artifact.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType", - "markdownDescription": "The type of the artifact. Valid values are `wheel` or `jar`" + "markdownDescription": "Required. The type of the artifact. Valid values are `whl`." } }, "additionalProperties": false, @@ -1074,7 +1086,7 @@ "type": "object", "properties": { "source": { - "description": "The path of the files used to build the artifact.", + "description": "Required. The path of the files used to build the artifact.", "$ref": "#/$defs/string" } }, @@ -1113,12 +1125,12 @@ "deployment": { "description": "The definition of the bundle deployment", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment", - "markdownDescription": "The definition of the bundle deployment. For supported attributes, see [deployment](https://docs.databricks.com/dev-tools/bundles/reference.html#deployment) and [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." + "markdownDescription": "The definition of the bundle deployment. For supported attributes see [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." }, "git": { "description": "The Git version control details that are associated with your bundle.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git", - "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes, see [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git) and [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." + "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes see [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." }, "name": { "description": "The name of the bundle.", @@ -1154,8 +1166,7 @@ }, "lock": { "description": "The deployment lock attributes.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock", - "markdownDescription": "The deployment lock attributes. See [lock](https://docs.databricks.com/dev-tools/bundles/reference.html#lock)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" } }, "additionalProperties": false @@ -1180,15 +1191,15 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python" }, "python_wheel_wrapper": { - "description": "Whether to use a Python wheel wrapper", + "description": "Whether to use a Python wheel wrapper.", "$ref": "#/$defs/bool" }, "scripts": { - "description": "The commands to run", + "description": "The commands to run.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" }, "use_legacy_run_as": { - "description": "Whether to use the legacy run_as behavior", + "description": "Whether to use the legacy run_as behavior.", "$ref": "#/$defs/bool" } }, @@ -1352,57 +1363,59 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App" }, "clusters": { - "description": "The cluster definitions for the bundle.", + "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", - "markdownDescription": "The cluster definitions for the bundle. See [cluster](https://docs.databricks.com/dev-tools/bundles/resources.html#cluster)" + "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [clusters](https://docs.databricks.com/dev-tools/bundles/resources.html#clusters)" }, "dashboards": { - "description": "The dashboard definitions for the bundle.", + "description": "The dashboard definitions for the bundle, where each key is the name of the dashboard.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard", - "markdownDescription": "The dashboard definitions for the bundle. See [dashboard](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboard)" + "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)" }, "experiments": { - "description": "The experiment definitions for the bundle.", + "description": "The experiment definitions for the bundle, where each key is the name of the experiment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment", - "markdownDescription": "The experiment definitions for the bundle. See [experiment](https://docs.databricks.com/dev-tools/bundles/resources.html#experiment)" + "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)" }, "jobs": { - "description": "The job definitions for the bundle.", + "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", - "markdownDescription": "The job definitions for the bundle. See [job](https://docs.databricks.com/dev-tools/bundles/resources.html#job)" + "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)" }, "model_serving_endpoints": { - "description": "The model serving endpoint definitions for the bundle.", + "description": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint", - "markdownDescription": "The model serving endpoint definitions for the bundle. See [model_serving_endpoint](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoint)" + "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoints](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoints)" }, "models": { - "description": "The model definitions for the bundle.", + "description": "The model definitions for the bundle, where each key is the name of the model.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel", - "markdownDescription": "The model definitions for the bundle. See [model](https://docs.databricks.com/dev-tools/bundles/resources.html#model)" + "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [models](https://docs.databricks.com/dev-tools/bundles/resources.html#models)" }, "pipelines": { - "description": "The pipeline definitions for the bundle.", + "description": "The pipeline definitions for the bundle, where each key is the name of the pipeline.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline", - "markdownDescription": "The pipeline definitions for the bundle. See [pipeline](https://docs.databricks.com/dev-tools/bundles/resources.html#pipeline)" + "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipelines](https://docs.databricks.com/dev-tools/bundles/resources.html#pipelines)" }, "quality_monitors": { - "description": "The quality monitor definitions for the bundle.", + "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor", - "markdownDescription": "The quality monitor definitions for the bundle. See [quality_monitor](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitor)" + "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitors)" }, "registered_models": { - "description": "The registered model definitions for the bundle.", + "description": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel", - "markdownDescription": "The registered model definitions for the bundle. See [registered_model](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_model)" + "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model. See [registered_models](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_models)" }, "schemas": { - "description": "The schema definitions for the bundle.", + "description": "The schema definitions for the bundle, where each key is the name of the schema.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema", - "markdownDescription": "The schema definitions for the bundle. See [schema](https://docs.databricks.com/dev-tools/bundles/resources.html#schema)" + "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)" }, "volumes": { - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume" + "description": "The volume definitions for the bundle, where each key is the name of the volume.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", + "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volumes](https://docs.databricks.com/dev-tools/bundles/resources.html#volumes)" } }, "additionalProperties": false @@ -1446,11 +1459,10 @@ "properties": { "artifacts": { "description": "The artifacts to include in the target deployment.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact", - "markdownDescription": "The artifacts to include in the target deployment. See [artifact](https://docs.databricks.com/dev-tools/bundles/reference.html#artifact)" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" }, "bundle": { - "description": "The name of the bundle when deploying to this target.", + "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" }, "cluster_id": { @@ -1467,8 +1479,7 @@ }, "git": { "description": "The Git version control settings for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git", - "markdownDescription": "The Git version control settings for the target. See [git](https://docs.databricks.com/dev-tools/bundles/reference.html#git)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" }, "mode": { "description": "The deployment mode for the target.", @@ -1477,38 +1488,32 @@ }, "permissions": { "description": "The permissions for deploying and running the bundle in the target.", - "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", - "markdownDescription": "The permissions for deploying and running the bundle in the target. See [permission](https://docs.databricks.com/dev-tools/bundles/reference.html#permission)." + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "presets": { "description": "The deployment presets for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets", - "markdownDescription": "The deployment presets for the target. See [preset](https://docs.databricks.com/dev-tools/bundles/reference.html#preset)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" }, "resources": { "description": "The resource definitions for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources", - "markdownDescription": "The resource definitions for the target. See [resources](https://docs.databricks.com/dev-tools/bundles/reference.html#resources)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" }, "run_as": { "description": "The identity to use to run the bundle.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", - "markdownDescription": "The identity to use to run the bundle. See [job_run_as](https://docs.databricks.com/dev-tools/bundles/reference.html#job_run_as) and [link](https://docs.databricks.com/dev-tools/bundles/run_as.html)." + "markdownDescription": "The identity to use to run the bundle, see [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." }, "sync": { "description": "The local paths to sync to the target workspace when a bundle is run or deployed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync", - "markdownDescription": "The local paths to sync to the target workspace when a bundle is run or deployed. See [sync](https://docs.databricks.com/dev-tools/bundles/reference.html#sync)." + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { "description": "The custom variable definitions for the target.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable", - "markdownDescription": "The custom variable definitions for the target. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables) and [link](https://docs.databricks.com/dev-tools/bundles/variables.html)." + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable" }, "workspace": { "description": "The Databricks workspace for the target.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace", - "markdownDescription": "The Databricks workspace for the target. [workspace](https://docs.databricks.com/dev-tools/bundles/reference.html#workspace)" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" } }, "additionalProperties": false @@ -1719,12 +1724,14 @@ "type": "object", "properties": { "description": { + "description": "Description of the App Resource.", "$ref": "#/$defs/string" }, "job": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob" }, "name": { + "description": "Name of the App Resource.", "$ref": "#/$defs/string" }, "secret": { @@ -1980,6 +1987,7 @@ "$ref": "#/$defs/string" }, "state": { + "description": "State of the app compute.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeState" } }, @@ -3701,7 +3709,7 @@ "description": "Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nEither `user_name` or `service_principal_name` should be specified. If not, an error is thrown.", "properties": { "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "description": "The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", "$ref": "#/$defs/string" }, "user_name": { @@ -7227,12 +7235,13 @@ "properties": { "artifacts": { "description": "Defines the attributes to build an artifact", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact", + "markdownDescription": "Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [artifacts](https://docs.databricks.com/dev-tools/bundles/settings.html#artifacts).\n\nArtifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [link](https://docs.databricks.com/dev-tools/bundles/artifact-overrides.html)." }, "bundle": { - "description": "The attributes of the bundle.", + "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle", - "markdownDescription": "The attributes of the bundle. See [bundle](https://docs.databricks.com/dev-tools/bundles/settings.html#bundle)" + "markdownDescription": "The bundle attributes when deploying to this target," }, "experimental": { "description": "Defines attributes for experimental features.", @@ -7244,9 +7253,9 @@ "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)" }, "permissions": { - "description": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle", + "description": "Defines a permission for a specific entity.", "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", - "markdownDescription": "Defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle. See [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." + "markdownDescription": "A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." }, "presets": { "description": "Defines bundle deployment presets.", @@ -7254,22 +7263,24 @@ "markdownDescription": "Defines bundle deployment presets. See [presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#presets)." }, "resources": { - "description": "Specifies information about the Databricks resources used by the bundle", + "description": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources", - "markdownDescription": "Specifies information about the Databricks resources used by the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/resources.html)." + "markdownDescription": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about \u003cDABS\u003e supported resources, and resource definition reference, see [link](https://docs.databricks.com/dev-tools/bundles/resources.html).\n\n```yaml\nresources:\n \u003cresource-type\u003e:\n \u003cresource-name\u003e:\n \u003cresource-field-name\u003e: \u003cresource-field-value\u003e\n```" }, "run_as": { - "description": "The identity to use to run the bundle.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + "description": "The identity to use when running \u003cDABS\u003e workflows.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", + "markdownDescription": "The identity to use when running \u003cDABS\u003e workflows. See [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." }, "sync": { "description": "The files and file paths to include or exclude in the bundle.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync", - "markdownDescription": "The files and file paths to include or exclude in the bundle. See [link](https://docs.databricks.com/dev-tools/bundles/)" + "markdownDescription": "The files and file paths to include or exclude in the bundle. See [sync](https://docs.databricks.com/dev-tools/bundles/settings.html#sync)." }, "targets": { "description": "Defines deployment targets for the bundle.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target", + "markdownDescription": "Defines deployment targets for the bundle. See [targets](https://docs.databricks.com/dev-tools/bundles/settings.html#targets)" }, "variables": { "description": "A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable.", @@ -7277,7 +7288,8 @@ }, "workspace": { "description": "Defines the Databricks workspace for the bundle.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace", + "markdownDescription": "Defines the Databricks workspace for the bundle. See [workspace](https://docs.databricks.com/dev-tools/bundles/settings.html#workspace)." } }, "additionalProperties": {} diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 6f8f39d96..ce25cb023 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -111,6 +111,10 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) ( return res, nil } +func TypePath(typ reflect.Type) string { + return typePath(typ) +} + // typePath computes a unique string representation of the type. $ref in the generated // JSON schema will refer to this path. See TestTypePath for examples outputs. func typePath(typ reflect.Type) string { diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index e63dde359..5028bb0d7 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -76,6 +76,10 @@ type Schema struct { // Title of the object, rendered as inline documentation in the IDE. // https://json-schema.org/understanding-json-schema/reference/annotations Title string `json:"title,omitempty"` + + // Examples of the value for properties in the schema. + // https://json-schema.org/understanding-json-schema/reference/annotations + Examples []any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string. From 59d6fbfee98bf102716f774de3de99aca129a9c3 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 29 Jan 2025 14:34:26 +0100 Subject: [PATCH 026/105] Restore variable file tests (#2220) ## Changes Uncomment flaky tests, they work properly with latest changes from main ## Tests --- .../bundle/variables/file-defaults/output.txt | 31 +++++++++++++++++++ .../bundle/variables/file-defaults/script | 8 ++--- .../bundle/variables/file-defaults/test.toml | 8 +++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/variables/file-defaults/test.toml diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt index 73830aae3..5b01a1b66 100644 --- a/acceptance/bundle/variables/file-defaults/output.txt +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -39,6 +39,37 @@ } } +=== file cannot be parsed +>>> errcode $CLI bundle validate -o json --target invalid_json +Error: failed to parse variables file $TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +=== file has wrong structure +>>> errcode $CLI bundle validate -o json --target wrong_file_structure +Error: failed to parse variables file $TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format + +Variables file must be a JSON object with the following format: +{"var1": "value1", "var2": "value2"} + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + === file has variable that is complex but default is string >>> errcode $CLI bundle validate -o json --target complex_to_string Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script index c5b208755..8e6fd0d75 100644 --- a/acceptance/bundle/variables/file-defaults/script +++ b/acceptance/bundle/variables/file-defaults/script @@ -14,11 +14,11 @@ trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate title "variable has value in config file" trace $CLI bundle validate -o json --target with_value | jq $cluster_expr -# title "file cannot be parsed" -# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr +title "file cannot be parsed" +trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr -# title "file has wrong structure" -# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr +title "file has wrong structure" +trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr title "file has variable that is complex but default is string" trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr diff --git a/acceptance/bundle/variables/file-defaults/test.toml b/acceptance/bundle/variables/file-defaults/test.toml new file mode 100644 index 000000000..08403b606 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/test.toml @@ -0,0 +1,8 @@ +# Fix for windows +[[Repls]] +Old = '\$TMPDIR\\.databricks\\bundle\\wrong_file_structure\\variable-overrides.json' +New = '$$TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json' + +[[Repls]] +Old = '\$TMPDIR\\.databricks\\bundle\\invalid_json\\variable-overrides.json' +New = '$$TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json' From ec7808da34cb93c5a4c220edd1dc70b6e40ca4af Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 29 Jan 2025 14:38:28 +0100 Subject: [PATCH 027/105] Added support for double underscore variable references (#2203) ## Changes Added support for double underscore variable references. Previously we made this restriction stronger with no particular reason, TF provider supports multiple underscores and thus DABs should do as well. Fixes #1753 ## Tests Added acceptance and integration tests --- .../double_underscore/databricks.yml | 14 +++++++ .../variables/double_underscore/output.txt | 7 ++++ .../bundle/variables/double_underscore/script | 1 + integration/bundle/basic_test.go | 39 +++++++++++++++++++ .../databricks_template_schema.json | 21 ++++++++++ .../template/databricks.yml.tmpl | 32 +++++++++++++++ .../template/hello_world.py | 1 + .../basic_with_variables/bundle_deploy.txt | 4 ++ .../basic_with_variables/bundle_validate.txt | 7 ++++ libs/dyn/dynvar/ref.go | 6 ++- libs/dyn/dynvar/ref_test.go | 12 +++--- 11 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/variables/double_underscore/databricks.yml create mode 100644 acceptance/bundle/variables/double_underscore/output.txt create mode 100644 acceptance/bundle/variables/double_underscore/script create mode 100644 integration/bundle/bundles/basic_with_variables/databricks_template_schema.json create mode 100644 integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl create mode 100644 integration/bundle/bundles/basic_with_variables/template/hello_world.py create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_deploy.txt create mode 100644 integration/bundle/testdata/basic_with_variables/bundle_validate.txt diff --git a/acceptance/bundle/variables/double_underscore/databricks.yml b/acceptance/bundle/variables/double_underscore/databricks.yml new file mode 100644 index 000000000..3bb15d42d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: double_underscore + +variables: + double__underscore: + description: "This is a variable with a double underscore" + default: "default" + +resources: + jobs: + test_job: + name: "test" + tasks: + - task_key: "test ${var.double__underscore}" diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt new file mode 100644 index 000000000..45529038d --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/output.txt @@ -0,0 +1,7 @@ + +>>> $CLI bundle validate -o json +[ + { + "task_key": "test default" + } +] diff --git a/acceptance/bundle/variables/double_underscore/script b/acceptance/bundle/variables/double_underscore/script new file mode 100644 index 000000000..a7394df77 --- /dev/null +++ b/acceptance/bundle/variables/double_underscore/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq .resources.jobs.test_job.tasks diff --git a/integration/bundle/basic_test.go b/integration/bundle/basic_test.go index 79301b850..53f8e3ef6 100644 --- a/integration/bundle/basic_test.go +++ b/integration/bundle/basic_test.go @@ -6,7 +6,9 @@ import ( "testing" "github.com/databricks/cli/integration/internal/acc" + "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/testdiff" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -35,3 +37,40 @@ func TestBasicBundleDeployWithFailOnActiveRuns(t *testing.T) { // deploy empty bundle again deployBundleWithFlags(t, ctx, root, []string{"--fail-on-active-runs"}) } + +func TestBasicBundleDeployWithDoubleUnderscoreVariables(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + nodeTypeId := testutil.GetCloud(t).NodeTypeID() + uniqueId := uuid.New().String() + root := initTestTemplate(t, ctx, "basic_with_variables", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + + currentUser, err := wt.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + ctx, replacements := testdiff.WithReplacementsMap(ctx) + replacements.Set(uniqueId, "$UNIQUE_PRJ") + replacements.Set(currentUser.UserName, "$USERNAME") + + t.Cleanup(func() { + destroyBundle(t, ctx, root) + }) + + testutil.Chdir(t, root) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "validate"}, + testutil.TestData("testdata/basic_with_variables/bundle_validate.txt"), + ) + testcli.AssertOutput( + t, + ctx, + []string{"bundle", "deploy", "--force-lock", "--auto-approve"}, + testutil.TestData("testdata/basic_with_variables/bundle_deploy.txt"), + ) +} diff --git a/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json new file mode 100644 index 000000000..41a723b0f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/databricks_template_schema.json @@ -0,0 +1,21 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + }, + "root_path": { + "type": "string", + "description": "Root path to deploy bundle to", + "default": "" + } + } +} diff --git a/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl new file mode 100644 index 000000000..cb02c9e2f --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/databricks.yml.tmpl @@ -0,0 +1,32 @@ +bundle: + name: basic + +workspace: + {{ if .root_path }} + root_path: "{{.root_path}}/.bundle/{{.unique_id}}" + {{ else }} + root_path: "~/.bundle/{{.unique_id}}" + {{ end }} + +variables: + task__key: # Note: the variable has double underscore + default: my_notebook_task + +resources: + jobs: + foo__bar: # Note: the resource has double underscore to check that TF provider can use such names + name: test-job-basic-{{.unique_id}} + tasks: + - task_key: ${var.task__key} + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + spark_python_task: + python_file: ./hello_world.py + foo: + name: test-job-basic-ref-{{.unique_id}} + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.foo__bar.id} diff --git a/integration/bundle/bundles/basic_with_variables/template/hello_world.py b/integration/bundle/bundles/basic_with_variables/template/hello_world.py new file mode 100644 index 000000000..f301245e2 --- /dev/null +++ b/integration/bundle/bundles/basic_with_variables/template/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt new file mode 100644 index 000000000..211164174 --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_deploy.txt @@ -0,0 +1,4 @@ +Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/integration/bundle/testdata/basic_with_variables/bundle_validate.txt b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt new file mode 100644 index 000000000..dc9016a0f --- /dev/null +++ b/integration/bundle/testdata/basic_with_variables/bundle_validate.txt @@ -0,0 +1,7 @@ +Name: basic +Target: default +Workspace: + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ + +Validation OK! diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index a28938823..ba397267a 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -1,12 +1,16 @@ package dynvar import ( + "fmt" "regexp" "github.com/databricks/cli/libs/dyn" ) -var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`) +var ( + baseVarDef = `[a-zA-Z]+([-_]*[a-zA-Z0-9]+)*` + re = regexp.MustCompile(fmt.Sprintf(`\$\{(%s(\.%s(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, baseVarDef, baseVarDef)) +) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/dyn/dynvar/ref_test.go b/libs/dyn/dynvar/ref_test.go index 4110732f8..637ecb98e 100644 --- a/libs/dyn/dynvar/ref_test.go +++ b/libs/dyn/dynvar/ref_test.go @@ -15,9 +15,13 @@ func TestNewRefNoString(t *testing.T) { func TestNewRefValidPattern(t *testing.T) { for in, refs := range map[string][]string{ - "${hello_world.world_world}": {"hello_world.world_world"}, - "${helloworld.world-world}": {"helloworld.world-world"}, - "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world_world}": {"hello_world.world_world"}, + "${helloworld.world-world}": {"helloworld.world-world"}, + "${hello-world.world-world}": {"hello-world.world-world"}, + "${hello_world.world__world}": {"hello_world.world__world"}, + "${hello_world.world--world}": {"hello_world.world--world"}, + "${hello_world.world-_world}": {"hello_world.world-_world"}, + "${hello_world.world_-world}": {"hello_world.world_-world"}, } { ref, ok := newRef(dyn.V(in)) require.True(t, ok, "should match valid pattern: %s", in) @@ -36,8 +40,6 @@ func TestNewRefInvalidPattern(t *testing.T) { "${_-_._-_.id}", // cannot use _- in sequence "${0helloworld.world-world}", // interpolated first section shouldn't start with number "${helloworld.9world-world}", // interpolated second section shouldn't start with number - "${a-a.a-_a-a.id}", // fails because of -_ in the second segment - "${a-a.a--a-a.id}", // fails because of -- in the second segment } for _, v := range invalid { _, ok := newRef(dyn.V(v)) From 13596eb6052fc90a4df1ab3a0130a0a568959c18 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 29 Jan 2025 14:56:57 +0100 Subject: [PATCH 028/105] PythonMutator: Fix relative path error (#2253) ## Changes Fix relative path errors in the Python mutator that was failing during deployment since v0.239.1. Before that: ``` % databricks bundle deploy Deploying resources... Updating deployment state... Error: failed to compute relative path for job jobs_as_code_project_job: Rel: can't make resources/jobs_as_code_project_job.py relative to /Users/$USER/jobs_as_code_project ``` As a result, the bundle was deployed, but the deployment state wasn't updated. ## Tests Unit tests, adding acceptance tests in https://github.com/databricks/cli/pull/2254 --- .../config/mutator/python/python_locations.go | 9 ++++++- .../mutator/python/python_locations_test.go | 24 +++++++++++++++---- .../config/mutator/python/python_mutator.go | 6 ++--- .../mutator/python/python_mutator_test.go | 7 +++++- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/bundle/config/mutator/python/python_locations.go b/bundle/config/mutator/python/python_locations.go index 2fa86bea0..9cb65c302 100644 --- a/bundle/config/mutator/python/python_locations.go +++ b/bundle/config/mutator/python/python_locations.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + pathlib "path" "path/filepath" "github.com/databricks/cli/libs/dyn" @@ -99,7 +100,7 @@ func removeVirtualLocations(locations []dyn.Location) []dyn.Location { // parsePythonLocations parses locations.json from the Python mutator. // // locations file is newline-separated JSON objects with pythonLocationEntry structure. -func parsePythonLocations(input io.Reader) (*pythonLocations, error) { +func parsePythonLocations(bundleRoot string, input io.Reader) (*pythonLocations, error) { decoder := json.NewDecoder(input) locations := newPythonLocations() @@ -116,6 +117,12 @@ func parsePythonLocations(input io.Reader) (*pythonLocations, error) { return nil, fmt.Errorf("failed to parse python location: %s", err) } + // Output can contain both relative paths and absolute paths outside of bundle root. + // Mutator pipeline expects all path to be absolute at this point, so make all paths absolute. + if !pathlib.IsAbs(entry.File) { + entry.File = filepath.Join(bundleRoot, entry.File) + } + location := dyn.Location{ File: entry.File, Line: entry.Line, diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go index 32afcc92b..2860af820 100644 --- a/bundle/config/mutator/python/python_locations_test.go +++ b/bundle/config/mutator/python/python_locations_test.go @@ -165,12 +165,28 @@ func TestLoadOutput(t *testing.T) { require.Equal(t, filepath.Join(bundleRoot, generatedFileName), notebookPath.Locations()[0].File) } -func TestParsePythonLocations(t *testing.T) { - expected := dyn.Location{File: "foo.py", Line: 1, Column: 2} +func TestParsePythonLocations_absolutePath(t *testing.T) { + // output can contain absolute path that is outside of the bundle root + expected := dyn.Location{File: "/Shared/foo.py", Line: 1, Column: 2} - input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}` + input := `{"path": "foo", "file": "/Shared/foo.py", "line": 1, "column": 2}` reader := bytes.NewReader([]byte(input)) - locations, err := parsePythonLocations(reader) + locations, err := parsePythonLocations("/tmp/", reader) + + assert.NoError(t, err) + + assert.True(t, locations.keys["foo"].exists) + assert.Equal(t, expected, locations.keys["foo"].location) +} + +func TestParsePythonLocations_relativePath(t *testing.T) { + // output can contain relative paths, we expect all locations to be absolute + // at this stage of mutator pipeline + expected := dyn.Location{File: filepath.Clean("/tmp/my_project/foo.py"), Line: 1, Column: 2} + + input := `{"path": "foo", "file": "foo.py", "line": 1, "column": 2}` + reader := bytes.NewReader([]byte(input)) + locations, err := parsePythonLocations(filepath.Clean("/tmp/my_project"), reader) assert.NoError(t, err) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index cd2e286e5..f75f111cf 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -331,7 +331,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) } - locations, err := loadLocationsFile(locationsPath) + locations, err := loadLocationsFile(opts.bundleRootPath, locationsPath) if err != nil { return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err) } @@ -381,7 +381,7 @@ func writeInputFile(inputPath string, input dyn.Value) error { } // loadLocationsFile loads locations.json containing source locations for generated YAML. -func loadLocationsFile(locationsPath string) (*pythonLocations, error) { +func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, error) { locationsFile, err := os.Open(locationsPath) if errors.Is(err, fs.ErrNotExist) { return newPythonLocations(), nil @@ -391,7 +391,7 @@ func loadLocationsFile(locationsPath string) (*pythonLocations, error) { defer locationsFile.Close() - return parsePythonLocations(locationsFile) + return parsePythonLocations(bundleRoot, locationsFile) } func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 322fb79e8..9d957e797 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -54,6 +54,8 @@ func TestPythonMutator_Name_applyMutators(t *testing.T) { func TestPythonMutator_loadResources(t *testing.T) { withFakeVEnv(t, ".venv") + rootPath := filepath.Join(t.TempDir(), "my_project") + b := loadYaml("databricks.yml", ` experimental: python: @@ -64,6 +66,9 @@ func TestPythonMutator_loadResources(t *testing.T) { job0: name: job_0`) + // set rootPath so that we can make absolute paths in dyn.Location + b.BundleRootPath = rootPath + ctx := withProcessStub( t, []string{ @@ -120,7 +125,7 @@ func TestPythonMutator_loadResources(t *testing.T) { assert.Equal(t, []dyn.Location{ { - File: "src/examples/job1.py", + File: filepath.Join(rootPath, "src/examples/job1.py"), Line: 5, Column: 7, }, From c3a6e11627e1cb0b10c2289fbfe39a2a36fa641a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 19:35:58 +0530 Subject: [PATCH 029/105] Add integration test for the /telemetry-ext endpoint (#2259) ## Changes Followup from https://github.com/databricks/cli/pull/2209#pullrequestreview-2580308075. This PR adds an integration test to validate that the API type bindings work against the telemetry endpoint. ## Tests N/A --------- Co-authored-by: Pieter Noordhuis --- integration/libs/telemetry/telemetry_test.go | 65 ++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 integration/libs/telemetry/telemetry_test.go diff --git a/integration/libs/telemetry/telemetry_test.go b/integration/libs/telemetry/telemetry_test.go new file mode 100644 index 000000000..d329c238e --- /dev/null +++ b/integration/libs/telemetry/telemetry_test.go @@ -0,0 +1,65 @@ +package telemetry + +import ( + "encoding/json" + "testing" + "time" + + "github.com/databricks/cli/integration/internal/acc" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/client" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTelemetryEndpoint(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + + apiClient, err := client.New(w.Config) + require.NoError(t, err) + + logs := []protos.FrontendLog{ + { + FrontendLogEventID: uuid.New().String(), + Entry: protos.FrontendLogEntry{ + DatabricksCliLog: protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}, + }, + }, + }, + { + FrontendLogEventID: uuid.New().String(), + Entry: protos.FrontendLogEntry{ + DatabricksCliLog: protos.DatabricksCliLog{ + CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue2}, + }, + }, + }, + } + + protoLogs := make([]string, len(logs)) + for i, log := range logs { + b, err := json.Marshal(log) + require.NoError(t, err) + protoLogs[i] = string(b) + } + + reqB := telemetry.RequestBody{ + UploadTime: time.Now().UnixMilli(), + Items: []string{}, + ProtoLogs: protoLogs, + } + + respB := telemetry.ResponseBody{} + + err = apiClient.Do(ctx, "POST", "/telemetry-ext", nil, nil, reqB, &respB) + require.NoError(t, err) + + assert.Equal(t, telemetry.ResponseBody{ + Errors: []telemetry.LogError{}, + NumProtoSuccess: int64(2), + }, respB) +} From 38efedcd736fad3af8f985fd9cab26f246d69dfd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 29 Jan 2025 15:15:52 +0100 Subject: [PATCH 030/105] Remove bundle.git.inferred (#2258) The only use case for it was to emit a warning and based on the discussion here https://github.com/databricks/cli/pull/2213/files#r1933558087 the warning it not useful and logging that with reduced severity is also not useful. --- acceptance/bundle/git-permerror/output.txt | 18 ++++++------------ .../variables/prepend-workspace-var/output.txt | 3 +-- bundle/config/git.go | 3 --- bundle/config/mutator/load_git_details.go | 1 - bundle/config/mutator/process_target_mode.go | 5 ----- bundle/deploy/metadata/compute_test.go | 4 ---- bundle/tests/environment_git_test.go | 2 -- bundle/tests/git_test.go | 3 --- .../testdata/default_python/bundle_summary.txt | 3 +-- 9 files changed, 8 insertions(+), 34 deletions(-) diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt index 2b52134ab..60e77ca0e 100644 --- a/acceptance/bundle/git-permerror/output.txt +++ b/acceptance/bundle/git-permerror/output.txt @@ -21,8 +21,7 @@ Error: unable to load repository specific gitconfig: open config: permission den Exit code: 1 { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } >>> withdir subdir/a/b $CLI bundle validate -o json @@ -31,8 +30,7 @@ Error: unable to load repository specific gitconfig: open config: permission den Exit code: 1 { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } @@ -42,14 +40,12 @@ Exit code: 1 >>> $CLI bundle validate -o json { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } >>> withdir subdir/a/b $CLI bundle validate -o json { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } @@ -63,8 +59,7 @@ Error: unable to load repository specific gitconfig: open config: permission den Exit code: 1 { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } >>> withdir subdir/a/b $CLI bundle validate -o json @@ -73,6 +68,5 @@ Error: unable to load repository specific gitconfig: open config: permission den Exit code: 1 { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." } diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index 575fac6d4..706d134ff 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -3,8 +3,7 @@ "bundle": { "environment": "dev", "git": { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." }, "target": "dev", "terraform": { diff --git a/bundle/config/git.go b/bundle/config/git.go index f9f2f83e5..4b89bc2d2 100644 --- a/bundle/config/git.go +++ b/bundle/config/git.go @@ -8,9 +8,6 @@ type Git struct { // Path to bundle root relative to the git repository root. BundleRootPath string `json:"bundle_root_path,omitempty" bundle:"readonly"` - // Inferred is set to true if the Git details were inferred and weren't set explicitly - Inferred bool `json:"inferred,omitempty" bundle:"readonly"` - // The actual branch according to Git (may be different from the configured branch) ActualBranch string `json:"actual_branch,omitempty" bundle:"readonly"` } diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 3661c6bcd..dea948fcb 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -40,7 +40,6 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn b.Config.Bundle.Git.ActualBranch = info.CurrentBranch if b.Config.Bundle.Git.Branch == "" { // Only load branch if there's no user defined value - b.Config.Bundle.Git.Inferred = true b.Config.Bundle.Git.Branch = info.CurrentBranch } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 0fe6bd54f..576f0c352 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -135,11 +135,6 @@ func findNonUserPath(b *bundle.Bundle) string { } func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUsed bool) diag.Diagnostics { - if b.Config.Bundle.Git.Inferred { - env := b.Config.Bundle.Target - log.Warnf(ctx, "target with 'mode: production' should specify an explicit 'targets.%s.git' configuration", env) - } - r := b.Config.Resources for i := range r.Pipelines { if r.Pipelines[i].Development { diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index c6fa9bddb..64f899695 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -31,7 +31,6 @@ func TestComputeMetadataMutator(t *testing.T) { OriginURL: "www.host.com", Commit: "abcd", BundleRootPath: "a/b/c/d", - Inferred: true, }, }, Resources: config.Resources{ @@ -72,9 +71,6 @@ func TestComputeMetadataMutator(t *testing.T) { OriginURL: "www.host.com", Commit: "abcd", BundleRootPath: "a/b/c/d", - - // Test that this field doesn't carry over into the metadata. - Inferred: false, }, }, Resources: metadata.Resources{ diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go index 848b972b1..901d2867b 100644 --- a/bundle/tests/environment_git_test.go +++ b/bundle/tests/environment_git_test.go @@ -13,7 +13,6 @@ import ( func TestGitAutoLoadWithEnvironment(t *testing.T) { b := load(t, "./environments_autoload_git") bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) - assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } @@ -21,7 +20,6 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) { func TestGitManuallySetBranchWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_autoload_git", "production") bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) - assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go index 41293e450..dd79e26a4 100644 --- a/bundle/tests/git_test.go +++ b/bundle/tests/git_test.go @@ -14,7 +14,6 @@ import ( func TestGitAutoLoad(t *testing.T) { b := load(t, "./autoload_git") bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) - assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } @@ -22,7 +21,6 @@ func TestGitAutoLoad(t *testing.T) { func TestGitManuallySetBranch(t *testing.T) { b := loadTarget(t, "./autoload_git", "production") bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) - assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) @@ -36,7 +34,6 @@ func TestGitBundleBranchValidation(t *testing.T) { b := load(t, "./git_branch_validation") bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) - assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "feature-a", b.Config.Bundle.Git.Branch) assert.Equal(t, "feature-b", b.Config.Bundle.Git.ActualBranch) diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 88ccdc496..0b4c15764 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -7,8 +7,7 @@ "exec_path": "/tmp/.../terraform" }, "git": { - "bundle_root_path": ".", - "inferred": true + "bundle_root_path": "." }, "mode": "development", "deployment": { From ce965b22b2dc194ce90d101bf52f892d18e3773d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 29 Jan 2025 16:55:53 +0100 Subject: [PATCH 031/105] [Release] Release v0.240.0 (#2264) Bundles: * Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)). * Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)). * Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)). * libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)). API Changes: * Added `databricks access-control` command group. * Added `databricks serving-endpoints http-request` command. * Changed `databricks serving-endpoints create` command with new required argument order. * Changed `databricks serving-endpoints get-open-api` command return type to become non-empty. * Changed `databricks recipients update` command return type to become non-empty. OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22) Dependency updates: * Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)). * Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)). --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 255bfb0a8..449c30288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Version changelog +## [Release] Release v0.240.0 + +Bundles: + * Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)). + * Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)). + * Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)). + * libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)). + +API Changes: + * Added `databricks access-control` command group. + * Added `databricks serving-endpoints http-request` command. + * Changed `databricks serving-endpoints create` command with new required argument order. + * Changed `databricks serving-endpoints get-open-api` command return type to become non-empty. + * Changed `databricks recipients update` command return type to become non-empty. + +OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22) +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)). + * Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)). + ## [Release] Release v0.239.1 CLI: From 55c03cc119c2792b084928f12a15b93b3a6ed14a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 21:24:33 +0530 Subject: [PATCH 032/105] Always close test HTTP server during cleanup (#2261) ## Changes This PR registers the `server.Close()` function to be run during test cleanup in the server initialization function. This ensures that all test servers are closed as soon as the test they are scoped to finish. Motivated by https://github.com/databricks/cli/pull/2255/files where a regression was introduced where we did not close the test server. ## Tests N/A --- acceptance/cmd_server_test.go | 3 ++- acceptance/server_test.go | 9 --------- libs/testserver/server.go | 5 +---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 3f5a6356e..9af63d0db 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -13,7 +13,8 @@ import ( ) func StartCmdServer(t *testing.T) *testserver.Server { - server := StartServer(t) + server := testserver.New(t) + server.Handle("/", func(r *http.Request) (any, error) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 66de5dcbf..98e351739 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -2,7 +2,6 @@ package acceptance_test import ( "net/http" - "testing" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -11,14 +10,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) -func StartServer(t *testing.T) *testserver.Server { - server := testserver.New(t) - t.Cleanup(func() { - server.Close() - }) - return server -} - func AddHandlers(server *testserver.Server) { server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) { return compute.ListPoliciesResponse{ diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 10269af8f..9ebfe3ba0 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -18,6 +18,7 @@ type Server struct { func New(t testutil.TestingT) *Server { mux := http.NewServeMux() server := httptest.NewServer(mux) + t.Cleanup(server.Close) return &Server{ Server: server, @@ -28,10 +29,6 @@ func New(t testutil.TestingT) *Server { type HandlerFunc func(req *http.Request) (resp any, err error) -func (s *Server) Close() { - s.Server.Close() -} - func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { resp, err := handler(r) From 58ef34f320f4865d2a62d08b80798fdbdcf7294d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 29 Jan 2025 18:35:03 +0100 Subject: [PATCH 033/105] acc: Include "id" into /api/2.0/preview/scim/v2/Me response (#2266) This is something terraform provider expects. Related to https://github.com/databricks/cli/pull/2242 --- acceptance/bundle/variables/git-branch/output.txt | 2 ++ acceptance/bundle/variables/prepend-workspace-var/output.txt | 1 + acceptance/bundle/variables/resolve-builtin/output.txt | 1 + .../bundle/variables/resolve-vars-in-root-path/output.txt | 1 + acceptance/server_test.go | 1 + 5 files changed, 6 insertions(+) diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index d6d824394..fb3ab805a 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -29,6 +29,7 @@ "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, @@ -78,6 +79,7 @@ Validation OK! "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index 706d134ff..fcaa25b4a 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -54,6 +54,7 @@ "workspace": { "artifact_path": "/Users/$USERNAME/path/to/root/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt index f060c472e..0c1678f84 100644 --- a/acceptance/bundle/variables/resolve-builtin/output.txt +++ b/acceptance/bundle/variables/resolve-builtin/output.txt @@ -1,6 +1,7 @@ { "artifact_path": "TestResolveVariableReferences/bar/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt index c56fbe415..51eb40c91 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt @@ -1,6 +1,7 @@ { "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 98e351739..4957a7668 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -54,6 +54,7 @@ func AddHandlers(server *testserver.Server) { server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) { return iam.User{ + Id: "1000012345", UserName: "tester@databricks.com", }, nil }) From a03ea730112c7f51c8fc82d55ab8d11908ef8e9f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 30 Jan 2025 10:52:41 +0100 Subject: [PATCH 034/105] Add ruff.toml with increased line-length (#2268) The default is 88 which reformats too much. This has no effect on templates but affects Python script in this PR https://github.com/databricks/cli/pull/2267 For context, we do not set any line length for golang and have 177 .go files with max line length 150 or more. --- ruff.toml | 1 + 1 file changed, 1 insertion(+) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..802a3ca67 --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 150 From f1efbd7d9fb19c4f9457999e5ab8740293f34502 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 30 Jan 2025 11:38:54 +0100 Subject: [PATCH 035/105] acc: add -norepl flag that disables replacements (for debugging) (#2269) --- acceptance/acceptance_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 91ad09e9e..e7104a1c1 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -25,7 +25,10 @@ import ( "github.com/stretchr/testify/require" ) -var KeepTmp bool +var ( + KeepTmp bool + NoRepl bool +) // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. @@ -40,6 +43,7 @@ var InprocessMode bool func init() { flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)") flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run") + flag.BoolVar(&NoRepl, "norepl", false, "Do not apply any replacements (for debugging)") } const ( @@ -272,7 +276,9 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // Apply replacements to the new value only. // The reference value is stored after applying replacements. - valueNew = repls.Replace(valueNew) + if !NoRepl { + valueNew = repls.Replace(valueNew) + } // The test did not produce an expected output file. if okRef && !okNew { From 3c6eacb05b6980c59324b1da4ac091f22cb5733d Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:13:07 +0530 Subject: [PATCH 036/105] Add feature to mock server APIs in acceptance tests (#2226) ## Changes This PR allows us to define custom server stubs in a `test.toml` file. Note: A followup PR will add functionality to do assertions on the API request itself. ## Tests New acceptance test. --- acceptance/acceptance_test.go | 25 +++++++++++++++++---- acceptance/config_test.go | 23 +++++++++++++++++++ acceptance/workspace/jobs/create/output.txt | 5 +++++ acceptance/workspace/jobs/create/script | 1 + acceptance/workspace/jobs/create/test.toml | 7 ++++++ 5 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 acceptance/workspace/jobs/create/output.txt create mode 100644 acceptance/workspace/jobs/create/script create mode 100644 acceptance/workspace/jobs/create/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e7104a1c1..60f7945df 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -112,10 +113,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cloudEnv := os.Getenv("CLOUD_ENV") if cloudEnv == "" { - server := testserver.New(t) - AddHandlers(server) + defaultServer := testserver.New(t) + AddHandlers(defaultServer) // Redirect API access to local server: - t.Setenv("DATABRICKS_HOST", server.URL) + t.Setenv("DATABRICKS_HOST", defaultServer.URL) t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() @@ -214,6 +215,22 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont args := []string{"bash", "-euo", "pipefail", EntryPointScript} cmd := exec.Command(args[0], args[1:]...) + cmd.Env = os.Environ() + + // Start a new server with a custom configuration if the acceptance test + // specifies a custom server stubs. + if len(config.Server) > 0 { + server := testserver.New(t) + + for _, stub := range config.Server { + require.NotEmpty(t, stub.Pattern) + server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) { + return stub.Response.Body, nil + }) + } + cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL) + } + if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one // results in sporadic failures like this one (only if tests are running in parallel): @@ -221,7 +238,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--")) err := os.MkdirAll(coverDir, os.ModePerm) require.NoError(t, err) - cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir) + cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } // Write combined output to a file diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 41866c4a7..f340f0367 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -29,6 +29,29 @@ type TestConfig struct { // List of additional replacements to apply on this test. // Old is a regexp, New is a replacement expression. Repls []testdiff.Replacement + + // List of server stubs to load. Example configuration: + // + // [[Server]] + // Pattern = "POST /api/2.1/jobs/create" + // Response.Body = ''' + // { + // "job_id": 1111 + // } + // ''' + Server []ServerStub +} + +type ServerStub struct { + // The HTTP method and path to match. Examples: + // 1. /api/2.0/clusters/list (matches all methods) + // 2. GET /api/2.0/clusters/list + Pattern string + + // The response body to return. + Response struct { + Body string + } } // FindConfig finds the closest config file. diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt new file mode 100644 index 000000000..a9487fe5b --- /dev/null +++ b/acceptance/workspace/jobs/create/output.txt @@ -0,0 +1,5 @@ + +>>> $CLI jobs create --json {"name":"abc"} +{ + "job_id":1111 +} diff --git a/acceptance/workspace/jobs/create/script b/acceptance/workspace/jobs/create/script new file mode 100644 index 000000000..9ff7b5b87 --- /dev/null +++ b/acceptance/workspace/jobs/create/script @@ -0,0 +1 @@ +trace $CLI jobs create --json '{"name":"abc"}' diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml new file mode 100644 index 000000000..94e5eee13 --- /dev/null +++ b/acceptance/workspace/jobs/create/test.toml @@ -0,0 +1,7 @@ +[[Server]] +Pattern = "POST /api/2.1/jobs/create" +Response.Body = ''' +{ + "job_id": 1111 +} +''' From 787dbe909912ab5c4894cb21218cb39d6626e95a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:01:23 +0530 Subject: [PATCH 037/105] Add request body assertions to acceptance tests (#2263) ## Changes With this PR, any acceptance tests that define custom server stubs in `test.toml` will automatically record all HTTP requests made and assert on them. Builds on top of https://github.com/databricks/cli/pull/2226 ## Tests Modifying existing acceptance test. --- acceptance/acceptance_test.go | 37 ++++++++++++++++++- acceptance/config_test.go | 4 ++ .../workspace/jobs/create/out.requests.txt | 1 + acceptance/workspace/jobs/create/test.toml | 2 + libs/testserver/server.go | 25 +++++++++++++ 5 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 acceptance/workspace/jobs/create/out.requests.txt diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 60f7945df..bb1d0f44f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -2,6 +2,7 @@ package acceptance_test import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -219,8 +220,21 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // Start a new server with a custom configuration if the acceptance test // specifies a custom server stubs. - if len(config.Server) > 0 { - server := testserver.New(t) + var server *testserver.Server + + // Start a new server for this test if either: + // 1. A custom server spec is defined in the test configuration. + // 2. The test is configured to record requests and assert on them. We need + // a duplicate of the default server to record requests because the default + // server otherwise is a shared resource. + if len(config.Server) > 0 || config.RecordRequests { + server = testserver.New(t) + server.RecordRequests = config.RecordRequests + + // If no custom server stubs are defined, add the default handlers. + if len(config.Server) == 0 { + AddHandlers(server) + } for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) @@ -249,6 +263,25 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Dir = tmpDir err = cmd.Run() + // Write the requests made to the server to a output file if the test is + // configured to record requests. + if config.RecordRequests { + f, err := os.OpenFile(filepath.Join(tmpDir, "out.requests.txt"), os.O_CREATE|os.O_WRONLY, 0o644) + require.NoError(t, err) + + for _, req := range server.Requests { + reqJson, err := json.Marshal(req) + require.NoError(t, err) + + line := fmt.Sprintf("%s\n", reqJson) + _, err = f.WriteString(line) + require.NoError(t, err) + } + + err = f.Close() + require.NoError(t, err) + } + // Include exit code in output (if non-zero) formatOutput(out, err) require.NoError(t, out.Close()) diff --git a/acceptance/config_test.go b/acceptance/config_test.go index f340f0367..beceb6a08 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -40,6 +40,10 @@ type TestConfig struct { // } // ''' Server []ServerStub + + // Record the requests made to the server and write them as output to + // out.requests.txt + RecordRequests bool } type ServerStub struct { diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt new file mode 100644 index 000000000..b22876b70 --- /dev/null +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -0,0 +1 @@ +{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index 94e5eee13..1bf36547b 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,3 +1,5 @@ +RecordRequests = true + [[Server]] Pattern = "POST /api/2.1/jobs/create" Response.Body = ''' diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 9ebfe3ba0..2e8dbdfda 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -2,9 +2,12 @@ package testserver import ( "encoding/json" + "io" "net/http" "net/http/httptest" + "github.com/stretchr/testify/assert" + "github.com/databricks/cli/internal/testutil" ) @@ -13,6 +16,16 @@ type Server struct { Mux *http.ServeMux t testutil.TestingT + + RecordRequests bool + + Requests []Request +} + +type Request struct { + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -37,6 +50,18 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { return } + if s.RecordRequests { + body, err := io.ReadAll(r.Body) + assert.NoError(s.t, err) + + s.Requests = append(s.Requests, Request{ + Method: r.Method, + Path: r.URL.Path, + Body: json.RawMessage(body), + }) + + } + w.Header().Set("Content-Type", "application/json") var respBytes []byte From e5730bf57ecb39049047ba394734831589a81bbb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 31 Jan 2025 14:53:13 +0100 Subject: [PATCH 038/105] Use real terraform in acceptance tests (#2267) ## Changes - Add a script install_terraform.py that downloads terraform and provider and generates a config to use, inspired by https://gist.github.com/pietern/1cb6b6f3e0a452328e13cdc75031105e - Make acceptance tests run this script once before running the tests and set the required env vars to make cli use this terraform installation. - Use OS-specific directory for things that are build by acceptance test runner (CLI and terraform). This enables acceptance tests against cloud #2242 and local test for bundle deploy #2254. ## Tests - Add an acceptance test for standalone terraform. This is useful to debug terraform with TF_LOG=DEBUG to see that it uses local provider. - Other acceptance tests are updated with regard to terraform exec path. - The overall time for tests locally is unchanged (if terraform is already fetched). --- acceptance/.gitignore | 1 + acceptance/acceptance_test.go | 61 ++++++--- acceptance/build/.gitignore | 1 - .../bundle/variables/git-branch/output.txt | 4 +- .../prepend-workspace-var/output.txt | 2 +- acceptance/install_terraform.py | 122 ++++++++++++++++++ acceptance/terraform/main.tf | 25 ++++ acceptance/terraform/output.txt | 51 ++++++++ acceptance/terraform/script | 14 ++ 9 files changed, 256 insertions(+), 25 deletions(-) create mode 100644 acceptance/.gitignore delete mode 100644 acceptance/build/.gitignore create mode 100755 acceptance/install_terraform.py create mode 100644 acceptance/terraform/main.tf create mode 100644 acceptance/terraform/output.txt create mode 100644 acceptance/terraform/script diff --git a/acceptance/.gitignore b/acceptance/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/acceptance/.gitignore @@ -0,0 +1 @@ +build diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bb1d0f44f..5f0030eec 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -77,6 +77,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cwd, err := os.Getwd() require.NoError(t, err) + buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + + // Download terraform and provider and create config; this also creates build directory. + RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".") + coverDir := os.Getenv("CLI_GOCOVERDIR") if coverDir != "" { @@ -93,7 +98,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { t.Setenv("CMD_SERVER_URL", cmdServer.URL) execPath = filepath.Join(cwd, "bin", "callserver.py") } else { - execPath = BuildCLI(t, cwd, coverDir) + execPath = BuildCLI(t, buildDir, coverDir) } t.Setenv("CLI", execPath) @@ -123,11 +128,24 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { homeDir := t.TempDir() // Do not read user's ~/.databrickscfg t.Setenv(env.HomeEnvVar(), homeDir) - - // Prevent CLI from downloading terraform in each test: - t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) } + terraformrcPath := filepath.Join(buildDir, ".terraformrc") + t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) + t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) + repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE") + + terraformExecPath := filepath.Join(buildDir, "terraform") + if runtime.GOOS == "windows" { + terraformExecPath += ".exe" + } + t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) + t.Setenv("TERRAFORM", terraformExecPath) + repls.SetPath(terraformExecPath, "$TERRAFORM") + + // do it last so that full paths match first: + repls.SetPath(buildDir, "$BUILD_DIR") + workspaceClient, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -406,13 +424,12 @@ func readMergedScriptContents(t *testing.T, dir string) string { return strings.Join(prepares, "\n") } -func BuildCLI(t *testing.T, cwd, coverDir string) string { - execPath := filepath.Join(cwd, "build", "databricks") +func BuildCLI(t *testing.T, buildDir, coverDir string) string { + execPath := filepath.Join(buildDir, "databricks") if runtime.GOOS == "windows" { execPath += ".exe" } - start := time.Now() args := []string{ "go", "build", "-mod", "vendor", @@ -430,20 +447,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string { args = append(args, "-buildvcs=false") } - cmd := exec.Command(args[0], args[1:]...) - cmd.Dir = ".." - out, err := cmd.CombinedOutput() - elapsed := time.Since(start) - t.Logf("%s took %s", args, elapsed) - require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out) - if len(out) > 0 { - t.Logf("go build output: %s: %s", args, out) - } - - // Quick check + warm up cache: - cmd = exec.Command(execPath, "--version") - out, err = cmd.CombinedOutput() - require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out) + RunCommand(t, args, "..") + RunCommand(t, []string{execPath, "--version"}, ".") return execPath } @@ -581,3 +586,17 @@ func getUVDefaultCacheDir(t *testing.T) string { return cacheDir + "/uv" } } + +func RunCommand(t *testing.T, args []string, dir string) { + start := time.Now() + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + elapsed := time.Since(start) + t.Logf("%s took %s", args, elapsed) + + require.NoError(t, err, "%s failed: %s\n%s", args, err, out) + if len(out) > 0 { + t.Logf("%s output: %s", args, out) + } +} diff --git a/acceptance/build/.gitignore b/acceptance/build/.gitignore deleted file mode 100644 index a48b4db25..000000000 --- a/acceptance/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -databricks diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index fb3ab805a..5e7664f61 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -11,7 +11,7 @@ "name": "git", "target": "prod", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { @@ -61,7 +61,7 @@ Validation OK! "name": "git", "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index fcaa25b4a..ed6c2b2af 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -7,7 +7,7 @@ }, "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "resources": { diff --git a/acceptance/install_terraform.py b/acceptance/install_terraform.py new file mode 100755 index 000000000..4cf6a9729 --- /dev/null +++ b/acceptance/install_terraform.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +Script to set up terraform and databricks terraform provider in a local directory: + +- Download terraform. +- Download databricks provider. +- Write a .terraformrc config file that uses this directory. +- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider. +""" + +import os +import platform +import zipfile +import argparse +import json +from pathlib import Path +from urllib.request import urlretrieve + +os_name = platform.system().lower() + +arch = platform.machine().lower() +arch = {"x86_64": "amd64"}.get(arch, arch) +if os_name == "windows" and arch not in ("386", "amd64"): + # terraform 1.5.5 only has builds for these two. + arch = "amd64" + +terraform_version = "1.5.5" +terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip" +terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}" +terraform_binary = "terraform.exe" if os_name == "windows" else "terraform" + + +def retrieve(url, path): + if not path.exists(): + print(f"Downloading {url} -> {path}") + urlretrieve(url, path) + + +def read_version(path): + for line in path.open(): + if "ProviderVersion" in line: + # Expecting 'const ProviderVersion = "1.64.1"' + items = line.strip().split() + assert len(items) >= 3, items + assert items[-3:-1] == ["ProviderVersion", "="], items + version = items[-1].strip('"') + assert version, items + return version + raise SystemExit(f"Could not find ProviderVersion in {path}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--targetdir", default="build", type=Path) + parser.add_argument("--provider-version") + args = parser.parse_args() + target = args.targetdir + + if not args.provider_version: + version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go" + assert version_file.exists(), version_file + terraform_provider_version = read_version(version_file) + print(f"Read version {terraform_provider_version} from {version_file}") + else: + terraform_provider_version = args.provider_version + + terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip" + terraform_provider_url = ( + f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}" + ) + + target.mkdir(exist_ok=True, parents=True) + + zip_path = target / terraform_file + terraform_path = target / terraform_binary + terraform_provider_path = target / terraform_provider_file + + retrieve(terraform_url, zip_path) + retrieve(terraform_provider_url, terraform_provider_path) + + if not terraform_path.exists(): + print(f"Extracting {zip_path} -> {terraform_path}") + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(target) + + terraform_path.chmod(0o755) + + tfplugins_path = target / "tfplugins" + provider_dir = Path(tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}") + if not provider_dir.exists(): + print(f"Extracting {terraform_provider_path} -> {provider_dir}") + os.makedirs(provider_dir, exist_ok=True) + with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref: + zip_ref.extractall(provider_dir) + + files = list(provider_dir.iterdir()) + assert files, provider_dir + + for f in files: + f.chmod(0o755) + + terraformrc_path = target / ".terraformrc" + if not terraformrc_path.exists(): + path = json.dumps(str(tfplugins_path.absolute())) + text = f"""# Set these env variables before running databricks cli: +# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()} +# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()} + +provider_installation {{ + filesystem_mirror {{ + path = {path} + include = ["registry.terraform.io/databricks/databricks"] + }} +}} +""" + print(f"Writing {terraformrc_path}:\n{text}") + terraformrc_path.write_text(text) + + +if __name__ == "__main__": + main() diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf new file mode 100644 index 000000000..93f665ff4 --- /dev/null +++ b/acceptance/terraform/main.tf @@ -0,0 +1,25 @@ +terraform { + required_providers { + databricks = { + source = "databricks/databricks" + version = "1.64.1" + } + } + + required_version = "= 1.5.5" +} + +provider "databricks" { + # Optionally, specify the Databricks host and token + # host = "https://" + # token = "" +} + +data "databricks_current_user" "me" { + # Retrieves the current user's information +} + +output "username" { + description = "Username" + value = "${data.databricks_current_user.me.user_name}" +} diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt new file mode 100644 index 000000000..c3d453ea5 --- /dev/null +++ b/acceptance/terraform/output.txt @@ -0,0 +1,51 @@ + +>>> $TERRAFORM init -no-color -get=false + +Initializing the backend... + +Initializing provider plugins... +- Finding databricks/databricks versions matching "1.64.1"... +- Installing databricks/databricks v1.64.1... +- Installed databricks/databricks v1.64.1 (unauthenticated) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + + +Warning: Incomplete lock file information for providers + +Due to your customized provider installation methods, Terraform was forced to +calculate lock file checksums locally for the following providers: + - databricks/databricks + + +To calculate additional checksums for another platform, run: + terraform providers lock -platform=linux_amd64 +(where linux_amd64 is the platform to generate) + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. + +>>> $TERRAFORM plan -no-color +data.databricks_current_user.me: Reading... +data.databricks_current_user.me: Read complete after 0s [id=$USER.Id] + +Changes to Outputs: + + username = "$USERNAME" + +You can apply this plan to save these new output values to the Terraform +state, without changing any real infrastructure. + +───────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so Terraform can't +guarantee to take exactly these actions if you run "terraform apply" now. diff --git a/acceptance/terraform/script b/acceptance/terraform/script new file mode 100644 index 000000000..78e35049d --- /dev/null +++ b/acceptance/terraform/script @@ -0,0 +1,14 @@ +# Want to filter out these message: +# Mac: +# The current .terraform.lock.hcl file only includes checksums for +# darwin_arm64, so Terraform running on another platform will fail to install +# these providers. +# +# Linux: +# The current .terraform.lock.hcl file only includes checksums for linux_amd64, +# so Terraform running on another platform will fail to install these +# providers. + +trace $TERRAFORM init -no-color -get=false | grep -v 'includes checksums for' | grep -v 'so Terraform running on another' | grep -v 'providers\.' +trace $TERRAFORM plan -no-color +rm -fr .terraform.lock.hcl .terraform From 2f798c4dedd7e4fc18d9f31cbd55135345e1b208 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 3 Feb 2025 11:03:18 +0100 Subject: [PATCH 039/105] acc: Remove initial '$CLI --version' call (#2280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is proven to be not necessary. ``` ~/work/cli/acceptance % hyperfine -w 2 'go test' # with change: Benchmark 1: go test Time (mean ± σ): 4.983 s ± 0.209 s [User: 6.073 s, System: 9.869 s] Range (min … max): 4.792 s … 5.483 s 10 runs ~/work/cli/acceptance % git stash # without change: ~/work/cli/acceptance % hyperfine -w 2 'go test' Benchmark 1: go test Time (mean ± σ): 5.018 s ± 0.100 s [User: 6.142 s, System: 10.234 s] Range (min … max): 4.899 s … 5.182 s 10 runs ``` --- acceptance/acceptance_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 5f0030eec..571168ca8 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -448,7 +448,6 @@ func BuildCLI(t *testing.T, buildDir, coverDir string) string { } RunCommand(t, args, "..") - RunCommand(t, []string{execPath, "--version"}, ".") return execPath } From fcedfe4c781a1f6de8f23d6e477ed045d91921ea Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 3 Feb 2025 11:29:13 +0100 Subject: [PATCH 040/105] acc: Consistent & detailed output for file issues (#2279) ## Changes - Include compact relPath in the error message title. Include full paths in separate lines below. - Previously sometimes full paths were printed, sometime only rel path. ## Tests Manually trigger the errors. --- acceptance/acceptance_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 571168ca8..cad8acf4f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -335,7 +335,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN bufRef, okRef := tryReading(t, pathRef) bufNew, okNew := tryReading(t, pathNew) if !okRef && !okNew { - t.Errorf("Both files are missing or have errors: %s, %s", pathRef, pathNew) + t.Errorf("Both files are missing or have errors: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew) return } @@ -350,7 +350,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // The test did not produce an expected output file. if okRef && !okNew { - t.Errorf("Missing output file: %s", relPath) + t.Errorf("Missing output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew) testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew) if testdiff.OverwriteMode { t.Logf("Removing output file: %s", relPath) @@ -361,7 +361,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // The test produced an unexpected output file. if !okRef && okNew { - t.Errorf("Unexpected output file: %s", relPath) + t.Errorf("Unexpected output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew) testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew) if testdiff.OverwriteMode { t.Logf("Writing output file: %s", relPath) From f267318bb940849c8833308128ff9eaf08c35aa6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 3 Feb 2025 11:43:25 +0100 Subject: [PATCH 041/105] Include acceptance tests in integration tests (#2242) ## Changes - Include acceptance directory in integration tests. Acceptance tests will not start local server if CLOUD_ENV is set, so they become integration tests. - Add dependency for vendor to integration, so that CLI can be build there. - Implement LocalOnly option in test.toml to opt out of running acceptance tests as integration tests. Use it in certain tests that are difficult or not necessary to fix when run as integration tests. - Update terraform test to redact timings out. - Clean up .workspace.current_user from outputs of the tests. ## Tests Existing tests. --- Makefile | 6 +++--- acceptance/acceptance_test.go | 5 +++++ acceptance/bundle/scripts/test.toml | 1 + acceptance/bundle/templates/test.toml | 2 ++ .../bundle/variables/env_overrides/test.toml | 2 ++ acceptance/bundle/variables/git-branch/output.txt | 14 ++------------ acceptance/bundle/variables/git-branch/script | 6 +++--- .../variables/prepend-workspace-var/output.txt | 7 +------ .../bundle/variables/prepend-workspace-var/script | 2 +- .../bundle/variables/resolve-builtin/output.txt | 5 ----- acceptance/bundle/variables/resolve-builtin/script | 2 +- .../bundle/variables/resolve-builtin/test.toml | 2 ++ .../variables/resolve-vars-in-root-path/output.txt | 5 ----- .../variables/resolve-vars-in-root-path/script | 2 +- .../variables/resolve-vars-in-root-path/test.toml | 2 ++ acceptance/config_test.go | 3 +++ acceptance/terraform/output.txt | 2 +- acceptance/terraform/test.toml | 3 +++ acceptance/workspace/jobs/create/test.toml | 1 + 19 files changed, 34 insertions(+), 38 deletions(-) create mode 100644 acceptance/bundle/scripts/test.toml create mode 100644 acceptance/bundle/templates/test.toml create mode 100644 acceptance/bundle/variables/env_overrides/test.toml create mode 100644 acceptance/bundle/variables/resolve-builtin/test.toml create mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/test.toml create mode 100644 acceptance/terraform/test.toml diff --git a/Makefile b/Makefile index d30ccef14..e18727934 100644 --- a/Makefile +++ b/Makefile @@ -51,12 +51,12 @@ schema: docs: go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen -INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h +INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./acceptance ./integration/..." -- -parallel 4 -timeout=2h -integration: +integration: vendor $(INTEGRATION) -integration-short: +integration-short: vendor $(INTEGRATION) -short .PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index cad8acf4f..ccdf74bcb 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -209,6 +209,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont t.Skipf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath) } + cloudEnv := os.Getenv("CLOUD_ENV") + if config.LocalOnly && cloudEnv != "" { + t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv) + } + var tmpDir string var err error if KeepTmp { diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml new file mode 100644 index 000000000..1dbd78681 --- /dev/null +++ b/acceptance/bundle/scripts/test.toml @@ -0,0 +1 @@ +LocalOnly = true # Deployment currently fails when run locally; once that is fixed, remove this setting diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml new file mode 100644 index 000000000..90539263d --- /dev/null +++ b/acceptance/bundle/templates/test.toml @@ -0,0 +1,2 @@ +# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +LocalOnly = true diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml new file mode 100644 index 000000000..439c2fab1 --- /dev/null +++ b/acceptance/bundle/variables/env_overrides/test.toml @@ -0,0 +1,2 @@ +# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist +LocalOnly = true diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index 5e7664f61..21ed9e7de 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -6,7 +6,7 @@ "git": { "actual_branch": "main", "branch": "", - "bundle_root_path": ".", + "bundle_root_path": "." }, "name": "git", "target": "prod", @@ -28,11 +28,6 @@ }, "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts", - "current_user": { - "id": "$USER.Id", - "short_name": "$USERNAME", - "userName": "$USERNAME" - }, "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files", "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources", "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod", @@ -56,7 +51,7 @@ Validation OK! "git": { "actual_branch": "main", "branch": "dev-branch", - "bundle_root_path": ".", + "bundle_root_path": "." }, "name": "git", "target": "dev", @@ -78,11 +73,6 @@ Validation OK! }, "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts", - "current_user": { - "id": "$USER.Id", - "short_name": "$USERNAME", - "userName": "$USERNAME" - }, "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files", "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources", "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev", diff --git a/acceptance/bundle/variables/git-branch/script b/acceptance/bundle/variables/git-branch/script index aed881f1f..8f99cc01b 100644 --- a/acceptance/bundle/variables/git-branch/script +++ b/acceptance/bundle/variables/git-branch/script @@ -1,6 +1,6 @@ git-repo-init -trace $CLI bundle validate -o json | grep -v '"commit"' +trace $CLI bundle validate -o json | jq 'del(.workspace.current_user, .bundle.git.commit)' trace $CLI bundle validate -trace $CLI bundle validate -o json -t dev | grep -v '"commit"' -trace $CLI bundle validate -t dev | grep -v '"commit"' +trace $CLI bundle validate -o json -t dev | jq 'del(.workspace.current_user, .bundle.git.commit)' +trace $CLI bundle validate -t dev rm -fr .git diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index ed6c2b2af..93b652894 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -53,15 +53,10 @@ }, "workspace": { "artifact_path": "/Users/$USERNAME/path/to/root/artifacts", - "current_user": { - "id": "$USER.Id", - "short_name": "$USERNAME", - "userName": "$USERNAME" - }, "file_path": "/Users/$USERNAME/path/to/root/files", "profile": "profile_name", "resource_path": "/Users/$USERNAME/path/to/root/resources", "root_path": "/Users/$USERNAME/path/to/root", "state_path": "/Users/$USERNAME/path/to/root/state" } -} \ No newline at end of file +} diff --git a/acceptance/bundle/variables/prepend-workspace-var/script b/acceptance/bundle/variables/prepend-workspace-var/script index de6bc8a17..e30ffb9c4 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/script +++ b/acceptance/bundle/variables/prepend-workspace-var/script @@ -1,2 +1,2 @@ echo /Workspace should be prepended on all paths, but it is not the case: #2181 -$CLI bundle validate -o json +$CLI bundle validate -o json | jq 'del(.workspace.current_user)' diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt index 0c1678f84..f37a2a19e 100644 --- a/acceptance/bundle/variables/resolve-builtin/output.txt +++ b/acceptance/bundle/variables/resolve-builtin/output.txt @@ -1,10 +1,5 @@ { "artifact_path": "TestResolveVariableReferences/bar/artifacts", - "current_user": { - "id": "$USER.Id", - "short_name": "$USERNAME", - "userName": "$USERNAME" - }, "file_path": "TestResolveVariableReferences/bar/baz", "resource_path": "TestResolveVariableReferences/bar/resources", "root_path": "TestResolveVariableReferences/bar", diff --git a/acceptance/bundle/variables/resolve-builtin/script b/acceptance/bundle/variables/resolve-builtin/script index fefd9abe6..558d0a7ca 100644 --- a/acceptance/bundle/variables/resolve-builtin/script +++ b/acceptance/bundle/variables/resolve-builtin/script @@ -1 +1 @@ -$CLI bundle validate -o json | jq .workspace +$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)' diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml new file mode 100644 index 000000000..085fab6c0 --- /dev/null +++ b/acceptance/bundle/variables/resolve-builtin/test.toml @@ -0,0 +1,2 @@ +# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/' +LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt index 51eb40c91..fb828d826 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt @@ -1,10 +1,5 @@ { "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts", - "current_user": { - "id": "$USER.Id", - "short_name": "$USERNAME", - "userName": "$USERNAME" - }, "file_path": "TestResolveVariableReferencesToBundleVariables/bar/files", "resource_path": "TestResolveVariableReferencesToBundleVariables/bar/resources", "root_path": "TestResolveVariableReferencesToBundleVariables/bar", diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/script b/acceptance/bundle/variables/resolve-vars-in-root-path/script index fefd9abe6..558d0a7ca 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/script +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/script @@ -1 +1 @@ -$CLI bundle validate -o json | jq .workspace +$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)' diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml new file mode 100644 index 000000000..d833bd848 --- /dev/null +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml @@ -0,0 +1,2 @@ +# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/' +LocalOnly = true diff --git a/acceptance/config_test.go b/acceptance/config_test.go index beceb6a08..c7be223de 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -26,6 +26,9 @@ type TestConfig struct { // If absent, default to true. GOOS map[string]bool + // If true, do not run this test against cloud environment + LocalOnly bool + // List of additional replacements to apply on this test. // Old is a regexp, New is a replacement expression. Repls []testdiff.Replacement diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt index c3d453ea5..32589ddab 100644 --- a/acceptance/terraform/output.txt +++ b/acceptance/terraform/output.txt @@ -37,7 +37,7 @@ commands will detect it and remind you to do so if necessary. >>> $TERRAFORM plan -no-color data.databricks_current_user.me: Reading... -data.databricks_current_user.me: Read complete after 0s [id=$USER.Id] +data.databricks_current_user.me: Read complete after (redacted) [id=$USER.Id] Changes to Outputs: + username = "$USERNAME" diff --git a/acceptance/terraform/test.toml b/acceptance/terraform/test.toml new file mode 100644 index 000000000..a6849e30f --- /dev/null +++ b/acceptance/terraform/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = 'Read complete after [^\s]+' +New = 'Read complete after (redacted)' diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index 1bf36547b..e69569c18 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,3 +1,4 @@ +LocalOnly = true # request recording currently does not work with cloud environment RecordRequests = true [[Server]] From 91e04cc4441d011a568eaf6b064b85ff6c190d6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:36:30 +0100 Subject: [PATCH 042/105] Bump golangci/golangci-lint-action from 6.1.1 to 6.2.0 (#2273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.1.1 to 6.2.0.
Release notes

Sourced from golangci/golangci-lint-action's releases.

v6.2.0

What's Changed

Changes

Documentation

Dependencies

New Contributors

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v6.1.1...v6.2.0

Commits
  • ec5d184 feat: support linux arm64 public preview (#1144)
  • a0297a1 build(deps-dev): bump the dev-dependencies group with 3 updates (#1143)
  • 58eda26 build(deps): bump @​types/node from 22.10.2 to 22.10.5 in the dependencies gro...
  • 44c2434 build(deps-dev): bump the dev-dependencies group with 2 updates (#1141)
  • 2f13b80 build(deps-dev): bump the dev-dependencies group with 2 updates (#1139)
  • 1ac3686 build(deps-dev): bump the dev-dependencies group with 2 updates (#1138)
  • 9937fdf build(deps): bump @​types/node from 22.10.1 to 22.10.2 in the dependencies gro...
  • cb60b26 build(deps-dev): bump the dev-dependencies group with 2 updates (#1136)
  • 774c35b build(deps): bump @​actions/cache from 3.3.0 to 4.0.0 in the dependencies grou...
  • 7ce5487 build(deps-dev): bump the dev-dependencies group with 3 updates (#1134)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golangci/golangci-lint-action&package-manager=github_actions&previous-version=6.1.1&new-version=6.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2a8a68862..5921b4e5f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -95,7 +95,7 @@ jobs: # Exit with status code 1 if there are differences (i.e. unformatted files) git diff --exit-code - name: golangci-lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 with: version: v1.63.4 args: --timeout=15m From 75932198f72a2e0e658d36148b539ef545e8d828 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:36:42 +0100 Subject: [PATCH 043/105] Bump astral-sh/ruff-action from 3.0.1 to 3.1.0 (#2274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [astral-sh/ruff-action](https://github.com/astral-sh/ruff-action) from 3.0.1 to 3.1.0.
Release notes

Sourced from astral-sh/ruff-action's releases.

v3.1.0 🌈 Determine ruff version from optional or dependency groups

Changes

Big thank you to @​AA-Turner for expanding the pyproject.toml parsing to also find the ruff version to use in the following scenarios:

[dependency-groups]
dev = [
    { include-group = "docs" },
    { include-group = "lint" },
]
docs = [
    "sphinx",
]
lint = [
    "ruff==0.8.3",
]

[project.optional-dependencies]
lint = [
    "ruff==0.8.3",
]

🚀 Enhancements

  • Read the [project.optional-dependencies] and [dependency-groups] tables @​AA-Turner (#66)

v3.0.2 🌈 Full support for GHES

Changes

This release fixes some issues that prevented use with GitHub Enterprise Server instances. Parsing the ruff version from pyproject.toml now also uses a library that is fully TOML 1.0.0 compliant.

🐛 Bug fixes

🧰 Maintenance

📚 Documentation

... (truncated)

Commits
  • f14634c Read the [project.optional-dependencies] and [dependency-groups] tables (...
  • 47de3de Bump @​types/node from 22.10.10 to 22.12.0 (#60)
  • d8281c7 Do not expect GITHUB_TOKEN to be set or valid (#65)
  • a634044 Bump eifinger/actionlint-action from 1.9.0 to 1.9.1 (#59)
  • 2993ff4 Fix compiled known versions (#62)
  • 20a3b17 chore: update known checksums for 0.9.3 (#61)
  • 1c1aef9 Bump typescript from 5.7.2 to 5.7.3 (#41)
  • 0ceb04d Bump release-drafter/release-drafter from 6.0.0 to 6.1.0 (#50)
  • 18db80c Bump @​types/node from 22.10.5 to 22.10.10 (#53)
  • 0a5dfb8 Fix Markdown link to Install the latest version (#58)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=astral-sh/ruff-action&package-manager=github_actions&previous-version=3.0.1&new-version=3.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5921b4e5f..f27459baa 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -100,7 +100,7 @@ jobs: version: v1.63.4 args: --timeout=15m - name: Run ruff - uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1 + uses: astral-sh/ruff-action@f14634c415d3e63ffd4d550a22f037df4c734a60 # v3.1.0 with: version: "0.9.1" args: "format --check" From 4f3a289333094951fe7b55b00a7f3bc70cd138eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:36:50 +0100 Subject: [PATCH 044/105] Bump actions/stale from 9.0.0 to 9.1.0 (#2275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/stale](https://github.com/actions/stale) from 9.0.0 to 9.1.0.
Release notes

Sourced from actions/stale's releases.

v9.1.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/stale/compare/v9...v9.1.0

Commits
  • 5bef64f build(deps): bump @​actions/cache from 3.2.2 to 4.0.0 (#1194)
  • fa77dfd build(deps-dev): bump @​types/jest from 29.5.11 to 29.5.14 (#1193)
  • f04443d build(deps): bump @​actions/core from 1.10.1 to 1.11.1 (#1191)
  • 5c715b0 build(deps-dev): bump ts-jest from 29.1.1 to 29.2.5 (#1175)
  • f691222 build(deps): bump actions/publish-action from 0.2.2 to 0.3.0 (#1147)
  • df990c2 build(deps): bump actions/checkout from 3 to 4 (#1091)
  • 6e472ce Merge pull request #1179 from actions/Jcambass-patch-1
  • d10ba64 Merge pull request #1150 from actions/dependabot/npm_and_yarn/undici-5.28.4
  • bbf3da5 resolve check failures
  • 6a2e61d Add workflow file for publishing releases to immutable action package
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/stale&package-manager=github_actions&previous-version=9.0.0&new-version=9.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/close-stale-issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml index ea9558caf..fc764fb0d 100644 --- a/.github/workflows/close-stale-issues.yml +++ b/.github/workflows/close-stale-issues.yml @@ -18,7 +18,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0 + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled. stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled. From 75db82ae1f5f3fc12716cb7441d0374121dcf6b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:37:17 +0100 Subject: [PATCH 045/105] Bump actions/create-github-app-token from 1.11.1 to 1.11.2 (#2276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/create-github-app-token](https://github.com/actions/create-github-app-token) from 1.11.1 to 1.11.2.
Release notes

Sourced from actions/create-github-app-token's releases.

v1.11.2

1.11.2 (2025-01-30)

Bug Fixes

Commits
  • 136412a build(release): 1.11.2 [skip ci]
  • b4192a5 fix(deps): bump @​octokit/request from 9.1.3 to 9.1.4 in the production-depend...
  • 29aa051 fix(deps): bump undici from 6.19.8 to 7.2.0 (#198)
  • a5f8600 build(deps-dev): bump @​sinonjs/fake-timers from 13.0.2 to 14.0.0 (#199)
  • 0edddd7 build(deps-dev): bump the development-dependencies group with 2 updates (#197)
  • bb3ca76 docs(README): remove extra space in variable syntax in README example (#201)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/create-github-app-token&package-manager=github_actions&previous-version=1.11.1&new-version=1.11.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/integration-main.yml | 2 +- .github/workflows/integration-pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index 84dd7263a..f737c48e6 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Generate GitHub App Token id: generate-token - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 + uses: actions/create-github-app-token@136412a57a7081aa63c935a2cc2918f76c34f514 # v1.11.2 with: app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }} private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }} diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index 7a62113cd..bf096c863 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Generate GitHub App Token id: generate-token - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 + uses: actions/create-github-app-token@136412a57a7081aa63c935a2cc2918f76c34f514 # v1.11.2 with: app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }} private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }} From 838de2fde23b2234843671ff38f021d1356edf67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:38:16 +0100 Subject: [PATCH 046/105] Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0 (#2237) Bumps [github.com/hashicorp/terraform-exec](https://github.com/hashicorp/terraform-exec) from 0.21.0 to 0.22.0.
Release notes

Sourced from github.com/hashicorp/terraform-exec's releases.

v0.22.0

ENHANCEMENTS:

  • tfexec: Add support for terraform init --json via InitJSON (#478)

INTERNAL:

  • go: Require Go 1.22 (previously 1.18) (#499)
Changelog

Sourced from github.com/hashicorp/terraform-exec's changelog.

0.22.0 (January 21, 2025)

ENHANCEMENTS:

  • tfexec: Add support for terraform init --json via InitJSON (#478)

INTERNAL:

  • go: Require Go 1.22 (previously 1.18) (#499)
Commits
  • 6801a6e v0.22.0 [skip ci]
  • dd2bc9a Update CHANGELOG.md (#501)
  • b5e5740 build(deps): bump github.com/hashicorp/hc-install from 0.8.0 to 0.9.1 (#494)
  • abfb5ba tfexec: add InitJSON (#478)
  • 840ecad ci/e2etests: Add latest major Terraform versions (#498)
  • 4497f9e go: Require Go 1.22 (previously 1.18) (#499)
  • b13b10b build(deps): bump github.com/zclconf/go-cty from 1.16.0 to 1.16.1 (#496)
  • 6b0d5eb build(deps): bump github.com/zclconf/go-cty from 1.15.1 to 1.16.0 (#495)
  • ef0b6c3 build(deps): Bump workflows to latest trusted versions (#493)
  • c75d998 build(deps): bump github.com/hashicorp/terraform-json from 0.23.0 to 0.24.0 (...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/terraform-exec&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bd8997190..151133944 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 github.com/hashicorp/hc-install v0.9.1 // MPL 2.0 - github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 - github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0 + github.com/hashicorp/terraform-exec v0.22.0 // MPL 2.0 + github.com/hashicorp/terraform-json v0.24.0 // MPL 2.0 github.com/hexops/gotextdiff v1.0.3 // BSD 3-Clause "New" or "Revised" License github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause github.com/mattn/go-isatty v0.0.20 // MIT @@ -62,7 +62,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/zclconf/go-cty v1.15.0 // indirect + github.com/zclconf/go-cty v1.16.1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect diff --git a/go.sum b/go.sum index dec1d40b2..3c7f20937 100644 --- a/go.sum +++ b/go.sum @@ -107,10 +107,10 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hc-install v0.9.1 h1:gkqTfE3vVbafGQo6VZXcy2v5yoz2bE0+nhZXruCuODQ= github.com/hashicorp/hc-install v0.9.1/go.mod h1:pWWvN/IrfeBK4XPeXXYkL6EjMufHkCK5DvwxeLKuBf0= -github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= -github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= -github.com/hashicorp/terraform-json v0.23.0 h1:sniCkExU4iKtTADReHzACkk8fnpQXrdD2xoR+lppBkI= -github.com/hashicorp/terraform-json v0.23.0/go.mod h1:MHdXbBAbSg0GvzuWazEGKAn/cyNfIB7mN6y7KJN6y2c= +github.com/hashicorp/terraform-exec v0.22.0 h1:G5+4Sz6jYZfRYUCg6eQgDsqTzkNXV+fP8l+uRmZHj64= +github.com/hashicorp/terraform-exec v0.22.0/go.mod h1:bjVbsncaeh8jVdhttWYZuBGj21FcYw6Ia/XfHcNO7lQ= +github.com/hashicorp/terraform-json v0.24.0 h1:rUiyF+x1kYawXeRth6fKFm/MdfBS6+lW4NbeATsYz8Q= +github.com/hashicorp/terraform-json v0.24.0/go.mod h1:Nfj5ubo9xbu9uiAoZVBsNOjvNKB66Oyrvtit74kC7ow= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -174,8 +174,8 @@ github.com/wI2L/jsondiff v0.6.1 h1:ISZb9oNWbP64LHnu4AUhsMF5W0FIj5Ok3Krip9Shqpw= github.com/wI2L/jsondiff v0.6.1/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ= -github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.16.1 h1:a5TZEPzBFFR53udlIKApXzj8JIF4ZNQ6abH79z5R1S0= +github.com/zclconf/go-cty v1.16.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= From 9320bd1682a22df57950765d71d21b44852084a1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 3 Feb 2025 15:10:19 +0100 Subject: [PATCH 047/105] acc: Use [VARNAME] instead of $VARNAME (#2282) $VARNAME is what we use for environment variables, it's good to separate. Some people use envsubst for homemade variable interpolation, it's also good to have separation there. --- acceptance/acceptance_test.go | 12 ++-- acceptance/bundle/git-permerror/output.txt | 18 +++--- .../bundle/help/bundle-deploy/output.txt | 2 +- .../bundle/help/bundle-deployment/output.txt | 2 +- .../bundle/help/bundle-destroy/output.txt | 2 +- .../help/bundle-generate-dashboard/output.txt | 2 +- .../help/bundle-generate-job/output.txt | 2 +- .../help/bundle-generate-pipeline/output.txt | 2 +- .../bundle/help/bundle-generate/output.txt | 2 +- acceptance/bundle/help/bundle-init/output.txt | 2 +- acceptance/bundle/help/bundle-open/output.txt | 2 +- acceptance/bundle/help/bundle-run/output.txt | 2 +- .../bundle/help/bundle-schema/output.txt | 2 +- .../bundle/help/bundle-summary/output.txt | 2 +- acceptance/bundle/help/bundle-sync/output.txt | 2 +- .../bundle/help/bundle-validate/output.txt | 2 +- acceptance/bundle/help/bundle/output.txt | 2 +- .../bundle/override/clusters/output.txt | 4 +- .../bundle/override/job_cluster/output.txt | 8 +-- .../override/job_cluster_var/output.txt | 20 +++---- .../job_tasks/out.development.stderr.txt | 2 +- .../bundle/override/job_tasks/output.txt | 8 +-- .../override/merge-string-map/output.txt | 8 +-- .../override/pipeline_cluster/output.txt | 8 +-- .../bundle/paths/fallback/output.job.json | 8 +-- .../paths/fallback/output.pipeline.json | 8 +-- acceptance/bundle/paths/fallback/output.txt | 8 +-- .../bundle/paths/nominal/output.job.json | 12 ++-- .../bundle/paths/nominal/output.pipeline.json | 8 +-- acceptance/bundle/paths/nominal/output.txt | 8 +-- .../relative_path_translation/output.txt | 4 +- acceptance/bundle/quality_monitor/output.txt | 6 +- acceptance/bundle/scripts/output.txt | 12 ++-- .../bundle/syncroot/dotdot-git/output.txt | 6 +- .../bundle/syncroot/dotdot-nogit/output.txt | 4 +- .../bundle/templates/dbt-sql/output.txt | 20 +++---- .../dbt-sql/output/my_dbt_sql/databricks.yml | 12 ++-- .../output/my_dbt_sql/profile_template.yml | 4 +- .../my_dbt_sql/resources/my_dbt_sql.job.yml | 2 +- .../templates/default-python/output.txt | 20 +++---- .../output/my_default_python/databricks.yml | 12 ++-- .../resources/my_default_python.job.yml | 2 +- .../output/my_default_python/setup.py | 2 +- .../bundle/templates/default-sql/output.txt | 20 +++---- .../output/my_default_sql/databricks.yml | 12 ++-- .../resources/my_default_sql_sql.job.yml | 2 +- .../experimental-jobs-as-code/output.txt | 18 +++--- .../output/my_jobs_as_code/databricks.yml | 12 ++-- .../resources/my_jobs_as_code_job.py | 2 +- .../bundle/templates/wrong-url/output.txt | 2 +- .../bundle/variables/arg-repeat/output.txt | 4 +- .../variables/complex-cycle-self/output.txt | 4 +- .../bundle/variables/complex-cycle/output.txt | 4 +- .../bundle/variables/complex/out.default.json | 2 +- .../bundle/variables/complex/out.dev.json | 2 +- .../bundle/variables/complex/output.txt | 4 +- .../complex_multiple_files/output.txt | 2 +- .../variables/double_underscore/output.txt | 2 +- acceptance/bundle/variables/empty/output.txt | 4 +- .../bundle/variables/env_overrides/output.txt | 16 ++--- .../bundle/variables/file-defaults/output.txt | 22 +++---- .../bundle/variables/file-defaults/test.toml | 8 +-- .../bundle/variables/git-branch/output.txt | 40 ++++++------- acceptance/bundle/variables/host/output.txt | 4 +- .../prepend-workspace-var/output.txt | 20 +++---- .../variables/resolve-nonstrings/output.txt | 2 +- .../bundle/variables/vanilla/output.txt | 8 +-- .../variable_overrides_in_target/output.txt | 16 ++--- acceptance/selftest/output.txt | 8 +-- acceptance/selftest/test.toml | 4 +- acceptance/terraform/output.txt | 8 +-- acceptance/workspace/jobs/create/output.txt | 2 +- .../bundle/testdata/apps/bundle_deploy.txt | 2 +- .../bundle/testdata/apps/bundle_validate.txt | 4 +- .../testdata/default_python/bundle_deploy.txt | 2 +- .../testdata/default_python/bundle_init.txt | 2 +- .../default_python/bundle_summary.txt | 58 +++++++++---------- .../default_python/bundle_validate.txt | 6 +- libs/testdiff/replacement.go | 49 ++++++++-------- 79 files changed, 324 insertions(+), 329 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ccdf74bcb..871b8bd62 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -102,13 +102,13 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { } t.Setenv("CLI", execPath) - repls.SetPath(execPath, "$CLI") + repls.SetPath(execPath, "[CLI]") // Make helper scripts available t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH"))) tempHomeDir := t.TempDir() - repls.SetPath(tempHomeDir, "$TMPHOME") + repls.SetPath(tempHomeDir, "[TMPHOME]") t.Logf("$TMPHOME=%v", tempHomeDir) // Make use of uv cache; since we set HomeEnvVar to temporary directory, it is not picked up automatically @@ -133,7 +133,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { terraformrcPath := filepath.Join(buildDir, ".terraformrc") t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) - repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE") + repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]") terraformExecPath := filepath.Join(buildDir, "terraform") if runtime.GOOS == "windows" { @@ -141,10 +141,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { } t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) t.Setenv("TERRAFORM", terraformExecPath) - repls.SetPath(terraformExecPath, "$TERRAFORM") + repls.SetPath(terraformExecPath, "[TERRAFORM]") // do it last so that full paths match first: - repls.SetPath(buildDir, "$BUILD_DIR") + repls.SetPath(buildDir, "[BUILD_DIR]") workspaceClient, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -226,7 +226,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont tmpDir = t.TempDir() } - repls.SetPathWithParents(tmpDir, "$TMPDIR") + repls.SetPathWithParents(tmpDir, "[TMPDIR]") repls.Repls = append(repls.Repls, config.Repls...) scriptContents := readMergedScriptContents(t, dir) diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt index 60e77ca0e..03ab93442 100644 --- a/acceptance/bundle/git-permerror/output.txt +++ b/acceptance/bundle/git-permerror/output.txt @@ -2,20 +2,20 @@ >>> chmod 000 .git ->>> $CLI bundle validate +>>> [CLI] bundle validate Error: unable to load repository specific gitconfig: open config: permission denied Name: git-permerror Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/git-permerror/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/git-permerror/default Found 1 error Exit code: 1 ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json Error: unable to load repository specific gitconfig: open config: permission denied @@ -24,7 +24,7 @@ Exit code: 1 "bundle_root_path": "." } ->>> withdir subdir/a/b $CLI bundle validate -o json +>>> withdir subdir/a/b [CLI] bundle validate -o json Error: unable to load repository specific gitconfig: open config: permission denied @@ -38,12 +38,12 @@ Exit code: 1 >>> chmod 000 .git/HEAD ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json { "bundle_root_path": "." } ->>> withdir subdir/a/b $CLI bundle validate -o json +>>> withdir subdir/a/b [CLI] bundle validate -o json { "bundle_root_path": "." } @@ -53,7 +53,7 @@ Exit code: 1 >>> chmod 000 .git/config ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json Error: unable to load repository specific gitconfig: open config: permission denied @@ -62,7 +62,7 @@ Exit code: 1 "bundle_root_path": "." } ->>> withdir subdir/a/b $CLI bundle validate -o json +>>> withdir subdir/a/b [CLI] bundle validate -o json Error: unable to load repository specific gitconfig: open config: permission denied diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 13c903f3e..84351e375 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle deploy --help +>>> [CLI] bundle deploy --help Deploy bundle Usage: diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index ddf5b3305..4199703b3 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle deployment --help +>>> [CLI] bundle deployment --help Deployment related commands Usage: diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index d70164301..5ed9c1b7b 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle destroy --help +>>> [CLI] bundle destroy --help Destroy deployed bundle resources Usage: diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index a63ce0ff8..683175940 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle generate dashboard --help +>>> [CLI] bundle generate dashboard --help Generate configuration for a dashboard Usage: diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index adc3f45ae..6a4274223 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle generate job --help +>>> [CLI] bundle generate job --help Generate bundle configuration for a job Usage: diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index cf5f70920..05c5573b8 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle generate pipeline --help +>>> [CLI] bundle generate pipeline --help Generate bundle configuration for a pipeline Usage: diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 1d77dfdbd..725f19af0 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle generate --help +>>> [CLI] bundle generate --help Generate bundle configuration Usage: diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index bafe5a187..fbafedea2 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle init --help +>>> [CLI] bundle init --help Initialize using a bundle template. TEMPLATE_PATH optionally specifies which template to use. It can be one of the following: diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index 8b98aa850..b8f3f118b 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle open --help +>>> [CLI] bundle open --help Open a resource in the browser Usage: diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 17763a295..4b9efbf2a 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle run --help +>>> [CLI] bundle run --help Run the job or pipeline identified by KEY. The KEY is the unique identifier of the resource to run. In addition to diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index 8f2983f5b..8b8a6b8e9 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle schema --help +>>> [CLI] bundle schema --help Generate JSON Schema for bundle configuration Usage: diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index 935c4bdc5..534bb8214 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle summary --help +>>> [CLI] bundle summary --help Summarize resources deployed by this bundle Usage: diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index 6588e6978..992138a20 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle sync --help +>>> [CLI] bundle sync --help Synchronize bundle tree to the workspace Usage: diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index a0c350faf..7fd1ae7ea 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate --help +>>> [CLI] bundle validate --help Validate configuration Usage: diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index e0e2ea47c..fc6dd623d 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle --help +>>> [CLI] bundle --help Databricks Asset Bundles let you express data/AI/analytics projects as code. Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html diff --git a/acceptance/bundle/override/clusters/output.txt b/acceptance/bundle/override/clusters/output.txt index cff30b3af..a30a7bbff 100644 --- a/acceptance/bundle/override/clusters/output.txt +++ b/acceptance/bundle/override/clusters/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json -t default +>>> [CLI] bundle validate -o json -t default { "autoscale": { "max_workers": 7, @@ -15,7 +15,7 @@ "spark_version": "13.3.x-scala2.12" } ->>> $CLI bundle validate -o json -t development +>>> [CLI] bundle validate -o json -t development { "autoscale": { "max_workers": 3, diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt index ff6e8316e..e4120e1c3 100644 --- a/acceptance/bundle/override/job_cluster/output.txt +++ b/acceptance/bundle/override/job_cluster/output.txt @@ -1,10 +1,10 @@ ->>> $CLI bundle validate -o json -t development +>>> [CLI] bundle validate -o json -t development { "foo": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -27,12 +27,12 @@ } } ->>> $CLI bundle validate -o json -t staging +>>> [CLI] bundle validate -o json -t staging { "foo": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt index 0b19e5eb2..3545d6987 100644 --- a/acceptance/bundle/override/job_cluster_var/output.txt +++ b/acceptance/bundle/override/job_cluster_var/output.txt @@ -1,10 +1,10 @@ ->>> $CLI bundle validate -o json -t development +>>> [CLI] bundle validate -o json -t development { "foo": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -27,21 +27,21 @@ } } ->>> $CLI bundle validate -t development +>>> [CLI] bundle validate -t development Name: override_job_cluster Target: development Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/development + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/override_job_cluster/development Validation OK! ->>> $CLI bundle validate -o json -t staging +>>> [CLI] bundle validate -o json -t staging { "foo": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -64,11 +64,11 @@ Validation OK! } } ->>> $CLI bundle validate -t staging +>>> [CLI] bundle validate -t staging Name: override_job_cluster Target: staging Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/override_job_cluster/staging Validation OK! diff --git a/acceptance/bundle/override/job_tasks/out.development.stderr.txt b/acceptance/bundle/override/job_tasks/out.development.stderr.txt index 7b6fef0cc..1873feb35 100644 --- a/acceptance/bundle/override/job_tasks/out.development.stderr.txt +++ b/acceptance/bundle/override/job_tasks/out.development.stderr.txt @@ -1,5 +1,5 @@ ->>> errcode $CLI bundle validate -o json -t development +>>> errcode [CLI] bundle validate -o json -t development Error: file ./test1.py not found diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index 915351d4e..1f7796217 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -28,7 +28,7 @@ ] } ->>> errcode $CLI bundle validate -o json -t staging +>>> errcode [CLI] bundle validate -o json -t staging Error: file ./test1.py not found @@ -63,14 +63,14 @@ Exit code: 1 ] } ->>> errcode $CLI bundle validate -t staging +>>> errcode [CLI] bundle validate -t staging Error: file ./test1.py not found Name: override_job_tasks Target: staging Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/override_job_tasks/staging + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/override_job_tasks/staging Found 1 error diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt index b566aa07f..6e2aef87b 100644 --- a/acceptance/bundle/override/merge-string-map/output.txt +++ b/acceptance/bundle/override/merge-string-map/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json -t dev +>>> [CLI] bundle validate -o json -t dev Warning: expected map, found string at resources.clusters.my_cluster in databricks.yml:6:17 @@ -13,7 +13,7 @@ Warning: expected map, found string } } ->>> $CLI bundle validate -t dev +>>> [CLI] bundle validate -t dev Warning: expected map, found string at resources.clusters.my_cluster in databricks.yml:6:17 @@ -21,7 +21,7 @@ Warning: expected map, found string Name: merge-string-map Target: dev Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/merge-string-map/dev + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/merge-string-map/dev Found 1 warning diff --git a/acceptance/bundle/override/pipeline_cluster/output.txt b/acceptance/bundle/override/pipeline_cluster/output.txt index 8babed0ec..d1a67f6b9 100644 --- a/acceptance/bundle/override/pipeline_cluster/output.txt +++ b/acceptance/bundle/override/pipeline_cluster/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json -t development +>>> [CLI] bundle validate -o json -t development { "foo": { "clusters": [ @@ -14,14 +14,14 @@ ], "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/development/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/development/state/metadata.json" }, "name": "job", "permissions": [] } } ->>> $CLI bundle validate -o json -t staging +>>> [CLI] bundle validate -o json -t staging { "foo": { "clusters": [ @@ -36,7 +36,7 @@ ], "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/staging/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/staging/state/metadata.json" }, "name": "job", "permissions": [] diff --git a/acceptance/bundle/paths/fallback/output.job.json b/acceptance/bundle/paths/fallback/output.job.json index fe9e1cf3d..ac79e0cf6 100644 --- a/acceptance/bundle/paths/fallback/output.job.json +++ b/acceptance/bundle/paths/fallback/output.job.json @@ -2,14 +2,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py" + "python_file": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -19,7 +19,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -28,7 +28,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql" + "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, diff --git a/acceptance/bundle/paths/fallback/output.pipeline.json b/acceptance/bundle/paths/fallback/output.pipeline.json index 38521cb22..7ed4f74e6 100644 --- a/acceptance/bundle/paths/fallback/output.pipeline.json +++ b/acceptance/bundle/paths/fallback/output.pipeline.json @@ -1,22 +1,22 @@ [ { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py" + "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1" + "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py" + "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2" + "path": "/Workspace/Users/[USERNAME]/.bundle/fallback/development/files/src/notebook2" } } ] diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index 63121f3d7..85f185851 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -1,15 +1,15 @@ ->>> $CLI bundle validate -t development -o json +>>> [CLI] bundle validate -t development -o json ->>> $CLI bundle validate -t error +>>> [CLI] bundle validate -t error Error: notebook this value is overridden not found. Local notebook references are expected to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] Name: fallback Target: error Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/fallback/error + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/fallback/error Found 1 error diff --git a/acceptance/bundle/paths/nominal/output.job.json b/acceptance/bundle/paths/nominal/output.job.json index 9e1cb4d90..26d19d77c 100644 --- a/acceptance/bundle/paths/nominal/output.job.json +++ b/acceptance/bundle/paths/nominal/output.job.json @@ -2,14 +2,14 @@ { "job_cluster_key": "default", "notebook_task": { - "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook" }, "task_key": "notebook_example" }, { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py" }, "task_key": "spark_python_example" }, @@ -19,7 +19,7 @@ "dbt run", "dbt run" ], - "project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project" + "project_directory": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/dbt_project" }, "job_cluster_key": "default", "task_key": "dbt_example" @@ -28,7 +28,7 @@ "job_cluster_key": "default", "sql_task": { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql" + "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/sql.sql" }, "warehouse_id": "cafef00d" }, @@ -68,7 +68,7 @@ "for_each_task": { "task": { "notebook_task": { - "notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook" } } }, @@ -80,7 +80,7 @@ "task": { "job_cluster_key": "default", "spark_python_task": { - "python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py" + "python_file": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file.py" } } }, diff --git a/acceptance/bundle/paths/nominal/output.pipeline.json b/acceptance/bundle/paths/nominal/output.pipeline.json index 277b0c4a1..c6f2e0868 100644 --- a/acceptance/bundle/paths/nominal/output.pipeline.json +++ b/acceptance/bundle/paths/nominal/output.pipeline.json @@ -1,22 +1,22 @@ [ { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py" + "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file1.py" } }, { "notebook": { - "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1" + "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook1" } }, { "file": { - "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py" + "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/file2.py" } }, { "notebook": { - "path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2" + "path": "/Workspace/Users/[USERNAME]/.bundle/nominal/development/files/src/notebook2" } } ] diff --git a/acceptance/bundle/paths/nominal/output.txt b/acceptance/bundle/paths/nominal/output.txt index 1badcdec6..40670f4cb 100644 --- a/acceptance/bundle/paths/nominal/output.txt +++ b/acceptance/bundle/paths/nominal/output.txt @@ -1,15 +1,15 @@ ->>> $CLI bundle validate -t development -o json +>>> [CLI] bundle validate -t development -o json ->>> $CLI bundle validate -t error +>>> [CLI] bundle validate -t error Error: notebook this value is overridden not found. Local notebook references are expected to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb] Name: nominal Target: error Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/nominal/error + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/nominal/error Found 1 error diff --git a/acceptance/bundle/paths/relative_path_translation/output.txt b/acceptance/bundle/paths/relative_path_translation/output.txt index 362f2ec7b..b13d612b6 100644 --- a/acceptance/bundle/paths/relative_path_translation/output.txt +++ b/acceptance/bundle/paths/relative_path_translation/output.txt @@ -1,4 +1,4 @@ ->>> $CLI bundle validate -t default -o json +>>> [CLI] bundle validate -t default -o json ->>> $CLI bundle validate -t override -o json +>>> [CLI] bundle validate -t override -o json diff --git a/acceptance/bundle/quality_monitor/output.txt b/acceptance/bundle/quality_monitor/output.txt index b3718c802..8a7f64ef2 100644 --- a/acceptance/bundle/quality_monitor/output.txt +++ b/acceptance/bundle/quality_monitor/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json -t development +>>> [CLI] bundle validate -o json -t development { "mode": "development", "quality_monitors": { @@ -21,7 +21,7 @@ } } ->>> $CLI bundle validate -o json -t staging +>>> [CLI] bundle validate -o json -t staging { "mode": null, "quality_monitors": { @@ -46,7 +46,7 @@ } } ->>> $CLI bundle validate -o json -t production +>>> [CLI] bundle validate -o json -t production { "mode": null, "quality_monitors": { diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt index ec5978380..2deedb0e7 100644 --- a/acceptance/bundle/scripts/output.txt +++ b/acceptance/bundle/scripts/output.txt @@ -1,5 +1,5 @@ ->>> EXITCODE=0 errcode $CLI bundle validate +>>> EXITCODE=0 errcode [CLI] bundle validate Executing 'preinit' script from myscript.py 0 preinit: hello stdout! from myscript.py 0 preinit: hello stderr! @@ -9,12 +9,12 @@ from myscript.py 0 postinit: hello stderr! Name: scripts Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/scripts/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/scripts/default Validation OK! ->>> EXITCODE=1 errcode $CLI bundle validate +>>> EXITCODE=1 errcode [CLI] bundle validate Executing 'preinit' script from myscript.py 1 preinit: hello stdout! from myscript.py 1 preinit: hello stderr! @@ -26,7 +26,7 @@ Found 1 error Exit code: 1 ->>> EXITCODE=0 errcode $CLI bundle deploy +>>> EXITCODE=0 errcode [CLI] bundle deploy Executing 'preinit' script from myscript.py 0 preinit: hello stdout! from myscript.py 0 preinit: hello stderr! @@ -42,7 +42,7 @@ from myscript.py 0 postbuild: hello stderr! Executing 'predeploy' script from myscript.py 0 predeploy: hello stdout! from myscript.py 0 predeploy: hello stderr! -Error: unable to deploy to /Workspace/Users/$USERNAME/.bundle/scripts/default/state as $USERNAME. +Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME]. Please make sure the current user or one of their groups is listed under the permissions of this bundle. For assistance, contact the owners of this project. They may need to redeploy the bundle to apply the new permissions. diff --git a/acceptance/bundle/syncroot/dotdot-git/output.txt b/acceptance/bundle/syncroot/dotdot-git/output.txt index f1dc5fb01..dbfc8451f 100644 --- a/acceptance/bundle/syncroot/dotdot-git/output.txt +++ b/acceptance/bundle/syncroot/dotdot-git/output.txt @@ -1,10 +1,10 @@ -Error: path "$TMPDIR" is not within repository root "$TMPDIR/myrepo" +Error: path "[TMPDIR]" is not within repository root "[TMPDIR]/myrepo" Name: test-bundle Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Found 1 error diff --git a/acceptance/bundle/syncroot/dotdot-nogit/output.txt b/acceptance/bundle/syncroot/dotdot-nogit/output.txt index 46f617f35..4f189effd 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/output.txt +++ b/acceptance/bundle/syncroot/dotdot-nogit/output.txt @@ -1,7 +1,7 @@ Name: test-bundle Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/test-bundle/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Validation OK! diff --git a/acceptance/bundle/templates/dbt-sql/output.txt b/acceptance/bundle/templates/dbt-sql/output.txt index 972c7e152..2699ad554 100644 --- a/acceptance/bundle/templates/dbt-sql/output.txt +++ b/acceptance/bundle/templates/dbt-sql/output.txt @@ -1,32 +1,32 @@ ->>> $CLI bundle init dbt-sql --config-file ./input.json --output-dir output +>>> [CLI] bundle init dbt-sql --config-file ./input.json --output-dir output Welcome to the dbt template for Databricks Asset Bundles! A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. -workspace_host: $DATABRICKS_URL +workspace_host: [DATABRICKS_URL] 📊 Your new project has been created in the 'my_dbt_sql' directory! If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. Refer to the README.md file for full "getting started" guide and production setup instructions. ->>> $CLI bundle validate -t dev +>>> [CLI] bundle validate -t dev Name: my_dbt_sql Target: dev Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_dbt_sql/dev Validation OK! ->>> $CLI bundle validate -t prod +>>> [CLI] bundle validate -t prod Name: my_dbt_sql Target: prod Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_dbt_sql/prod Validation OK! diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml index cdf3704b9..3651ef12d 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml @@ -19,16 +19,16 @@ targets: # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. mode: development workspace: - host: $DATABRICKS_URL + host: [DATABRICKS_URL] prod: mode: production workspace: - host: $DATABRICKS_URL - # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy. - root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target} + host: [DATABRICKS_URL] + # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} permissions: - - user_name: $USERNAME + - user_name: [USERNAME] level: CAN_MANAGE run_as: - user_name: $USERNAME + user_name: [USERNAME] diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml index 5e0f0fc29..bdb41ab20 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml @@ -5,7 +5,7 @@ fixed: type: databricks prompts: host: - default: $DATABRICKS_HOST + default: [DATABRICKS_HOST] token: hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' hide_input: true @@ -16,7 +16,7 @@ prompts: hint: 'initial catalog' default: main schema: - hint: 'personal schema where dbt will build objects during development, example: $USERNAME' + hint: 'personal schema where dbt will build objects during development, example: [USERNAME]' threads: hint: 'threads to use during development, 1 or more' type: 'int' diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index d52f8ed50..b522931f9 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -11,7 +11,7 @@ resources: email_notifications: on_failure: - - $USERNAME + - [USERNAME] tasks: diff --git a/acceptance/bundle/templates/default-python/output.txt b/acceptance/bundle/templates/default-python/output.txt index 5493ac2cf..930e756de 100644 --- a/acceptance/bundle/templates/default-python/output.txt +++ b/acceptance/bundle/templates/default-python/output.txt @@ -1,30 +1,30 @@ ->>> $CLI bundle init default-python --config-file ./input.json --output-dir output +>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): $DATABRICKS_URL +Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] ✨ Your new project has been created in the 'my_default_python' directory! Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. ->>> $CLI bundle validate -t dev +>>> [CLI] bundle validate -t dev Name: my_default_python Target: dev Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev Validation OK! ->>> $CLI bundle validate -t prod +>>> [CLI] bundle validate -t prod Name: my_default_python Target: prod Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod Validation OK! diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml index 3fa777219..6df75c209 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml @@ -16,16 +16,16 @@ targets: mode: development default: true workspace: - host: $DATABRICKS_URL + host: [DATABRICKS_URL] prod: mode: production workspace: - host: $DATABRICKS_URL - # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy. - root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target} + host: [DATABRICKS_URL] + # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} permissions: - - user_name: $USERNAME + - user_name: [USERNAME] level: CAN_MANAGE run_as: - user_name: $USERNAME + user_name: [USERNAME] diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml index e6148a4ad..d9e31691a 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml @@ -12,7 +12,7 @@ resources: email_notifications: on_failure: - - $USERNAME + - [USERNAME] tasks: - task_key: notebook_task diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py index 84b24ecb8..548f1035e 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py +++ b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py @@ -23,7 +23,7 @@ setup( # to ensure that changes to wheel package are picked up when used on all-purpose clusters version=my_default_python.__version__ + "+" + local_version, url="https://databricks.com", - author="$USERNAME", + author="[USERNAME]", description="wheel file based on my_default_python/src", packages=find_packages(where="./src"), package_dir={"": "src"}, diff --git a/acceptance/bundle/templates/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt index fe0139093..06eff962b 100644 --- a/acceptance/bundle/templates/default-sql/output.txt +++ b/acceptance/bundle/templates/default-sql/output.txt @@ -1,32 +1,32 @@ ->>> $CLI bundle init default-sql --config-file ./input.json --output-dir output +>>> [CLI] bundle init default-sql --config-file ./input.json --output-dir output Welcome to the default SQL template for Databricks Asset Bundles! A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. -workspace_host: $DATABRICKS_URL +workspace_host: [DATABRICKS_URL] ✨ Your new project has been created in the 'my_default_sql' directory! Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. ->>> $CLI bundle validate -t dev +>>> [CLI] bundle validate -t dev Name: my_default_sql Target: dev Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev Validation OK! ->>> $CLI bundle validate -t prod +>>> [CLI] bundle validate -t prod Name: my_default_sql Target: prod Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod Validation OK! diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml index 16292bc84..6ef09cf3b 100644 --- a/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml @@ -25,7 +25,7 @@ targets: mode: development default: true workspace: - host: $DATABRICKS_URL + host: [DATABRICKS_URL] variables: warehouse_id: f00dcafe catalog: main @@ -34,15 +34,15 @@ targets: prod: mode: production workspace: - host: $DATABRICKS_URL - # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy. - root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target} + host: [DATABRICKS_URL] + # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} variables: warehouse_id: f00dcafe catalog: main schema: default permissions: - - user_name: $USERNAME + - user_name: [USERNAME] level: CAN_MANAGE run_as: - user_name: $USERNAME + user_name: [USERNAME] diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml index 86de0f9db..34d60e3d5 100644 --- a/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml @@ -12,7 +12,7 @@ resources: email_notifications: on_failure: - - $USERNAME + - [USERNAME] parameters: - name: catalog diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt index 10aca003e..984dad604 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt @@ -1,28 +1,28 @@ ->>> $CLI bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output +>>> [CLI] bundle init experimental-jobs-as-code --config-file ./input.json --output-dir output Welcome to (EXPERIMENTAL) "Jobs as code" template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_jobs_as_code/databricks.yml'): $DATABRICKS_URL +Workspace to use (auto-detected, edit in 'my_jobs_as_code/databricks.yml'): [DATABRICKS_URL] ✨ Your new project has been created in the 'my_jobs_as_code' directory! Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. ->>> $CLI bundle validate -t dev --output json -Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: $DEV_VERSION +>>> [CLI] bundle validate -t dev --output json +Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.238.0, current: [DEV_VERSION] { "jobs": { "my_jobs_as_code_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", "email_notifications": { "on_failure": [ - "$USERNAME" + "[USERNAME]" ] }, "format": "MULTI_TASK", @@ -40,19 +40,19 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi } ], "max_concurrent_runs": 4, - "name": "[dev $USERNAME] my_jobs_as_code_job", + "name": "[dev [USERNAME]] my_jobs_as_code_job", "permissions": [], "queue": { "enabled": true }, "tags": { - "dev": "$USERNAME" + "dev": "[USERNAME]" }, "tasks": [ { "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/$USERNAME/.bundle/my_jobs_as_code/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src/notebook" }, "task_key": "notebook_task" }, diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml index 54e69a256..9299c96e8 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/databricks.yml @@ -34,16 +34,16 @@ targets: mode: development default: true workspace: - host: $DATABRICKS_URL + host: [DATABRICKS_URL] prod: mode: production workspace: - host: $DATABRICKS_URL - # We explicitly specify /Workspace/Users/$USERNAME to make sure we only have a single copy. - root_path: /Workspace/Users/$USERNAME/.bundle/${bundle.name}/${bundle.target} + host: [DATABRICKS_URL] + # We explicitly specify /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} permissions: - - user_name: $USERNAME + - user_name: [USERNAME] level: CAN_MANAGE run_as: - user_name: $USERNAME + user_name: [USERNAME] diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py index 4854d656f..e8406fd7b 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py @@ -17,7 +17,7 @@ my_jobs_as_code_job = Job.from_dict( }, "email_notifications": { "on_failure": [ - "$USERNAME", + "[USERNAME]", ], }, "tasks": [ diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates/wrong-url/output.txt index b78cf4b68..6b4f9c459 100644 --- a/acceptance/bundle/templates/wrong-url/output.txt +++ b/acceptance/bundle/templates/wrong-url/output.txt @@ -1,4 +1,4 @@ -Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world $TMPDIR_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '$TMPDIR_GPARENT/world-123456'... +Error: git clone failed: git clone https://invalid-domain-123.databricks.com/hello/world [TMPDIR]_GPARENT/world-123456 --no-tags --depth=1: exit status 128. Cloning into '[TMPDIR]_GPARENT/world-123456'... fatal: unable to access 'https://invalid-domain-123.databricks.com/hello/world/': Could not resolve host: invalid-domain-123.databricks.com diff --git a/acceptance/bundle/variables/arg-repeat/output.txt b/acceptance/bundle/variables/arg-repeat/output.txt index 2f9de1a3c..4b97d70a1 100644 --- a/acceptance/bundle/variables/arg-repeat/output.txt +++ b/acceptance/bundle/variables/arg-repeat/output.txt @@ -1,5 +1,5 @@ ->>> errcode $CLI bundle validate --var a=one -o json +>>> errcode [CLI] bundle validate --var a=one -o json { "a": { "default": "hello", @@ -7,7 +7,7 @@ } } ->>> errcode $CLI bundle validate --var a=one --var a=two +>>> errcode [CLI] bundle validate --var a=one --var a=two Error: failed to assign two to a: variable has already been assigned value: one Name: arg-repeat diff --git a/acceptance/bundle/variables/complex-cycle-self/output.txt b/acceptance/bundle/variables/complex-cycle-self/output.txt index fa80154ca..7447de349 100644 --- a/acceptance/bundle/variables/complex-cycle-self/output.txt +++ b/acceptance/bundle/variables/complex-cycle-self/output.txt @@ -3,7 +3,7 @@ Warning: Detected unresolved variables after 11 resolution rounds Name: cycle Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/cycle/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/cycle/default Found 1 warning diff --git a/acceptance/bundle/variables/complex-cycle/output.txt b/acceptance/bundle/variables/complex-cycle/output.txt index fa80154ca..7447de349 100644 --- a/acceptance/bundle/variables/complex-cycle/output.txt +++ b/acceptance/bundle/variables/complex-cycle/output.txt @@ -3,7 +3,7 @@ Warning: Detected unresolved variables after 11 resolution rounds Name: cycle Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/cycle/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/cycle/default Found 1 warning diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json index a1ccd52bc..0804ad588 100644 --- a/acceptance/bundle/variables/complex/out.default.json +++ b/acceptance/bundle/variables/complex/out.default.json @@ -4,7 +4,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables/default/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json index bb939091b..e93c2c297 100644 --- a/acceptance/bundle/variables/complex/out.dev.json +++ b/acceptance/bundle/variables/complex/out.dev.json @@ -4,7 +4,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/variables/complex/output.txt b/acceptance/bundle/variables/complex/output.txt index ce295421f..f1d4c04cc 100644 --- a/acceptance/bundle/variables/complex/output.txt +++ b/acceptance/bundle/variables/complex/output.txt @@ -1,10 +1,10 @@ ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json >>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json "task with spark version 13.2.x-scala2.11 and jar /path/to/jar" ->>> $CLI bundle validate -o json -t dev +>>> [CLI] bundle validate -o json -t dev >>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar" diff --git a/acceptance/bundle/variables/complex_multiple_files/output.txt b/acceptance/bundle/variables/complex_multiple_files/output.txt index ec2cad1ce..433e6da0c 100644 --- a/acceptance/bundle/variables/complex_multiple_files/output.txt +++ b/acceptance/bundle/variables/complex_multiple_files/output.txt @@ -4,7 +4,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables-multiple-files/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/complex-variables-multiple-files/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/variables/double_underscore/output.txt b/acceptance/bundle/variables/double_underscore/output.txt index 45529038d..0124f5442 100644 --- a/acceptance/bundle/variables/double_underscore/output.txt +++ b/acceptance/bundle/variables/double_underscore/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json [ { "task_key": "test default" diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 8933443df..cbd0f1989 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -3,8 +3,8 @@ Error: no value assigned to required variable a. Assignment can be done using "- Name: empty${var.a} Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/empty${var.a}/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/empty${var.a}/default Found 1 error diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index 06e6e518b..93b3b6716 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -1,27 +1,27 @@ ->>> $CLI bundle validate -t env-with-single-variable-override -o json +>>> [CLI] bundle validate -t env-with-single-variable-override -o json "default-a dev-b" ->>> $CLI bundle validate -t env-with-two-variable-overrides -o json +>>> [CLI] bundle validate -t env-with-two-variable-overrides -o json "prod-a prod-b" ->>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json +>>> BUNDLE_VAR_b=env-var-b [CLI] bundle validate -t env-with-two-variable-overrides -o json "prod-a env-var-b" ->>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment +>>> errcode [CLI] bundle validate -t env-missing-a-required-variable-assignment Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: test bundle Target: env-missing-a-required-variable-assignment Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/test bundle/env-missing-a-required-variable-assignment + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test bundle/env-missing-a-required-variable-assignment Found 1 error Exit code: 1 ->>> errcode $CLI bundle validate -t env-using-an-undefined-variable +>>> errcode [CLI] bundle validate -t env-using-an-undefined-variable Error: variable c is not defined but is assigned a value Name: test bundle @@ -30,7 +30,7 @@ Found 1 error Exit code: 1 ->>> $CLI bundle validate -t env-overrides-lookup -o json +>>> [CLI] bundle validate -t env-overrides-lookup -o json { "a": "default-a", "b": "prod-b", diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt index 5b01a1b66..234ddcbbd 100644 --- a/acceptance/bundle/variables/file-defaults/output.txt +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -1,6 +1,6 @@ === variable file ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json { "job_cluster_key": "mlops_stacks-cluster", "new_cluster": { @@ -10,7 +10,7 @@ } === variable file and variable flag ->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden +>>> [CLI] bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden { "job_cluster_key": "mlops_stacks-cluster-overriden", "new_cluster": { @@ -20,7 +20,7 @@ } === variable file and environment variable ->>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json +>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden [CLI] bundle validate -o json { "job_cluster_key": "mlops_stacks-cluster-overriden", "new_cluster": { @@ -30,7 +30,7 @@ } === variable has value in config file ->>> $CLI bundle validate -o json --target with_value +>>> [CLI] bundle validate -o json --target with_value { "job_cluster_key": "mlops_stacks-cluster-from-file", "new_cluster": { @@ -40,8 +40,8 @@ } === file cannot be parsed ->>> errcode $CLI bundle validate -o json --target invalid_json -Error: failed to parse variables file $TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') +>>> errcode [CLI] bundle validate -o json --target invalid_json +Error: failed to parse variables file [TMPDIR]/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') Exit code: 1 @@ -54,8 +54,8 @@ Exit code: 1 } === file has wrong structure ->>> errcode $CLI bundle validate -o json --target wrong_file_structure -Error: failed to parse variables file $TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format +>>> errcode [CLI] bundle validate -o json --target wrong_file_structure +Error: failed to parse variables file [TMPDIR]/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format Variables file must be a JSON object with the following format: {"var1": "value1", "var2": "value2"} @@ -71,7 +71,7 @@ Exit code: 1 } === file has variable that is complex but default is string ->>> errcode $CLI bundle validate -o json --target complex_to_string +>>> errcode [CLI] bundle validate -o json --target complex_to_string Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type @@ -85,7 +85,7 @@ Exit code: 1 } === file has variable that is string but default is complex ->>> errcode $CLI bundle validate -o json --target string_to_complex +>>> errcode [CLI] bundle validate -o json --target string_to_complex Error: variable cluster is of type complex, but the value in the variable file is not a complex type @@ -99,7 +99,7 @@ Exit code: 1 } === variable is required but it's not provided in the file ->>> errcode $CLI bundle validate -o json --target without_defaults +>>> errcode [CLI] bundle validate -o json --target without_defaults Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//variable-overrides.json file diff --git a/acceptance/bundle/variables/file-defaults/test.toml b/acceptance/bundle/variables/file-defaults/test.toml index 08403b606..da8854775 100644 --- a/acceptance/bundle/variables/file-defaults/test.toml +++ b/acceptance/bundle/variables/file-defaults/test.toml @@ -1,8 +1,4 @@ # Fix for windows [[Repls]] -Old = '\$TMPDIR\\.databricks\\bundle\\wrong_file_structure\\variable-overrides.json' -New = '$$TMPDIR/.databricks/bundle/wrong_file_structure/variable-overrides.json' - -[[Repls]] -Old = '\$TMPDIR\\.databricks\\bundle\\invalid_json\\variable-overrides.json' -New = '$$TMPDIR/.databricks/bundle/invalid_json/variable-overrides.json' +Old = '\\' +New = '/' diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index 21ed9e7de..68f27a3f1 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json +>>> [CLI] bundle validate -o json { "bundle": { "environment": "prod", @@ -11,7 +11,7 @@ "name": "git", "target": "prod", "terraform": { - "exec_path": "$TERRAFORM" + "exec_path": "[TERRAFORM]" } }, "sync": { @@ -27,24 +27,24 @@ } }, "workspace": { - "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts", - "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files", - "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources", - "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod", - "state_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/state" + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/artifacts", + "file_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/git/prod/state" } } ->>> $CLI bundle validate +>>> [CLI] bundle validate Name: git Target: prod Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/git/prod + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/git/prod Validation OK! ->>> $CLI bundle validate -o json -t dev +>>> [CLI] bundle validate -o json -t dev { "bundle": { "environment": "dev", @@ -56,7 +56,7 @@ Validation OK! "name": "git", "target": "dev", "terraform": { - "exec_path": "$TERRAFORM" + "exec_path": "[TERRAFORM]" } }, "sync": { @@ -72,19 +72,19 @@ Validation OK! } }, "workspace": { - "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts", - "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files", - "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources", - "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev", - "state_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/state" + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/artifacts", + "file_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/git/dev/state" } } ->>> $CLI bundle validate -t dev +>>> [CLI] bundle validate -t dev Name: git Target: dev Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/git/dev + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/git/dev Validation OK! diff --git a/acceptance/bundle/variables/host/output.txt b/acceptance/bundle/variables/host/output.txt index 89342908c..63c41426a 100644 --- a/acceptance/bundle/variables/host/output.txt +++ b/acceptance/bundle/variables/host/output.txt @@ -1,5 +1,5 @@ ->>> errcode $CLI bundle validate -o json +>>> errcode [CLI] bundle validate -o json Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name { @@ -25,7 +25,7 @@ Error: failed during request visitor: parse "https://${var.host}": invalid chara } Exit code: 1 ->>> errcode $CLI bundle validate +>>> errcode [CLI] bundle validate Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name Name: host diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index 93b652894..a48a58fba 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -7,7 +7,7 @@ }, "target": "dev", "terraform": { - "exec_path": "$TERRAFORM" + "exec_path": "[TERRAFORM]" } }, "resources": { @@ -15,7 +15,7 @@ "my_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Users/$USERNAME/path/to/root/state/metadata.json" + "metadata_file_path": "/Users/[USERNAME]/path/to/root/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", @@ -29,7 +29,7 @@ "existing_cluster_id": "500", "python_wheel_task": { "named_parameters": { - "conf-file": "/Users/$USERNAME/path/to/root/files/path/to/config.yaml" + "conf-file": "/Users/[USERNAME]/path/to/root/files/path/to/config.yaml" } }, "task_key": "" @@ -46,17 +46,17 @@ "targets": null, "variables": { "workspace_root": { - "default": "/Users/$USERNAME", + "default": "/Users/[USERNAME]", "description": "root directory in the Databricks workspace to store the asset bundle and associated artifacts", - "value": "/Users/$USERNAME" + "value": "/Users/[USERNAME]" } }, "workspace": { - "artifact_path": "/Users/$USERNAME/path/to/root/artifacts", - "file_path": "/Users/$USERNAME/path/to/root/files", + "artifact_path": "/Users/[USERNAME]/path/to/root/artifacts", + "file_path": "/Users/[USERNAME]/path/to/root/files", "profile": "profile_name", - "resource_path": "/Users/$USERNAME/path/to/root/resources", - "root_path": "/Users/$USERNAME/path/to/root", - "state_path": "/Users/$USERNAME/path/to/root/state" + "resource_path": "/Users/[USERNAME]/path/to/root/resources", + "root_path": "/Users/[USERNAME]/path/to/root", + "state_path": "/Users/[USERNAME]/path/to/root/state" } } diff --git a/acceptance/bundle/variables/resolve-nonstrings/output.txt b/acceptance/bundle/variables/resolve-nonstrings/output.txt index 3a1eb9c47..951ad7a0d 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/output.txt +++ b/acceptance/bundle/variables/resolve-nonstrings/output.txt @@ -20,7 +20,7 @@ "job1": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index e98882bb0..3958c39b9 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -1,15 +1,15 @@ ->>> BUNDLE_VAR_b=def $CLI bundle validate -o json +>>> BUNDLE_VAR_b=def [CLI] bundle validate -o json "abc def" ->>> errcode $CLI bundle validate +>>> errcode [CLI] bundle validate Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: ${var.a} ${var.b} Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/${var.a} ${var.b}/default + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/${var.a} ${var.b}/default Found 1 error diff --git a/acceptance/bundle/variables/variable_overrides_in_target/output.txt b/acceptance/bundle/variables/variable_overrides_in_target/output.txt index 8998b691d..d112cf2de 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/output.txt +++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt @@ -1,5 +1,5 @@ ->>> $CLI bundle validate -o json -t use-default-variable-values +>>> [CLI] bundle validate -o json -t use-default-variable-values { "pipelines": { "my_pipeline": { @@ -12,7 +12,7 @@ "continuous": true, "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/use-default-variable-values/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/use-default-variable-values/state/metadata.json" }, "name": "a_string", "permissions": [] @@ -20,7 +20,7 @@ } } ->>> $CLI bundle validate -o json -t override-string-variable +>>> [CLI] bundle validate -o json -t override-string-variable { "pipelines": { "my_pipeline": { @@ -33,7 +33,7 @@ "continuous": true, "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-string-variable/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-string-variable/state/metadata.json" }, "name": "overridden_string", "permissions": [] @@ -41,7 +41,7 @@ } } ->>> $CLI bundle validate -o json -t override-int-variable +>>> [CLI] bundle validate -o json -t override-int-variable { "pipelines": { "my_pipeline": { @@ -54,7 +54,7 @@ "continuous": true, "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-int-variable/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-int-variable/state/metadata.json" }, "name": "a_string", "permissions": [] @@ -62,7 +62,7 @@ } } ->>> $CLI bundle validate -o json -t override-both-bool-and-string-variables +>>> [CLI] bundle validate -o json -t override-both-bool-and-string-variables { "pipelines": { "my_pipeline": { @@ -75,7 +75,7 @@ "continuous": false, "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json" }, "name": "overridden_string", "permissions": [] diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/output.txt index 91aa8c33e..cadbdebb5 100644 --- a/acceptance/selftest/output.txt +++ b/acceptance/selftest/output.txt @@ -18,13 +18,13 @@ Exit code: 7 === Capturing pwd >>> python3 -c import os; print(os.getcwd()) -$TMPDIR +[TMPDIR] === Capturing subdir >>> mkdir -p subdir/a/b/c >>> withdir subdir/a/b/c python3 -c import os; print(os.getcwd()) -$TMPDIR/subdir/a/b/c +[TMPDIR]/subdir/a/b/c === Custom output files - everything starting with out is captured and compared >>> echo HELLO @@ -35,5 +35,5 @@ CUSTOM_NUMBER_REGEX 123456 === Testing --version ->>> $CLI --version -Databricks CLI v$DEV_VERSION +>>> [CLI] --version +Databricks CLI v[DEV_VERSION] diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml index 9607ec5df..762e28ceb 100644 --- a/acceptance/selftest/test.toml +++ b/acceptance/selftest/test.toml @@ -16,5 +16,5 @@ New = "CUSTOM_NUMBER_REGEX" [[Repls]] # Fix path with reverse slashes in the output for Windows. -Old = '\$TMPDIR\\subdir\\a\\b\\c' -New = '$$TMPDIR/subdir/a/b/c' +Old = 'TMPDIR]\\subdir\\a\\b\\c' +New = 'TMPDIR]/subdir/a/b/c' diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt index 32589ddab..6bdc809f6 100644 --- a/acceptance/terraform/output.txt +++ b/acceptance/terraform/output.txt @@ -1,5 +1,5 @@ ->>> $TERRAFORM init -no-color -get=false +>>> [TERRAFORM] init -no-color -get=false Initializing the backend... @@ -35,12 +35,12 @@ If you ever set or change modules or backend configuration for Terraform, rerun this command to reinitialize your working directory. If you forget, other commands will detect it and remind you to do so if necessary. ->>> $TERRAFORM plan -no-color +>>> [TERRAFORM] plan -no-color data.databricks_current_user.me: Reading... -data.databricks_current_user.me: Read complete after (redacted) [id=$USER.Id] +data.databricks_current_user.me: Read complete after (redacted) [id=[USERID]] Changes to Outputs: - + username = "$USERNAME" + + username = "[USERNAME]" You can apply this plan to save these new output values to the Terraform state, without changing any real infrastructure. diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt index a9487fe5b..50b823aa0 100644 --- a/acceptance/workspace/jobs/create/output.txt +++ b/acceptance/workspace/jobs/create/output.txt @@ -1,5 +1,5 @@ ->>> $CLI jobs create --json {"name":"abc"} +>>> [CLI] jobs create --json {"name":"abc"} { "job_id":1111 } diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt index 211164174..437a55596 100644 --- a/integration/bundle/testdata/apps/bundle_deploy.txt +++ b/integration/bundle/testdata/apps/bundle_deploy.txt @@ -1,4 +1,4 @@ -Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ/files... Deploying resources... Updating deployment state... Deployment complete! diff --git a/integration/bundle/testdata/apps/bundle_validate.txt b/integration/bundle/testdata/apps/bundle_validate.txt index dc9016a0f..567fafd24 100644 --- a/integration/bundle/testdata/apps/bundle_validate.txt +++ b/integration/bundle/testdata/apps/bundle_validate.txt @@ -1,7 +1,7 @@ Name: basic Target: default Workspace: - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/$UNIQUE_PRJ + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ Validation OK! diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt index d7b8cede9..076e7618f 100644 --- a/integration/bundle/testdata/default_python/bundle_deploy.txt +++ b/integration/bundle/testdata/default_python/bundle_deploy.txt @@ -1,6 +1,6 @@ Building project_name_$UNIQUE_PRJ... Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files... Deploying resources... Updating deployment state... Deployment complete! diff --git a/integration/bundle/testdata/default_python/bundle_init.txt b/integration/bundle/testdata/default_python/bundle_init.txt index c2917ea4e..6ea0801ad 100644 --- a/integration/bundle/testdata/default_python/bundle_init.txt +++ b/integration/bundle/testdata/default_python/bundle_init.txt @@ -1,6 +1,6 @@ Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): $DATABRICKS_URL +Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): [DATABRICKS_URL] ✨ Your new project has been created in the 'project_name_$UNIQUE_PRJ' directory! diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 0b4c15764..450f01c46 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -22,54 +22,54 @@ "resources/project_name_$UNIQUE_PRJ.pipeline.yml" ], "workspace": { - "host": "$DATABRICKS_URL", + "host": "[DATABRICKS_URL]", "current_user": { "active": true, - "displayName": "$USERNAME", + "displayName": "[USERNAME]", "emails": [ { "primary": true, "type": "work", - "value": "$USERNAME" + "value": "[USERNAME]" } ], "groups": [ { - "$ref": "Groups/$USER.Groups[0]", + "$ref": "Groups/[USERGROUP]", "display": "team.engineering", "type": "direct", - "value": "$USER.Groups[0]" + "value": "[USERGROUP]" } ], - "id": "$USER.Id", + "id": "[USERID]", "name": { - "familyName": "$USERNAME", - "givenName": "$USERNAME" + "familyName": "[USERNAME]", + "givenName": "[USERNAME]" }, "schemas": [ "urn:ietf:params:scim:schemas:core:2.0:User", "urn:ietf:params:scim:schemas:extension:workspace:2.0:User" ], - "short_name": "$USERNAME", - "userName": "$USERNAME" + "short_name": "[USERNAME]", + "userName": "[USERNAME]" }, - "root_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev", - "file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files", - "resource_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/resources", - "artifact_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts", - "state_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state" + "root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev", + "file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/resources", + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state" }, "resources": { "jobs": { "project_name_$UNIQUE_PRJ_job": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", "email_notifications": { "on_failure": [ - "$USERNAME" + "[USERNAME]" ] }, "format": "MULTI_TASK", @@ -88,18 +88,18 @@ } ], "max_concurrent_runs": 4, - "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_job", + "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_job", "queue": { "enabled": true }, "tags": { - "dev": "$USERNAME" + "dev": "[USERNAME]" }, "tasks": [ { "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook" }, "task_key": "notebook_task" }, @@ -140,31 +140,31 @@ "unit": "DAYS" } }, - "url": "$DATABRICKS_URL/jobs/[NUMID]?o=[NUMID]" + "url": "[DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]" } }, "pipelines": { "project_name_$UNIQUE_PRJ_pipeline": { "catalog": "main", "configuration": { - "bundle.sourcePath": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src" + "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src" }, "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" }, "development": true, "id": "[UUID]", "libraries": [ { "notebook": { - "path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline" + "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline" } } ], - "name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline", + "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", "target": "project_name_$UNIQUE_PRJ_dev", - "url": "$DATABRICKS_URL/pipelines/[UUID]?o=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" } } }, @@ -174,12 +174,12 @@ ] }, "presets": { - "name_prefix": "[dev $USERNAME] ", + "name_prefix": "[dev [USERNAME]] ", "pipelines_development": true, "trigger_pause_status": "PAUSED", "jobs_max_concurrent_runs": 4, "tags": { - "dev": "$USERNAME" + "dev": "[USERNAME]" } } -} +} \ No newline at end of file diff --git a/integration/bundle/testdata/default_python/bundle_validate.txt b/integration/bundle/testdata/default_python/bundle_validate.txt index 578fd6494..c5c62b521 100644 --- a/integration/bundle/testdata/default_python/bundle_validate.txt +++ b/integration/bundle/testdata/default_python/bundle_validate.txt @@ -1,8 +1,8 @@ Name: project_name_$UNIQUE_PRJ Target: dev Workspace: - Host: $DATABRICKS_URL - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev Validation OK! diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 7077e611b..5bbba1be1 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -2,7 +2,6 @@ package testdiff import ( "encoding/json" - "fmt" "path/filepath" "regexp" "runtime" @@ -16,7 +15,7 @@ import ( ) const ( - testerName = "$USERNAME" + testerName = "[USERNAME]" ) var ( @@ -140,25 +139,25 @@ func PrepareReplacementsWorkspaceClient(t testutil.TestingT, r *ReplacementsCont t.Helper() // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") - r.Set("https://"+host, "$DATABRICKS_URL") - r.Set("http://"+host, "$DATABRICKS_URL") - r.Set(host, "$DATABRICKS_HOST") - r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") - r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") - r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") - r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") - r.Set(w.Config.Username, "$DATABRICKS_USERNAME") - r.SetPath(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") - r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") - r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") - r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") + r.Set("https://"+host, "[DATABRICKS_URL]") + r.Set("http://"+host, "[DATABRICKS_URL]") + r.Set(host, "[DATABRICKS_HOST]") + r.Set(w.Config.ClusterID, "[DATABRICKS_CLUSTER_ID]") + r.Set(w.Config.WarehouseID, "[DATABRICKS_WAREHOUSE_ID]") + r.Set(w.Config.ServerlessComputeID, "[DATABRICKS_SERVERLESS_COMPUTE_ID]") + r.Set(w.Config.AccountID, "[DATABRICKS_ACCOUNT_ID]") + r.Set(w.Config.Username, "[DATABRICKS_USERNAME]") + r.SetPath(w.Config.Profile, "[DATABRICKS_CONFIG_PROFILE]") + r.Set(w.Config.ConfigFile, "[DATABRICKS_CONFIG_FILE]") + r.Set(w.Config.GoogleServiceAccount, "[DATABRICKS_GOOGLE_SERVICE_ACCOUNT]") + r.Set(w.Config.AzureResourceID, "[DATABRICKS_AZURE_RESOURCE_ID]") r.Set(w.Config.AzureClientID, testerName) - r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") - r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") - r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") - r.SetPath(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") + r.Set(w.Config.AzureTenantID, "[ARM_TENANT_ID]") + r.Set(w.Config.AzureEnvironment, "[ARM_ENVIRONMENT]") + r.Set(w.Config.ClientID, "[DATABRICKS_CLIENT_ID]") + r.SetPath(w.Config.DatabricksCliPath, "[DATABRICKS_CLI_PATH]") // This is set to words like "path" that happen too frequently - // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") + // r.Set(w.Config.AuthType, "[DATABRICKS_AUTH_TYPE]") } func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { @@ -179,14 +178,14 @@ func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam. r.Set(iamutil.GetShortUserName(&u), testerName) - for ind, val := range u.Groups { - r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind)) + for _, val := range u.Groups { + r.Set(val.Value, "[USERGROUP]") } - r.Set(u.Id, "$USER.Id") + r.Set(u.Id, "[USERID]") - for ind, val := range u.Roles { - r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind)) + for _, val := range u.Roles { + r.Set(val.Value, "[USERROLE]") } } @@ -207,5 +206,5 @@ func PrepareReplacementsTemporaryDirectory(t testutil.TestingT, r *ReplacementsC func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) { t.Helper() - r.append(devVersionRegex, "$$DEV_VERSION") + r.append(devVersionRegex, "[DEV_VERSION]") } From 2eb9abb5ee5db2984de5d791f1d347f7c91e5e21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:00:52 +0100 Subject: [PATCH 048/105] Bump github.com/spf13/pflag from 1.0.5 to 1.0.6 (#2281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/pflag](https://github.com/spf13/pflag) from 1.0.5 to 1.0.6.
Release notes

Sourced from github.com/spf13/pflag's releases.

v1.0.6

What's Changed

New Contributors

Full Changelog: https://github.com/spf13/pflag/compare/v1.0.5...v1.0.6

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/pflag&package-manager=go_modules&previous-version=1.0.5&new-version=1.0.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 151133944..b3f11e918 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT github.com/spf13/cobra v1.8.1 // Apache 2.0 - github.com/spf13/pflag v1.0.5 // BSD-3-Clause + github.com/spf13/pflag v1.0.6 // BSD-3-Clause github.com/stretchr/testify v1.10.0 // MIT github.com/wI2L/jsondiff v0.6.1 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 diff --git a/go.sum b/go.sum index 3c7f20937..4e295a82d 100644 --- a/go.sum +++ b/go.sum @@ -147,8 +147,9 @@ github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0 github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= From 07efe830231d0b2c43079faeb1daee30781eb6cc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 4 Feb 2025 17:08:01 +0100 Subject: [PATCH 049/105] Use go-version-file instead of go-version in github actions (#2290) This minimizes number of places where we hard-code go version. Note, since we have go version specified without patch version ("1.23") in go.mod, it will use most recent in 1.23.x line. I think this is fine. https://github.com/actions/setup-go?tab=readme-ov-file#getting-go-version-from-the-gomod-file --- .github/workflows/push.yml | 6 +++--- .github/workflows/release-snapshot.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f27459baa..c41afc18c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -50,7 +50,7 @@ jobs: - name: Setup Go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: - go-version: 1.23.4 + go-version-file: go.mod - name: Setup Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 @@ -82,7 +82,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: - go-version: 1.23.4 + go-version-file: go.mod # Use different schema from regular job, to avoid overwriting the same key cache-dependency-path: | go.sum @@ -116,7 +116,7 @@ jobs: - name: Setup Go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: - go-version: 1.23.4 + go-version-file: go.mod # Use different schema from regular job, to avoid overwriting the same key cache-dependency-path: | go.sum diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index 548d93e90..8b4684eab 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -34,7 +34,7 @@ jobs: - name: Setup Go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: - go-version: 1.23.4 + go-version-file: go.mod # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d5811b19..fe5b4170b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: - name: Setup Go uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: - go-version: 1.23.4 + go-version-file: go.mod # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action From d86ad9189902d629cc85e0d79250a7301bb29aeb Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 4 Feb 2025 22:08:11 +0530 Subject: [PATCH 050/105] Allow test servers to return errors responses (#2291) ## Changes The APIs at Databricks when returning a non `200` status code will return a response body of the format: ``` { "error_code": "Error code", "message": "Human-readable error message." } ``` This PR adds the ability to stub non-200 status codes in the test server, allowing us to mock API errors from Databricks. ## Tests New test --- acceptance/acceptance_test.go | 8 +++-- acceptance/cmd_server_test.go | 4 +-- acceptance/config_test.go | 3 +- acceptance/server_test.go | 32 +++++++++---------- .../jobs/create-error/out.requests.txt | 1 + .../workspace/jobs/create-error/output.txt | 5 +++ acceptance/workspace/jobs/create-error/script | 1 + .../workspace/jobs/create-error/test.toml | 12 +++++++ libs/testserver/server.go | 11 +++---- 9 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 acceptance/workspace/jobs/create-error/out.requests.txt create mode 100644 acceptance/workspace/jobs/create-error/output.txt create mode 100644 acceptance/workspace/jobs/create-error/script create mode 100644 acceptance/workspace/jobs/create-error/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 871b8bd62..6e302fa96 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -261,8 +261,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) { - return stub.Response.Body, nil + server.Handle(stub.Pattern, func(req *http.Request) (any, int) { + statusCode := http.StatusOK + if stub.Response.StatusCode != 0 { + statusCode = stub.Response.StatusCode + } + return stub.Response.Body, statusCode }) } cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 9af63d0db..04d56c7d4 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(r *http.Request) (any, error) { + server.Handle("/", func(r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") @@ -40,7 +40,7 @@ func StartCmdServer(t *testing.T) *testserver.Server { exitcode = 1 } result["exitcode"] = exitcode - return result, nil + return result, http.StatusOK }) return server } diff --git a/acceptance/config_test.go b/acceptance/config_test.go index c7be223de..97be457e2 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -57,7 +57,8 @@ type ServerStub struct { // The response body to return. Response struct { - Body string + Body string + StatusCode int } } diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 4957a7668..a7695b21e 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -11,7 +11,7 @@ import ( ) func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -23,10 +23,10 @@ func AddHandlers(server *testserver.Server) { Name: "some-test-cluster-policy", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -34,10 +34,10 @@ func AddHandlers(server *testserver.Server) { InstancePoolId: "1234", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -49,32 +49,32 @@ func AddHandlers(server *testserver.Server) { ClusterId: "9876", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) { return iam.User{ Id: "1000012345", UserName: "tester@databricks.com", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) { return workspace.ObjectInfo{ ObjectId: 1001, ObjectType: "DIRECTORY", Path: "", ResourceId: "1001", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) { return workspace.WorkspaceObjectPermissions{ ObjectId: "1001", ObjectType: "DIRECTORY", @@ -88,10 +88,10 @@ func AddHandlers(server *testserver.Server) { }, }, }, - }, nil + }, http.StatusOK }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, error) { - return "{}", nil + server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) { + return "{}", http.StatusOK }) } diff --git a/acceptance/workspace/jobs/create-error/out.requests.txt b/acceptance/workspace/jobs/create-error/out.requests.txt new file mode 100644 index 000000000..b22876b70 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/out.requests.txt @@ -0,0 +1 @@ +{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create-error/output.txt b/acceptance/workspace/jobs/create-error/output.txt new file mode 100644 index 000000000..0e69eeb4b --- /dev/null +++ b/acceptance/workspace/jobs/create-error/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] jobs create --json {"name":"abc"} +Error: Invalid access token. + +Exit code: 1 diff --git a/acceptance/workspace/jobs/create-error/script b/acceptance/workspace/jobs/create-error/script new file mode 100644 index 000000000..9ff7b5b87 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/script @@ -0,0 +1 @@ +trace $CLI jobs create --json '{"name":"abc"}' diff --git a/acceptance/workspace/jobs/create-error/test.toml b/acceptance/workspace/jobs/create-error/test.toml new file mode 100644 index 000000000..b45bf77e5 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/test.toml @@ -0,0 +1,12 @@ +LocalOnly = true # request recording currently does not work with cloud environment +RecordRequests = true + +[[Server]] +Pattern = "POST /api/2.1/jobs/create" +Response.Body = ''' +{ + "error_code": "PERMISSION_DENIED", + "message": "Invalid access token." +} +''' +Response.StatusCode = 403 diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 2e8dbdfda..a751531ed 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -40,15 +40,11 @@ func New(t testutil.TestingT) *Server { } } -type HandlerFunc func(req *http.Request) (resp any, err error) +type HandlerFunc func(req *http.Request) (resp any, statusCode int) func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, err := handler(r) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + resp, statusCode := handler(r) if s.RecordRequests { body, err := io.ReadAll(r.Body) @@ -63,9 +59,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) var respBytes []byte - + var err error respString, ok := resp.(string) if ok { respBytes = []byte(respString) From 84b694f2a158186fa6d641e55e51af29761fa419 Mon Sep 17 00:00:00 2001 From: Simon Poltier Date: Tue, 4 Feb 2025 19:28:19 +0100 Subject: [PATCH 051/105] accept JSON includes (#2265) #2201 disabled using JSON as part of a bundle definition. I believe this was not intended. ## Changes Accept json files as includes, just as YAML files. ## Tests Covered by the tests in #2201 --- acceptance/bundle/includes/non_yaml_in_include/output.txt | 4 ++-- bundle/config/loader/process_root_includes.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/includes/non_yaml_in_include/output.txt b/acceptance/bundle/includes/non_yaml_in_include/output.txt index 6006ca14e..f5211cc4b 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/output.txt +++ b/acceptance/bundle/includes/non_yaml_in_include/output.txt @@ -1,7 +1,7 @@ -Error: Files in the 'include' configuration section must be YAML files. +Error: Files in the 'include' configuration section must be YAML or JSON files. in databricks.yml:5:4 -The file test.py in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead. +The file test.py in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead. Name: non_yaml_in_includes diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 198095742..69e6dd4e4 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -71,11 +71,11 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. continue } seen[rel] = true - if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" { + if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" && filepath.Ext(rel) != ".json" { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "Files in the 'include' configuration section must be YAML files.", - Detail: fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel), + Summary: "Files in the 'include' configuration section must be YAML or JSON files.", + Detail: fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel), Locations: b.Config.GetLocations(fmt.Sprintf("include[%d]", i)), }) continue From dcc61cd7636b4cdc647256bbd07aac8d4651da9e Mon Sep 17 00:00:00 2001 From: rikjansen-hu Date: Tue, 4 Feb 2025 19:30:02 +0100 Subject: [PATCH 052/105] Fix env variable for AzureCli local config (#2248) ## Changes Solves #1722 (current solution passes wrong variable) ## Tests None, this is a simple find-and-replace on a previous PR. Proof that this is the correct [variable](https://learn.microsoft.com/en-us/cli/azure/azure-cli-configuration#cli-configuration-file). This just passes the variable along to the Terraform environment, which [should](https://github.com/hashicorp/terraform/issues/25416) be picked up by Terraform. Co-authored-by: Rik Jansen --- bundle/deploy/terraform/init.go | 4 ++-- bundle/deploy/terraform/init_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index 5957611a4..a204222d0 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -101,9 +101,9 @@ var envCopy = []string{ // same auxiliary programs (e.g. `az`, or `gcloud`) as the CLI. "PATH", - // Include $AZURE_CONFIG_FILE in set of environment variables to pass along. + // Include $AZURE_CONFIG_DIR in set of environment variables to pass along. // This is set in Azure DevOps by the AzureCLI@2 task. - "AZURE_CONFIG_FILE", + "AZURE_CONFIG_DIR", // Include $TF_CLI_CONFIG_FILE to override terraform provider in development. // See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index c7a4ffe4a..4645ed007 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -292,7 +292,7 @@ func TestInheritEnvVars(t *testing.T) { t.Setenv("HOME", "/home/testuser") t.Setenv("PATH", "/foo:/bar") t.Setenv("TF_CLI_CONFIG_FILE", "/tmp/config.tfrc") - t.Setenv("AZURE_CONFIG_FILE", "/tmp/foo/bar") + t.Setenv("AZURE_CONFIG_DIR", "/tmp/foo/bar") ctx := context.Background() env := map[string]string{} @@ -301,7 +301,7 @@ func TestInheritEnvVars(t *testing.T) { assert.Equal(t, "/home/testuser", env["HOME"]) assert.Equal(t, "/foo:/bar", env["PATH"]) assert.Equal(t, "/tmp/config.tfrc", env["TF_CLI_CONFIG_FILE"]) - assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_FILE"]) + assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_DIR"]) } } From 2e1455841cced1337ad8ff8b456f57f217a9a74d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 4 Feb 2025 22:20:02 +0100 Subject: [PATCH 053/105] Update CODEOWNERS for cmd/labs (#2295) ## Changes The `CODEOWNERS` file must live in one of the directories specified in the [docs][docs], so the existing file under `cmd/labs` didn't work. This change moves the contents to the top-level file and includes @alexott as owner. [docs]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-file-location --- .github/CODEOWNERS | 1 + cmd/labs/CODEOWNERS | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 cmd/labs/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 76835de7d..3c3895bc1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,2 @@ * @pietern @andrewnester @shreyas-goenka @denik +cmd/labs @alexott @nfx diff --git a/cmd/labs/CODEOWNERS b/cmd/labs/CODEOWNERS deleted file mode 100644 index cc93a75e6..000000000 --- a/cmd/labs/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @nfx From 1678503cb04abfed7fdd1d595d93ed722c75330e Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 5 Feb 2025 10:01:51 +0100 Subject: [PATCH 054/105] Fix docs template (#2283) ## Changes Comment breaks markdown front-matter and description cannot be read ## Tests --- bundle/docsgen/output/reference.md | 3 ++- bundle/docsgen/output/resources.md | 3 ++- bundle/docsgen/templates/reference.md | 3 ++- bundle/docsgen/templates/resources.md | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index a2241d017..8a89d354b 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,8 +1,9 @@ - --- description: Configuration reference for databricks.yml --- + + # Configuration reference This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index ff80ee635..df7578c73 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,8 +1,9 @@ - --- description: Learn about resources supported by Databricks Asset Bundles and how to configure them. --- + + # resources allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md index a17d53315..345afc509 100644 --- a/bundle/docsgen/templates/reference.md +++ b/bundle/docsgen/templates/reference.md @@ -1,8 +1,9 @@ - --- description: Configuration reference for databricks.yml --- + + # Configuration reference This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md index fccfac47d..e9a6c8c5b 100644 --- a/bundle/docsgen/templates/resources.md +++ b/bundle/docsgen/templates/resources.md @@ -1,8 +1,9 @@ - --- description: Learn about resources supported by Databricks Asset Bundles and how to configure them. --- + + # resources allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). From 57b8d336e03cb2b9aa0c353d9cce7d85d5c72c7f Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:02:15 +0530 Subject: [PATCH 055/105] Add ability to record headers in acceptance tests (#2296) ## Changes HTTP headers like the User-Agent are an important part of our internal ETL pipelines. This PR adds the ability to validate the headers used in an HTTP request as part of our acceptance tests. ## Tests Modifying existing test. --- acceptance/acceptance_test.go | 3 +++ acceptance/config_test.go | 3 +++ .../workspace/jobs/create/out.requests.txt | 2 +- acceptance/workspace/jobs/create/test.toml | 17 ++++++++++++ libs/testdiff/replacement.go | 18 +++++++++++++ libs/testserver/server.go | 26 ++++++++++++++----- 6 files changed, 61 insertions(+), 8 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6e302fa96..f205217ff 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -156,6 +156,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient) testdiff.PrepareReplacementsUUID(t, &repls) testdiff.PrepareReplacementsDevVersion(t, &repls) + testdiff.PrepareReplacementSdkVersion(t, &repls) + testdiff.PrepareReplacementsGoVersion(t, &repls) testDirs := getTests(t) require.NotEmpty(t, testDirs) @@ -253,6 +255,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if len(config.Server) > 0 || config.RecordRequests { server = testserver.New(t) server.RecordRequests = config.RecordRequests + server.IncludeRequestHeaders = config.IncludeRequestHeaders // If no custom server stubs are defined, add the default handlers. if len(config.Server) == 0 { diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 97be457e2..e24a683e7 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -47,6 +47,9 @@ type TestConfig struct { // Record the requests made to the server and write them as output to // out.requests.txt RecordRequests bool + + // List of request headers to include when recording requests. + IncludeRequestHeaders []string } type ServerStub struct { diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index b22876b70..4a85c4c43 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index e69569c18..1fd9b3cec 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,5 +1,6 @@ LocalOnly = true # request recording currently does not work with cloud environment RecordRequests = true +IncludeRequestHeaders = ["Authorization", "User-Agent"] [[Server]] Pattern = "POST /api/2.1/jobs/create" @@ -8,3 +9,19 @@ Response.Body = ''' "job_id": 1111 } ''' + +[[Repls]] +Old = "(linux|darwin|windows)" +New = "[OS]" + +[[Repls]] +Old = " upstream/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " upstream-version/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " cicd/[A-Za-z0-9.-]+" +New = "" diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 5bbba1be1..d4d5eb27b 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" + "golang.org/x/mod/semver" ) const ( @@ -208,3 +209,20 @@ func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) t.Helper() r.append(devVersionRegex, "[DEV_VERSION]") } + +func PrepareReplacementSdkVersion(t testutil.TestingT, r *ReplacementsContext) { + t.Helper() + r.Set(databricks.Version(), "[SDK_VERSION]") +} + +func goVersion() string { + gv := runtime.Version() + ssv := strings.ReplaceAll(gv, "go", "v") + sv := semver.Canonical(ssv) + return strings.TrimPrefix(sv, "v") +} + +func PrepareReplacementsGoVersion(t testutil.TestingT, r *ReplacementsContext) { + t.Helper() + r.Set(goVersion(), "[GO_VERSION]") +} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index a751531ed..5e3efe1c5 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "github.com/stretchr/testify/assert" @@ -17,15 +18,17 @@ type Server struct { t testutil.TestingT - RecordRequests bool + RecordRequests bool + IncludeRequestHeaders []string Requests []Request } type Request struct { - Method string `json:"method"` - Path string `json:"path"` - Body any `json:"body"` + Headers map[string]string `json:"headers,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -50,10 +53,19 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { body, err := io.ReadAll(r.Body) assert.NoError(s.t, err) + headers := make(map[string]string) + for k, v := range r.Header { + if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) { + continue + } + headers[k] = v[0] + } + s.Requests = append(s.Requests, Request{ - Method: r.Method, - Path: r.URL.Path, - Body: json.RawMessage(body), + Headers: headers, + Method: r.Method, + Path: r.URL.Path, + Body: json.RawMessage(body), }) } From 5c90752797d4cd2e49bdda5a05fb35e9a59d5edc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Feb 2025 11:53:36 +0000 Subject: [PATCH 056/105] acc: Added acceptance test for CLI commands inside bundle with and without profile flag (#2270) ## Changes This encodes existing behaviour in CLI as reported here: #1358 --- .../auth/bundle_and_profile/.databrickscfg | 5 +++ .../auth/bundle_and_profile/databricks.yml | 14 ++++++++ acceptance/auth/bundle_and_profile/output.txt | 32 +++++++++++++++++++ acceptance/auth/bundle_and_profile/script | 30 +++++++++++++++++ acceptance/auth/bundle_and_profile/test.toml | 8 +++++ 5 files changed, 89 insertions(+) create mode 100644 acceptance/auth/bundle_and_profile/.databrickscfg create mode 100644 acceptance/auth/bundle_and_profile/databricks.yml create mode 100644 acceptance/auth/bundle_and_profile/output.txt create mode 100644 acceptance/auth/bundle_and_profile/script create mode 100644 acceptance/auth/bundle_and_profile/test.toml diff --git a/acceptance/auth/bundle_and_profile/.databrickscfg b/acceptance/auth/bundle_and_profile/.databrickscfg new file mode 100644 index 000000000..628505286 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/.databrickscfg @@ -0,0 +1,5 @@ +[DEFAULT] +host = $DATABRICKS_HOST + +[profile_name] +host = https://test@non-existing-subdomain.databricks.com diff --git a/acceptance/auth/bundle_and_profile/databricks.yml b/acceptance/auth/bundle_and_profile/databricks.yml new file mode 100644 index 000000000..975661395 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-auth + +workspace: + host: $DATABRICKS_HOST + +targets: + dev: + default: true + workspace: + host: $DATABRICKS_HOST + prod: + workspace: + host: https://bar.com diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt new file mode 100644 index 000000000..022b3148d --- /dev/null +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -0,0 +1,32 @@ + +=== Inside the bundle, no flags +>>> errcode [CLI] current-user me +"[USERNAME]" + +=== Inside the bundle, target flags +>>> errcode [CLI] current-user me -t dev +"[USERNAME]" + +=== Inside the bundle, target and matching profile +>>> errcode [CLI] current-user me -t dev -p DEFAULT +"[USERNAME]" + +=== Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail +>>> errcode [CLI] current-user me -p profile_name +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] + +Exit code: 1 + +=== Inside the bundle, target and not matching profile +>>> errcode [CLI] current-user me -t dev -p profile_name +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] + +Exit code: 1 + +=== Outside the bundle, no flags +>>> errcode [CLI] current-user me +"[USERNAME]" + +=== Outside the bundle, profile flag +>>> errcode [CLI] current-user me -p profile_name +"[USERNAME]" diff --git a/acceptance/auth/bundle_and_profile/script b/acceptance/auth/bundle_and_profile/script new file mode 100644 index 000000000..b37f5e01d --- /dev/null +++ b/acceptance/auth/bundle_and_profile/script @@ -0,0 +1,30 @@ +# Replace placeholder with an actual host URL +envsubst < databricks.yml > out.yml && mv out.yml databricks.yml +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +host=$DATABRICKS_HOST +unset DATABRICKS_HOST + +title "Inside the bundle, no flags" +trace errcode $CLI current-user me | jq .userName + +title "Inside the bundle, target flags" +trace errcode $CLI current-user me -t dev | jq .userName + +title "Inside the bundle, target and matching profile" +trace errcode $CLI current-user me -t dev -p DEFAULT | jq .userName + +title "Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail" +trace errcode $CLI current-user me -p profile_name | jq .userName + +title "Inside the bundle, target and not matching profile" +trace errcode $CLI current-user me -t dev -p profile_name + +cd .. +export DATABRICKS_HOST=$host +title "Outside the bundle, no flags" +trace errcode $CLI current-user me | jq .userName + +title "Outside the bundle, profile flag" +trace errcode $CLI current-user me -p profile_name | jq .userName diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml new file mode 100644 index 000000000..b20190ca5 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/test.toml @@ -0,0 +1,8 @@ +Badness = "When -p flag is used inside the bundle folder for any CLI commands, CLI use bundle host anyway instead of profile one" + +# Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix +# In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST +# This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST +[[Repls]] +Old='DATABRICKS_HOST' +New='DATABRICKS_URL' From 27caf413f2d601de8d81d6cc3fb2614ff1bf354c Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 5 Feb 2025 14:24:15 +0100 Subject: [PATCH 057/105] Add support for extras to the labs CLI (#2288) ## Changes Added support for extras / optional Python dependencies in the labs CLI. Added new `extras` field under install. Example: ```yaml install: script: install.py extras: cli ``` Resolves: #2257 ## Tests Manual test --- cmd/labs/project/installer.go | 5 +++++ cmd/labs/project/schema.json | 5 +++++ .../.databricks/labs/blueprint/lib/labs.yml | 1 + 3 files changed, 11 insertions(+) diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 05f7d68aa..2e42ce43d 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -32,6 +32,7 @@ type hook struct { RequireDatabricksConnect bool `yaml:"require_databricks_connect,omitempty"` MinRuntimeVersion string `yaml:"min_runtime_version,omitempty"` WarehouseTypes whTypes `yaml:"warehouse_types,omitempty"` + Extras string `yaml:"extras,omitempty"` } func (h *hook) RequireRunningCluster() bool { @@ -258,6 +259,10 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr } } feedback <- "Installing Python library dependencies" + if i.Installer.Extras != "" { + // install main and optional dependencies + return i.installPythonDependencies(ctx, fmt.Sprintf(".[%s]", i.Installer.Extras)) + } return i.installPythonDependencies(ctx, ".") } diff --git a/cmd/labs/project/schema.json b/cmd/labs/project/schema.json index a779b15e4..7aa65813c 100644 --- a/cmd/labs/project/schema.json +++ b/cmd/labs/project/schema.json @@ -42,6 +42,11 @@ }, "warehouse_types": { "enum": [ "PRO", "CLASSIC", "TYPE_UNSPECIFIED" ] + }, + "extras": { + "type": "string", + "pattern": "^([^,]+)(,([^,]+))*$", + "default": "" } } }, diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml index 0ac4bf826..b8a0e695e 100644 --- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml +++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml @@ -8,6 +8,7 @@ install: warehouse_types: - PRO script: install.py + extras: "" entrypoint: main.py min_python: 3.9 commands: From e0903fbd3712c252874f193076a88d6a83cd966e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 5 Feb 2025 15:58:29 +0100 Subject: [PATCH 058/105] Include 'go mod tidy' into 'make' and 'make tidy' (#2298) Apparently, it's not part of golangci-lint, so you can send PRs that fail this check on CI. --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e18727934..7da7e4789 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -default: vendor fmt lint +default: vendor fmt lint tidy PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... . @@ -9,6 +9,10 @@ GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped lint: golangci-lint run --fix +tidy: + # not part of golangci-lint, apparently + go mod tidy + lintcheck: golangci-lint run ./... @@ -59,4 +63,4 @@ integration: vendor integration-short: vendor $(INTEGRATION) -short -.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs +.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs From 27eb0c40725aa978191ce3fe43eb07993fd6cfe4 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 6 Feb 2025 20:27:55 +0100 Subject: [PATCH 059/105] Allow 'any' examples in JSON schema (#2289) ## Changes 1. Allow `any` examples in json-schema type since we have many of them in open api spec 2. Fix issue with missing overrides annotations when re-generating the schema ## Tests --- bundle/docsgen/main.go | 2 +- bundle/docsgen/nodes.go | 4 +- bundle/docsgen/refs.go | 12 +- .../schema/annotations_openapi_overrides.yml | 161 +++++++++--------- bundle/internal/schema/parser.go | 10 +- libs/jsonschema/schema.go | 2 +- 6 files changed, 101 insertions(+), 90 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index ad737feea..84bf4779f 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.MarkdownDescription = a.MarkdownDescription } if a.MarkdownExamples != "" { - s.Examples = []any{a.MarkdownExamples} + s.Examples = []string{a.MarkdownExamples} } } diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go index 68ed86450..61d2c21cc 100644 --- a/bundle/docsgen/nodes.go +++ b/bundle/docsgen/nodes.go @@ -220,9 +220,9 @@ func isCycleField(field string) bool { } func getExample(v *jsonschema.Schema) string { - examples := v.Examples + examples := getExamples(v.Examples) if len(examples) == 0 { return "" } - return examples[0].(string) + return examples[0] } diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go index ca45e6ab2..7a4451129 100644 --- a/bundle/docsgen/refs.go +++ b/bundle/docsgen/refs.go @@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j node := s description := s.Description markdownDescription := s.MarkdownDescription - examples := s.Examples + examples := getExamples(s.Examples) for node.Reference != nil { ref := getRefType(node) @@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j markdownDescription = newNode.MarkdownDescription } if len(examples) == 0 { - examples = newNode.Examples + examples = getExamples(newNode.Examples) } node = newNode @@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j return &newNode } +func getExamples(examples any) []string { + typedExamples, ok := examples.([]string) + if !ok { + return []string{} + } + return typedExamples +} + func getRefType(node *jsonschema.Schema) string { if node.Reference == nil { return "" diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 912a4fda0..585886313 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). - "markdown_examples": |- The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: @@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). - "embed_credentials": "description": |- PLACEHOLDER @@ -356,7 +354,6 @@ github.com/databricks/cli/bundle/config/resources.Volume: - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). - "markdown_examples": |- The following example creates a volume with the key `my_volume`: @@ -376,6 +373,42 @@ github.com/databricks/cli/bundle/config/resources.Volume: "volume_type": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "deployment_artifacts": + "description": |- + PLACEHOLDER + "deployment_id": + "description": |- + PLACEHOLDER + "mode": + "description": |- + PLACEHOLDER + "source_code_path": + "description": |- + PLACEHOLDER + "status": + "description": |- + PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: + "source_code_path": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/apps.AppResource: "job": "description": |- @@ -389,6 +422,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource: "sql_warehouse": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: + "key": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER + "scope": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: + "name": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: + "message": + "description": |- + PLACEHOLDER + "state": {} github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: "availability": "description": |- @@ -473,85 +549,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: - "create_time": - "description": |- - PLACEHOLDER - "creator": - "description": |- - PLACEHOLDER - "deployment_artifacts": - "description": |- - PLACEHOLDER - "deployment_id": - "description": |- - PLACEHOLDER - "mode": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "status": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: - "source_code_path": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: - "key": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER - "scope": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: - "name": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: - "message": - "description": |- - PLACEHOLDER - "state": github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 50e69e7c8..ca8c27d4c 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "os" @@ -9,8 +10,9 @@ import ( "strings" "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" - "gopkg.in/yaml.v3" ) type Components struct { @@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if err != nil { return err } - err = yaml.Unmarshal(b, &overrides) + overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b)) + if err != nil { + return err + } + err = convert.ToTyped(&overrides, overridesDyn) if err != nil { return err } diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 5028bb0d7..85f6a0328 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -79,7 +79,7 @@ type Schema struct { // Examples of the value for properties in the schema. // https://json-schema.org/understanding-json-schema/reference/annotations - Examples []any `json:"examples,omitempty"` + Examples any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string. From 75127fe42eb5fb2221b49c39977de32e3441ae0d Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 7 Feb 2025 11:26:20 +0100 Subject: [PATCH 060/105] Extend testserver for deployment (#2299) ## Changes Extend testserver for bundle deployment: - Allocate a new workspace per test case to isolate test cases from each other - Support jobs get/list/create - Support creation and listing of workspace files ## Tests Using existing acceptance tests --- acceptance/acceptance_test.go | 28 ++- acceptance/bundle/scripts/output.txt | 14 +- acceptance/cmd_server_test.go | 2 +- acceptance/server_test.go | 107 +++++++++-- .../workspace/jobs/create/out.requests.txt | 2 +- libs/testserver/fake_workspace.go | 169 ++++++++++++++++++ libs/testserver/server.go | 50 +++++- 7 files changed, 332 insertions(+), 40 deletions(-) create mode 100644 libs/testserver/fake_workspace.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f205217ff..4c4404d55 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -19,6 +19,8 @@ import ( "time" "unicode/utf8" + "github.com/google/uuid" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testdiff" @@ -123,7 +125,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { AddHandlers(defaultServer) // Redirect API access to local server: t.Setenv("DATABRICKS_HOST", defaultServer.URL) - t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() // Do not read user's ~/.databrickscfg @@ -146,7 +147,15 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") - workspaceClient, err := databricks.NewWorkspaceClient() + var config databricks.Config + if cloudEnv == "" { + // use fake token for local tests + config = databricks.Config{Token: "dbapi1234"} + } else { + // non-local tests rely on environment variables + config = databricks.Config{} + } + workspaceClient, err := databricks.NewWorkspaceClient(&config) require.NoError(t, err) user, err := workspaceClient.CurrentUser.Me(ctx) @@ -264,7 +273,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(req *http.Request) (any, int) { + server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { statusCode := http.StatusOK if stub.Response.StatusCode != 0 { statusCode = stub.Response.StatusCode @@ -285,6 +294,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } + // Each local test should use a new token that will result into a new fake workspace, + // so that test don't interfere with each other. + if cloudEnv == "" { + tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "") + token := "dbapi" + tokenSuffix + cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token) + repls.Set(token, "[DATABRICKS_TOKEN]") + } + // Write combined output to a file out, err := os.Create(filepath.Join(tmpDir, "output.txt")) require.NoError(t, err) @@ -303,8 +321,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont reqJson, err := json.Marshal(req) require.NoError(t, err) - line := fmt.Sprintf("%s\n", reqJson) - _, err = f.WriteString(line) + reqJsonWithRepls := repls.Replace(string(reqJson)) + _, err = f.WriteString(reqJsonWithRepls + "\n") require.NoError(t, err) } diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt index 2deedb0e7..68afb2fec 100644 --- a/acceptance/bundle/scripts/output.txt +++ b/acceptance/bundle/scripts/output.txt @@ -42,11 +42,9 @@ from myscript.py 0 postbuild: hello stderr! Executing 'predeploy' script from myscript.py 0 predeploy: hello stdout! from myscript.py 0 predeploy: hello stderr! -Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME]. -Please make sure the current user or one of their groups is listed under the permissions of this bundle. -For assistance, contact the owners of this project. -They may need to redeploy the bundle to apply the new permissions. -Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. - - -Exit code: 1 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/scripts/default/files... +Deploying resources... +Deployment complete! +Executing 'postdeploy' script +from myscript.py 0 postdeploy: hello stdout! +from myscript.py 0 postdeploy: hello stderr! diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 04d56c7d4..0166dfe32 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(r *http.Request) (any, int) { + server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/server_test.go b/acceptance/server_test.go index a7695b21e..d21ab66e8 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -1,17 +1,23 @@ package acceptance_test import ( + "bytes" + "encoding/json" + "fmt" "net/http" - "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + + "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/workspace" ) func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -26,7 +32,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -37,7 +43,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -52,31 +58,74 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return iam.User{ Id: "1000012345", UserName: "tester@databricks.com", }, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) { - return workspace.ObjectInfo{ - ObjectId: 1001, - ObjectType: "DIRECTORY", - Path: "", - ResourceId: "1001", - }, http.StatusOK + server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceGetStatus(path) }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) { + server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := workspace.Mkdirs{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceMkdirs(request) + }) + + server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceExport(path) + }) + + server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + recursiveStr := r.URL.Query().Get("recursive") + var recursive bool + + if recursiveStr == "true" { + recursive = true + } else { + recursive = false + } + + return fakeWorkspace.WorkspaceDelete(path, recursive) + }) + + server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.PathValue("path") + + body := new(bytes.Buffer) + _, err := body.ReadFrom(r.Body) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes()) + }) + + server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + objectId := r.PathValue("objectId") + return workspace.WorkspaceObjectPermissions{ - ObjectId: "1001", + ObjectId: objectId, ObjectType: "DIRECTORY", AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ { @@ -91,7 +140,29 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) { - return "{}", http.StatusOK + server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := jobs.CreateJob{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.JobsCreate(request) + }) + + server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + jobId := r.URL.Query().Get("job_id") + + return fakeWorkspace.JobsGet(jobId) + }) + + server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return fakeWorkspace.JobsList() }) } + +func internalError(err error) (any, int) { + return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError +} diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 4a85c4c43..60977e3e3 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go new file mode 100644 index 000000000..c3e4f9a71 --- /dev/null +++ b/libs/testserver/fake_workspace.go @@ -0,0 +1,169 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// FakeWorkspace holds a state of a workspace for acceptance tests. +type FakeWorkspace struct { + directories map[string]bool + files map[string][]byte + // normally, ids are not sequential, but we make them sequential for deterministic diff + nextJobId int64 + jobs map[int64]jobs.Job +} + +func NewFakeWorkspace() *FakeWorkspace { + return &FakeWorkspace{ + directories: map[string]bool{ + "/Workspace": true, + }, + files: map[string][]byte{}, + jobs: map[int64]jobs.Job{}, + nextJobId: 1, + } +} + +func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) { + if s.directories[path] { + return workspace.ObjectInfo{ + ObjectType: "DIRECTORY", + Path: path, + }, http.StatusOK + } else if _, ok := s.files[path]; ok { + return workspace.ObjectInfo{ + ObjectType: "FILE", + Path: path, + Language: "SCALA", + }, http.StatusOK + } else { + return workspace.ObjectInfo{}, http.StatusNotFound + } +} + +func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) { + s.directories[request.Path] = true + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) { + file := s.files[path] + + if file == nil { + return nil, http.StatusNotFound + } + + return file, http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) { + if !recursive { + s.files[path] = nil + } else { + for key := range s.files { + if strings.HasPrefix(key, path) { + s.files[key] = nil + } + } + } + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + s.files[path] = body + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) { + jobId := s.nextJobId + s.nextJobId++ + + jobSettings := jobs.JobSettings{} + err := jsonConvert(request, &jobSettings) + if err != nil { + return internalError(err) + } + + s.jobs[jobId] = jobs.Job{ + JobId: jobId, + Settings: &jobSettings, + } + + return jobs.CreateResponse{JobId: jobId}, http.StatusOK +} + +func (s *FakeWorkspace) JobsGet(jobId string) (any, int) { + id := jobId + + jobIdInt, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return internalError(fmt.Errorf("failed to parse job id: %s", err)) + } + + job, ok := s.jobs[jobIdInt] + if !ok { + return jobs.Job{}, http.StatusNotFound + } + + return job, http.StatusOK +} + +func (s *FakeWorkspace) JobsList() (any, int) { + list := make([]jobs.BaseJob, 0, len(s.jobs)) + for _, job := range s.jobs { + baseJob := jobs.BaseJob{} + err := jsonConvert(job, &baseJob) + if err != nil { + return internalError(fmt.Errorf("failed to convert job to base job: %w", err)) + } + + list = append(list, baseJob) + } + + // sort to have less non-determinism in tests + sort.Slice(list, func(i, j int) bool { + return list[i].JobId < list[j].JobId + }) + + return jobs.ListJobsResponse{ + Jobs: list, + }, http.StatusOK +} + +// jsonConvert saves input to a value pointed by output +func jsonConvert(input, output any) error { + writer := new(bytes.Buffer) + encoder := json.NewEncoder(writer) + err := encoder.Encode(input) + if err != nil { + return fmt.Errorf("failed to encode: %w", err) + } + + decoder := json.NewDecoder(writer) + err = decoder.Decode(output) + if err != nil { + return fmt.Errorf("failed to decode: %w", err) + } + + return nil +} + +func internalError(err error) (string, int) { + return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError +} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5e3efe1c5..ffb83a49c 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -6,6 +6,8 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" + "sync" "github.com/stretchr/testify/assert" @@ -18,6 +20,9 @@ type Server struct { t testutil.TestingT + fakeWorkspaces map[string]*FakeWorkspace + mu *sync.Mutex + RecordRequests bool IncludeRequestHeaders []string @@ -37,17 +42,36 @@ func New(t testutil.TestingT) *Server { t.Cleanup(server.Close) return &Server{ - Server: server, - Mux: mux, - t: t, + Server: server, + Mux: mux, + t: t, + mu: &sync.Mutex{}, + fakeWorkspaces: map[string]*FakeWorkspace{}, } } -type HandlerFunc func(req *http.Request) (resp any, statusCode int) +type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, statusCode := handler(r) + // For simplicity we process requests sequentially. It's fast enough because + // we don't do any IO except reading and writing request/response bodies. + s.mu.Lock() + defer s.mu.Unlock() + + // Each test uses unique DATABRICKS_TOKEN, we simulate each token having + // it's own fake fakeWorkspace to avoid interference between tests. + var fakeWorkspace *FakeWorkspace = nil + token := getToken(r) + if token != "" { + if _, ok := s.fakeWorkspaces[token]; !ok { + s.fakeWorkspaces[token] = NewFakeWorkspace() + } + + fakeWorkspace = s.fakeWorkspaces[token] + } + + resp, statusCode := handler(fakeWorkspace, r) if s.RecordRequests { body, err := io.ReadAll(r.Body) @@ -75,9 +99,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { var respBytes []byte var err error - respString, ok := resp.(string) - if ok { + if respString, ok := resp.(string); ok { respBytes = []byte(respString) + } else if respBytes0, ok := resp.([]byte); ok { + respBytes = respBytes0 } else { respBytes, err = json.MarshalIndent(resp, "", " ") if err != nil { @@ -92,3 +117,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } }) } + +func getToken(r *http.Request) string { + header := r.Header.Get("Authorization") + prefix := "Bearer " + + if !strings.HasPrefix(header, prefix) { + return "" + } + + return header[len(prefix):] +} From 54e16d5f62ccd169c5f64319f3b174f6dc58e326 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 12:29:40 +0100 Subject: [PATCH 061/105] Always print warnings and errors; clean up format (#2213) ## Changes - Print warnings and errors by default. - Fix ErrAlreadyPrinted not to be logged at Error level. - Format log messages as "Warn: message" instead of "WARN" to make it more readable and in-line with the rest of the output. - Only print attributes (pid, mutator, etc) and time when the overall level is debug (so --debug output has not changed much). ## Tests - Existing acceptance tests show how warning messages appear in various test case. - Added new test for `--debug` output. - Add sort_lines.py helper to avoid dependency on 'sort' which is locale-sensitive. --- acceptance/bin/sort_lines.py | 10 ++ acceptance/bundle/debug/databricks.yml | 2 + .../bundle/debug/out.stderr.parallel.txt | 15 +++ acceptance/bundle/debug/out.stderr.txt | 92 +++++++++++++++++++ acceptance/bundle/debug/output.txt | 7 ++ acceptance/bundle/debug/script | 4 + acceptance/bundle/debug/test.toml | 18 ++++ acceptance/bundle/git-permerror/output.txt | 9 ++ acceptance/bundle/git-permerror/test.toml | 2 +- bundle/artifacts/upload.go | 2 +- bundle/mutator.go | 2 +- bundle/mutator_read_only.go | 2 +- cmd/root/root.go | 11 ++- libs/flags/log_level_flag.go | 2 +- libs/flags/log_level_flag_test.go | 4 +- libs/log/handler/friendly.go | 60 ++++++------ 16 files changed, 206 insertions(+), 36 deletions(-) create mode 100755 acceptance/bin/sort_lines.py create mode 100644 acceptance/bundle/debug/databricks.yml create mode 100644 acceptance/bundle/debug/out.stderr.parallel.txt create mode 100644 acceptance/bundle/debug/out.stderr.txt create mode 100644 acceptance/bundle/debug/output.txt create mode 100644 acceptance/bundle/debug/script create mode 100644 acceptance/bundle/debug/test.toml diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py new file mode 100755 index 000000000..9ac87feee --- /dev/null +++ b/acceptance/bin/sort_lines.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +""" +Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH. +""" + +import sys + +lines = sys.stdin.readlines() +lines.sort() +sys.stdout.write("".join(lines)) diff --git a/acceptance/bundle/debug/databricks.yml b/acceptance/bundle/debug/databricks.yml new file mode 100644 index 000000000..2c9dd3c90 --- /dev/null +++ b/acceptance/bundle/debug/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: debug diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt new file mode 100644 index 000000000..7dd770068 --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.parallel.txt @@ -0,0 +1,15 @@ +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt new file mode 100644 index 000000000..9cac8bb2b --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.txt @@ -0,0 +1,92 @@ +10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug" +10:07:59 Debug: Found bundle root at [TMPDIR] (file [TMPDIR]/databricks.yml) pid=12345 +10:07:59 Debug: Apply pid=12345 mutator=load +10:07:59 Info: Phase: load pid=12345 mutator=load +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EntryPoint +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=VerifyCliVersion +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EnvironmentsToTargets +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ComputeIdToClusterId +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=InitializeVariables +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=DefineDefaultTarget(default) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=PythonMutator(load) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=validate:unique_resource_keys +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget mutator=SelectTarget(default) +10:07:59 Debug: Apply pid=12345 mutator= +10:07:59 Debug: Apply pid=12345 mutator=initialize +10:07:59 Info: Phase: initialize pid=12345 mutator=initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=validate:AllResourcesHaveValues +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteSyncPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncDefaultPath +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncInferRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser +10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me +< HTTP/1.1 200 OK +< { +< "id": "[USERID]", +< "userName": "[USERNAME]" +< } pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser sdk=true +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=LoadGitDetails +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplySourceLinkedDeploymentPreset +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefineDefaultWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultWorkspacePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PrependWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetVariables +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(init) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(load_resources) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(apply_mutators) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveResourceReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobParameters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobTasks +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergePipelineClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeApps +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CaptureSchemaDependency +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CheckPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetRunAs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=OverrideCompute +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureDashboardDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureVolumeDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ProcessTargetMode +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyPresets +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultQueueing +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandPipelineGlobPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureWSFS +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=TranslatePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonWrapperWarning +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=apps.Validate +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ValidateSharedRootPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyBundlePermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=FilterCurrentUserFromPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotateJobs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotatePipelines +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: Apply pid=12345 mutator=validate +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 404 Not Found +10:07:59 Debug: POST /api/2.0/workspace/mkdirs +> { +> "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +> } +< HTTP/1.1 200 OK +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 200 OK +< { +< "object_type": "DIRECTORY", +< "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +10:07:59 Info: completed execution pid=12345 exit_code=0 diff --git a/acceptance/bundle/debug/output.txt b/acceptance/bundle/debug/output.txt new file mode 100644 index 000000000..ed72b360e --- /dev/null +++ b/acceptance/bundle/debug/output.txt @@ -0,0 +1,7 @@ +Name: debug +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/debug/default + +Validation OK! diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script new file mode 100644 index 000000000..1b4cfc8f0 --- /dev/null +++ b/acceptance/bundle/debug/script @@ -0,0 +1,4 @@ +$CLI bundle validate --debug 2> full.stderr.txt +grep -vw parallel full.stderr.txt > out.stderr.txt +grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt +rm full.stderr.txt diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml new file mode 100644 index 000000000..bb0fcb395 --- /dev/null +++ b/acceptance/bundle/debug/test.toml @@ -0,0 +1,18 @@ +LocalOnly = true + +[[Repls]] +# The keys are unsorted and also vary per OS +Old = 'Environment variables for Terraform: ([A-Z_ ,]+) ' +New = 'Environment variables for Terraform: ...redacted... ' + +[[Repls]] +Old = 'pid=[0-9]+' +New = 'pid=12345' + +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = '10:07:59' + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt index 03ab93442..730e8255b 100644 --- a/acceptance/bundle/git-permerror/output.txt +++ b/acceptance/bundle/git-permerror/output.txt @@ -3,6 +3,7 @@ >>> chmod 000 .git >>> [CLI] bundle validate +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied Name: git-permerror @@ -16,6 +17,7 @@ Found 1 error Exit code: 1 >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -25,6 +27,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -39,11 +42,15 @@ Exit code: 1 >>> chmod 000 .git/HEAD >>> [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } @@ -54,6 +61,7 @@ Exit code: 1 >>> chmod 000 .git/config >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -63,6 +71,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml index 3f96e551c..15305cff1 100644 --- a/acceptance/bundle/git-permerror/test.toml +++ b/acceptance/bundle/git-permerror/test.toml @@ -1,4 +1,4 @@ -Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct" +Badness = "inferred flag is set to true incorrect; bundle_root_path is not correct; Warn and Error talk about the same; Warn goes to stderr, Error goes to stdout (for backward compat); Warning about permissions repeated twice" [GOOS] # This test relies on chmod which does not work on Windows diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index c69939e8c..d4625d85d 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -29,7 +29,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics // We intentionally ignore the error because it is not critical to the deployment err := client.Delete(ctx, ".", filer.DeleteRecursively) if err != nil { - log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err) + log.Debugf(ctx, "failed to delete %s: %v", uploadPath, err) } err = client.Mkdir(ctx, ".") diff --git a/bundle/mutator.go b/bundle/mutator.go index 6c9968aac..16ef79ee7 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -42,7 +42,7 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics { // such that they are not logged multiple times. // If this is done, we can omit this block. if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go index ee4e36e0f..700a90d8d 100644 --- a/bundle/mutator_read_only.go +++ b/bundle/mutator_read_only.go @@ -22,7 +22,7 @@ func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) di log.Debugf(ctx, "ApplyReadOnly") diags := m.Apply(ctx, rb) if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/cmd/root/root.go b/cmd/root/root.go index 3b37d0176..d7adf47f4 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -114,10 +114,15 @@ func Execute(ctx context.Context, cmd *cobra.Command) error { if err == nil { logger.Info("completed execution", slog.String("exit_code", "0")) - } else { - logger.Error("failed execution", + } else if errors.Is(err, ErrAlreadyPrinted) { + logger.Debug("failed execution", slog.String("exit_code", "1"), - slog.String("error", err.Error())) + ) + } else { + logger.Info("failed execution", + slog.String("exit_code", "1"), + slog.String("error", err.Error()), + ) } } diff --git a/libs/flags/log_level_flag.go b/libs/flags/log_level_flag.go index 836d84b70..82e2abc4c 100644 --- a/libs/flags/log_level_flag.go +++ b/libs/flags/log_level_flag.go @@ -25,7 +25,7 @@ type LogLevelFlag struct { func NewLogLevelFlag() LogLevelFlag { return LogLevelFlag{ - l: log.LevelDisabled, + l: log.LevelWarn, } } diff --git a/libs/flags/log_level_flag_test.go b/libs/flags/log_level_flag_test.go index 11a50bc45..c81f90d18 100644 --- a/libs/flags/log_level_flag_test.go +++ b/libs/flags/log_level_flag_test.go @@ -10,8 +10,8 @@ import ( func TestLogLevelFlagDefault(t *testing.T) { f := NewLogLevelFlag() - assert.Equal(t, log.LevelDisabled, f.Level()) - assert.Equal(t, "disabled", f.String()) + assert.Equal(t, log.LevelWarn, f.Level()) + assert.Equal(t, "warn", f.String()) } func TestLogLevelFlagSetValid(t *testing.T) { diff --git a/libs/log/handler/friendly.go b/libs/log/handler/friendly.go index 33b88a9e2..5c60eb13d 100644 --- a/libs/log/handler/friendly.go +++ b/libs/log/handler/friendly.go @@ -53,11 +53,11 @@ func NewFriendlyHandler(out io.Writer, opts *Options) slog.Handler { // Cache (colorized) level strings. // The colors to use for each level are configured in `colors.go`. - h.levelTrace = h.sprintf(ttyColorLevelTrace, "%5s", "TRACE") - h.levelDebug = h.sprintf(ttyColorLevelDebug, "%5s", "DEBUG") - h.levelInfo = h.sprintf(ttyColorLevelInfo, "%5s", "INFO") - h.levelWarn = h.sprintf(ttyColorLevelWarn, "%5s", "WARN") - h.levelError = h.sprintf(ttyColorLevelError, "%5s", "ERROR") + h.levelTrace = h.sprintf(ttyColorLevelTrace, "%s", "Trace:") + h.levelDebug = h.sprintf(ttyColorLevelDebug, "%s", "Debug:") + h.levelInfo = h.sprintf(ttyColorLevelInfo, "%s", "Info:") + h.levelWarn = h.sprintf(ttyColorLevelWarn, "%s", "Warn:") + h.levelError = h.sprintf(ttyColorLevelError, "%s", "Error:") return h } @@ -185,33 +185,41 @@ func (s *handleState) appendAttr(a slog.Attr) { // Handle implements slog.Handler. func (h *friendlyHandler) Handle(ctx context.Context, r slog.Record) error { state := h.handleState() - state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + + if h.opts.Level.Level() <= slog.LevelDebug { + state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + } + state.appendf("%s ", h.coloredLevel(r)) state.append(h.sprint(ttyColorMessage, r.Message)) - // Handle state from WithGroup and WithAttrs. - goas := h.goas - if r.NumAttrs() == 0 { - // If the record has no Attrs, remove groups at the end of the list; they are empty. - for len(goas) > 0 && goas[len(goas)-1].group != "" { - goas = goas[:len(goas)-1] - } - } - for _, goa := range goas { - if goa.group != "" { - state.openGroup(goa.group) - } else { - for _, a := range goa.attrs { - state.appendAttr(a) + if h.opts.Level.Level() <= slog.LevelDebug { + + // Handle state from WithGroup and WithAttrs. + goas := h.goas + if r.NumAttrs() == 0 { + // If the record has no Attrs, remove groups at the end of the list; they are empty. + for len(goas) > 0 && goas[len(goas)-1].group != "" { + goas = goas[:len(goas)-1] + } + } + for _, goa := range goas { + if goa.group != "" { + state.openGroup(goa.group) + } else { + for _, a := range goa.attrs { + state.appendAttr(a) + } } } - } - // Add attributes from the record. - r.Attrs(func(a slog.Attr) bool { - state.appendAttr(a) - return true - }) + // Add attributes from the record. + r.Attrs(func(a slog.Attr) bool { + state.appendAttr(a) + return true + }) + + } // Add newline. state.append("\n") From 65ac9a336a92c3d3f7e0dece648437024a361c90 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:21:37 +0530 Subject: [PATCH 062/105] Add doc string for the `auth token` command (#2302) ## Changes The intent of this PR is to clarify that the `databricks auth token` command is not supported for M2M auth. Fixes: https://github.com/databricks/cli/issues/1939 ## Tests Manually. --- cmd/auth/token.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/auth/token.go b/cmd/auth/token.go index fbf8b68f6..f3468df40 100644 --- a/cmd/auth/token.go +++ b/cmd/auth/token.go @@ -46,6 +46,10 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { cmd := &cobra.Command{ Use: "token [HOST]", Short: "Get authentication token", + Long: `Get authentication token from the local cache in ~/.databricks/token-cache.json. +Refresh the access token if it is expired. Note: This command only works with +U2M authentication (using the 'databricks auth login' command). M2M authentication +using a client ID and secret is not supported.`, } var tokenTimeout time.Duration From ecc05689cafffcbc80ec14e708696ae4beff6629 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 14:13:12 +0100 Subject: [PATCH 063/105] Add a couple of tests for bundle init with custom template (#2293) These test custom template and what happens if helper function returns an error. --- .../helpers-error/databricks_template_schema.json | 1 + acceptance/bundle/templates/helpers-error/output.txt | 3 +++ acceptance/bundle/templates/helpers-error/script | 1 + .../templates/helpers-error/template/helpers.txt.tmpl | 1 + acceptance/bundle/templates/helpers-error/test.toml | 7 +++++++ .../templates/helpers/databricks_template_schema.json | 1 + acceptance/bundle/templates/helpers/output.txt | 2 ++ acceptance/bundle/templates/helpers/script | 3 +++ .../bundle/templates/helpers/template/helpers.txt.tmpl | 1 + acceptance/bundle/templates/helpers/test.toml | 1 + 10 files changed, 21 insertions(+) create mode 100644 acceptance/bundle/templates/helpers-error/databricks_template_schema.json create mode 100644 acceptance/bundle/templates/helpers-error/output.txt create mode 100644 acceptance/bundle/templates/helpers-error/script create mode 100644 acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl create mode 100644 acceptance/bundle/templates/helpers-error/test.toml create mode 100644 acceptance/bundle/templates/helpers/databricks_template_schema.json create mode 100644 acceptance/bundle/templates/helpers/output.txt create mode 100644 acceptance/bundle/templates/helpers/script create mode 100644 acceptance/bundle/templates/helpers/template/helpers.txt.tmpl create mode 100644 acceptance/bundle/templates/helpers/test.toml diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates/helpers-error/output.txt new file mode 100644 index 000000000..6020e944f --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/output.txt @@ -0,0 +1,3 @@ +Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: + +Exit code: 1 diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates/helpers-error/script new file mode 100644 index 000000000..d9fcbf62c --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/script @@ -0,0 +1 @@ +$CLI bundle init . diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates/helpers-error/test.toml new file mode 100644 index 000000000..77f4ed94b --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/test.toml @@ -0,0 +1,7 @@ +Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' +LocalOnly = true + +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/Me" +Response.Body = '{}' +Response.StatusCode = 500 diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates/helpers/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates/helpers/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates/helpers/output.txt new file mode 100644 index 000000000..3126ea5af --- /dev/null +++ b/acceptance/bundle/templates/helpers/output.txt @@ -0,0 +1,2 @@ +✨ Successfully initialized template +user_name: [USERNAME] diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates/helpers/script new file mode 100644 index 000000000..1773e7b03 --- /dev/null +++ b/acceptance/bundle/templates/helpers/script @@ -0,0 +1,3 @@ +$CLI bundle init . +cat helpers.txt +rm helpers.txt diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates/helpers/test.toml new file mode 100644 index 000000000..b76e712fb --- /dev/null +++ b/acceptance/bundle/templates/helpers/test.toml @@ -0,0 +1 @@ +LocalOnly = true From ecb816446e63382cb8871bb376eb26b9f80b29cc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 14:54:24 +0000 Subject: [PATCH 064/105] Update app deploy test to confirm app config changes on redeploy (#2301) ## Changes Adds additional step to integration test which changes the app config and confirms it's updated after redeploy ## Tests ``` helpers_test.go:156: stderr: Deleting files... helpers_test.go:156: stderr: Destroy complete! --- PASS: TestDeployBundleWithApp (470.25s) PASS coverage: [no statements] ok github.com/databricks/cli/integration/bundle 470.981s coverage: [no statements] ``` --- integration/bundle/apps_test.go | 18 ++++++++++++++++++ .../bundles/apps/template/databricks.yml.tmpl | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go index 01ab52e90..12bd2fcbf 100644 --- a/integration/bundle/apps_test.go +++ b/integration/bundle/apps_test.go @@ -98,6 +98,24 @@ func TestDeployBundleWithApp(t *testing.T) { - run env: - name: JOB_ID + value: "%d"`, job.JobId)) + + // Redeploy bundle with changed config env for app and confirm it's updated in app.yaml + deployBundleWithArgs(t, ctx, root, `--var="env_var_name=ANOTHER_JOB_ID"`, "--force-lock", "--auto-approve") + reader, err = wt.W.Workspace.Download(ctx, pathToAppYml) + require.NoError(t, err) + + data, err = io.ReadAll(reader) + require.NoError(t, err) + + content = string(data) + require.Contains(t, content, fmt.Sprintf(`command: + - flask + - --app + - app + - run +env: + - name: ANOTHER_JOB_ID value: "%d"`, job.JobId)) if testing.Short() { diff --git a/integration/bundle/bundles/apps/template/databricks.yml.tmpl b/integration/bundle/bundles/apps/template/databricks.yml.tmpl index 4d862a06f..e0937be71 100644 --- a/integration/bundle/bundles/apps/template/databricks.yml.tmpl +++ b/integration/bundle/bundles/apps/template/databricks.yml.tmpl @@ -4,6 +4,10 @@ bundle: workspace: root_path: "~/.bundle/{{.unique_id}}" +variables: + env_var_name: + default: "JOB_ID" + resources: apps: test_app: @@ -17,7 +21,7 @@ resources: - app - run env: - - name: JOB_ID + - name: ${var.env_var_name} value: ${resources.jobs.foo.id} resources: From 6b1a778fe10051d779beb22101825b5a72dcddae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 17:17:50 +0100 Subject: [PATCH 065/105] Fix flaky acceptance test (#2310) ## Changes Replace timestamps with fixed string before output is sorted (and before test runner replacements are applied). Otherwise the test sometimes fails with error below. Note, timestamps themselves do not show it, because they were replaced. ``` --- FAIL: TestAccept/bundle/debug (0.78s) acceptance_test.go:404: Diff: --- bundle/debug/out.stderr.parallel.txt +++ /var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAcceptbundledebug1859985035/001/out.stderr.parallel.txt @@ -8,8 +8,8 @@ 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns -10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync 10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true ``` ## Tests Running `hyperfine --min-runs 10 'go test ../.. -run ^TestAccept$/^bundle$/^debug$ -count=1' --show-output` detects flakiness on main but not with this PR. --- acceptance/bundle/debug/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script index 1b4cfc8f0..005a1a341 100644 --- a/acceptance/bundle/debug/script +++ b/acceptance/bundle/debug/script @@ -1,4 +1,4 @@ $CLI bundle validate --debug 2> full.stderr.txt grep -vw parallel full.stderr.txt > out.stderr.txt -grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt +grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt rm full.stderr.txt From f71583fbc09504dbbf0742e53bdcebca53d424a3 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 7 Feb 2025 21:56:48 +0530 Subject: [PATCH 066/105] Error when unknown API endpoint is used in testserver (#2292) ## Changes This PR fails the acceptance test when an unknown endpoint (i.e. not stubbed) is used. We want to ensure that all API endpoints used in an acceptance test are stubbed and do not otherwise silently fail with a 404. The logs on failure output include a configuration that developers can simply copy-paste to `test.toml` to stub the missing API endpoint. It'll look something like: ``` [[Server]] Pattern = " " Response.Body = ''' ''' Response.StatusCode = ``` ## Tests Manually: output.txt when an endpoint is not found: ``` >>> [CLI] jobs create --json {"name":"abc"} Error: No stub found for pattern: POST /api/2.1/jobs/create ``` How this renders in the test logs: ``` --- FAIL: TestAccept/workspace/jobs/create (0.03s) server.go:46: ---------------------------------------- No stub found for pattern: POST /api/2.1/jobs/create To stub a response for this request, you can add the following to test.toml: [[Server]] Pattern = "POST /api/2.1/jobs/create" Response.Body = ''' ''' Response.StatusCode = ---------------------------------------- ``` Manually checked that the debug mode still works. --- acceptance/cmd_server_test.go | 5 ++++- libs/testserver/server.go | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 0166dfe32..c8a52f4cd 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,10 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { + // {$} is a wildcard that only matches the end of the URL. We explicitly use + // /{$} to disambiguate it from the generic handler for '/' which is used to + // identify unhandled API endpoints in the test server. + server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/libs/testserver/server.go b/libs/testserver/server.go index ffb83a49c..9ccf34be0 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/apierr" ) type Server struct { @@ -41,13 +42,44 @@ func New(t testutil.TestingT) *Server { server := httptest.NewServer(mux) t.Cleanup(server.Close) - return &Server{ + s := &Server{ Server: server, Mux: mux, t: t, mu: &sync.Mutex{}, fakeWorkspaces: map[string]*FakeWorkspace{}, } + + // The server resolves conflicting handlers by using the one with higher + // specificity. This handler is the least specific, so it will be used as a + // fallback when no other handlers match. + s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) { + pattern := r.Method + " " + r.URL.Path + + t.Errorf(` + +---------------------------------------- +No stub found for pattern: %s + +To stub a response for this request, you can add +the following to test.toml: +[[Server]] +Pattern = %q +Response.Body = ''' + +''' +Response.StatusCode = +---------------------------------------- + + +`, pattern, pattern) + + return apierr.APIError{ + Message: "No stub found for pattern: " + pattern, + }, http.StatusNotFound + }) + + return s } type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) From ff4a5c22698dd31445524672c70f05d05031d4f3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 17:38:27 +0100 Subject: [PATCH 067/105] acc: Implement config merge (#2294) ## Changes Instead of using leaf-most config, all configs from root at acceptance/test.toml to all intermediate ones to leaf config are merged into one. Maps are merged, slices are appended, other values are overridden. I had to disable caching, because it is tricky when merging is involved - deep copy is needed. There is performance impact but currently it is tiny, about 1%. Also, remove empty root config. ## Tests Manually checked that inheritance of LocalOnly setting worked for these tests: Before - integration tests showed: ``` PASS acceptance.TestAccept/bundle/templates/wrong-url (0.70s) PASS acceptance.TestAccept/bundle/templates/wrong-path (0.44s) ``` After: ``` SKIP acceptance.TestAccept/bundle/templates/wrong-url (0.00s) SKIP acceptance.TestAccept/bundle/templates/wrong-path (0.00s) acceptance_test.go:216: Disabled via LocalOnly setting in bundle/templates/test.toml, bundle/templates/wrong-path/test.toml (CLOUD_ENV=***) ``` --- NOTICE | 5 ++++ acceptance/config_test.go | 56 +++++++++++++++++---------------------- acceptance/test.toml | 2 -- go.mod | 1 + go.sum | 4 +-- 5 files changed, 33 insertions(+), 35 deletions(-) delete mode 100644 acceptance/test.toml diff --git a/NOTICE b/NOTICE index ed22084cf..4331a2a32 100644 --- a/NOTICE +++ b/NOTICE @@ -109,3 +109,8 @@ License - https://github.com/hexops/gotextdiff/blob/main/LICENSE https://github.com/BurntSushi/toml Copyright (c) 2013 TOML authors https://github.com/BurntSushi/toml/blob/master/COPYING + +dario.cat/mergo +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. +https://github.com/darccio/mergo/blob/master/LICENSE diff --git a/acceptance/config_test.go b/acceptance/config_test.go index e24a683e7..920e713a1 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -3,9 +3,11 @@ package acceptance_test import ( "os" "path/filepath" - "sync" + "slices" + "strings" "testing" + "dario.cat/mergo" "github.com/BurntSushi/toml" "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" @@ -13,11 +15,6 @@ import ( const configFilename = "test.toml" -var ( - configCache map[string]TestConfig - configMutex sync.Mutex -) - type TestConfig struct { // Place to describe what's wrong with this test. Does not affect how the test is run. Badness string @@ -65,58 +62,55 @@ type ServerStub struct { } } -// FindConfig finds the closest config file. -func FindConfig(t *testing.T, dir string) (string, bool) { - shared := false +// FindConfigs finds all the config relevant for this test, +// ordered from the most outermost (at acceptance/) to current test directory (identified by dir). +// Argument dir must be a relative path from the root of acceptance tests (/acceptance/). +func FindConfigs(t *testing.T, dir string) []string { + configs := []string{} for { path := filepath.Join(dir, configFilename) _, err := os.Stat(path) if err == nil { - return path, shared + configs = append(configs, path) } - shared = true - if dir == "" || dir == "." { break } - if os.IsNotExist(err) { - dir = filepath.Dir(dir) + dir = filepath.Dir(dir) + + if err == nil || os.IsNotExist(err) { continue } t.Fatalf("Error while reading %s: %s", path, err) } - t.Fatal("Config not found: " + configFilename) - return "", shared + slices.Reverse(configs) + return configs } // LoadConfig loads the config file. Non-leaf configs are cached. func LoadConfig(t *testing.T, dir string) (TestConfig, string) { - path, leafConfig := FindConfig(t, dir) + configs := FindConfigs(t, dir) - if leafConfig { - return DoLoadConfig(t, path), path + if len(configs) == 0 { + return TestConfig{}, "(no config)" } - configMutex.Lock() - defer configMutex.Unlock() + result := DoLoadConfig(t, configs[0]) - if configCache == nil { - configCache = make(map[string]TestConfig) + for _, cfgName := range configs[1:] { + cfg := DoLoadConfig(t, cfgName) + err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + t.Fatalf("Error during config merge: %s: %s", cfgName, err) + } } - result, ok := configCache[path] - if ok { - return result, path - } - - result = DoLoadConfig(t, path) - configCache[path] = result - return result, path + return result, strings.Join(configs, ", ") } func DoLoadConfig(t *testing.T, path string) TestConfig { diff --git a/acceptance/test.toml b/acceptance/test.toml deleted file mode 100644 index eee94d0ea..000000000 --- a/acceptance/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration. -# The configurations are not merged across parents; the closest one is used fully. diff --git a/go.mod b/go.mod index b3f11e918..662fcd40b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 toolchain go1.23.4 require ( + dario.cat/mergo v1.0.1 // BSD 3-Clause github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 diff --git a/go.sum b/go.sum index 4e295a82d..bffc3b53d 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKF cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= From 5aa89230e96635efaec5aa32a6a06a8cab9dec2f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 17:22:51 +0000 Subject: [PATCH 068/105] Use CreatePipeline instead of PipelineSpec for resources.Pipeline struct (#2287) ## Changes `CreatePipeline` is a more complete structure (superset of PipelineSpec one) which enables support of additional fields such as `run_as` and `allow_duplicate_names` in DABs configuration. Note: these fields are subject to support in TF in order to correctly work. ## Tests Existing tests pass + no fields are removed from JSON schema --- bundle/config/mutator/apply_presets.go | 2 +- .../mutator/capture_schema_dependency.go | 4 +-- .../mutator/capture_schema_dependency_test.go | 30 +++++++++---------- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/initialize_urls_test.go | 4 +-- .../mutator/merge_pipeline_clusters_test.go | 4 +-- .../mutator/process_target_mode_test.go | 10 +++---- .../resolve_variable_references_test.go | 6 ++-- bundle/config/mutator/translate_paths_test.go | 16 +++++----- bundle/config/resources/pipeline.go | 4 +-- .../validate/single_node_cluster_test.go | 4 +-- bundle/deploy/metadata/annotate_pipelines.go | 4 +-- .../metadata/annotate_pipelines_test.go | 8 ++--- bundle/deploy/terraform/convert_test.go | 8 ++--- .../terraform/tfdyn/convert_pipeline.go | 5 ++++ .../terraform/tfdyn/convert_pipeline_test.go | 11 ++++++- bundle/internal/schema/annotations.yml | 10 +++++++ .../internal/schema/annotations_openapi.yml | 17 +++++++++++ .../schema/annotations_openapi_overrides.yml | 6 ++++ bundle/internal/schema/main.go | 15 ++++++++++ bundle/permissions/workspace_root_test.go | 8 ++--- bundle/render/render_text_output_test.go | 8 ++--- bundle/resources/completion_test.go | 4 +-- bundle/resources/lookup_test.go | 4 +-- bundle/run/pipeline.go | 4 +-- bundle/schema/jsonschema.json | 26 ++++++++++++++++ cmd/bundle/generate/pipeline.go | 2 +- libs/dyn/drop_keys.go | 27 +++++++++++++++++ libs/dyn/drop_keys_test.go | 24 +++++++++++++++ 29 files changed, 208 insertions(+), 69 deletions(-) create mode 100644 libs/dyn/drop_keys.go create mode 100644 libs/dyn/drop_keys_test.go diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index b402053e7..c8e7bf9e8 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -84,7 +84,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Pipelines presets: Prefix, PipelinesDevelopment for key, p := range r.Pipelines { - if p.PipelineSpec == nil { + if p.CreatePipeline == nil { diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) continue } diff --git a/bundle/config/mutator/capture_schema_dependency.go b/bundle/config/mutator/capture_schema_dependency.go index 5025c9a0d..2e17a8175 100644 --- a/bundle/config/mutator/capture_schema_dependency.go +++ b/bundle/config/mutator/capture_schema_dependency.go @@ -56,7 +56,7 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) { } func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Schema == "" { @@ -71,7 +71,7 @@ func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { } func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Target == "" { diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go index 0a94e7748..16fa636ee 100644 --- a/bundle/config/mutator/capture_schema_dependency_test.go +++ b/bundle/config/mutator/capture_schema_dependency_test.go @@ -118,43 +118,43 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Schema: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Schema: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "", Name: "whatever", @@ -179,7 +179,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema) assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"]) - assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec) + assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].CreatePipeline) for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} { assert.Empty(t, b.Config.Resources.Pipelines[k].Target) @@ -214,43 +214,43 @@ func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Target: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Target: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "", Name: "whatever", diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index 7cf3c9f3e..c5b1ad39d 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -47,7 +47,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index f07a7deb3..8c751079b 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -31,8 +31,8 @@ func TestInitializeURLs(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - ID: "3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}, + ID: "3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1"}, }, }, Experiments: map[string]*resources.MlflowExperiment{ diff --git a/bundle/config/mutator/merge_pipeline_clusters_test.go b/bundle/config/mutator/merge_pipeline_clusters_test.go index f117d9399..97ec44eea 100644 --- a/bundle/config/mutator/merge_pipeline_clusters_test.go +++ b/bundle/config/mutator/merge_pipeline_clusters_test.go @@ -19,7 +19,7 @@ func TestMergePipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { NodeTypeId: "i3.xlarge", @@ -68,7 +68,7 @@ func TestMergePipelineClustersCaseInsensitive(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { Label: "default", diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 723b01ee3..6a0fd8e03 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -88,7 +88,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}}, + "pipeline1": {CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}}, @@ -181,7 +181,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous) - assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.True(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) // Experiment 1 assert.Equal(t, "/Users/lennart.kats@databricks.com/[dev lennart] experiment1", b.Config.Resources.Experiments["experiment1"].Name) @@ -316,7 +316,7 @@ func TestProcessTargetModeDefault(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -362,7 +362,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -568,5 +568,5 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) { diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 44f6c8dbb..30969dc49 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -20,7 +20,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { true, func(t *testing.T, b *bundle.Bundle) { // Variables that use workspace file path should have SyncRootValue during resolution phase - require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) // The file path itself should remain the same require.Equal(t, "file/path", b.Config.Workspace.FilePath) @@ -29,7 +29,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { { false, func(t *testing.T, b *bundle.Bundle) { - require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) require.Equal(t, "file/path", b.Config.Workspace.FilePath) }, }, @@ -48,7 +48,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Configuration: map[string]string{ "source": "${workspace.file_path}", }, diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index aa6488ab0..6cfe5718a 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -179,7 +179,7 @@ func TestTranslatePaths(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -333,7 +333,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -488,7 +488,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -532,7 +532,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -572,7 +572,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -677,7 +677,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -712,7 +712,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -916,7 +916,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 5127d07ba..57d9c4f19 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,7 +16,7 @@ type Pipeline struct { ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` URL string `json:"url,omitempty" bundle:"internal"` - *pipelines.PipelineSpec + *pipelines.CreatePipeline } func (s *Pipeline) UnmarshalJSON(b []byte) error { @@ -59,5 +59,5 @@ func (s *Pipeline) GetURL() string { } func (s *Pipeline) IsNil() bool { - return s.PipelineSpec == nil + return s.CreatePipeline == nil } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index c3ead8ef6..be93420c6 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -238,7 +238,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, @@ -493,7 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go index 990f48907..407aaea6e 100644 --- a/bundle/deploy/metadata/annotate_pipelines.go +++ b/bundle/deploy/metadata/annotate_pipelines.go @@ -20,11 +20,11 @@ func (m *annotatePipelines) Name() string { func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for _, pipeline := range b.Config.Resources.Pipelines { - if pipeline.PipelineSpec == nil { + if pipeline.CreatePipeline == nil { continue } - pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{ + pipeline.CreatePipeline.Deployment = &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: metadataFilePath(b), } diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go index 448a022d0..606292724 100644 --- a/bundle/deploy/metadata/annotate_pipelines_test.go +++ b/bundle/deploy/metadata/annotate_pipelines_test.go @@ -21,12 +21,12 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "my-pipeline-1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline One", }, }, "my-pipeline-2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline Two", }, }, @@ -43,14 +43,14 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-1"].CreatePipeline.Deployment) assert.Equal(t, &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-2"].CreatePipeline.Deployment) } func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) { diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index afc1fb22a..53d861b32 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -203,7 +203,7 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) { func TestBundleToTerraformPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", Libraries: []pipelines.PipelineLibrary{ { @@ -759,7 +759,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, @@ -898,12 +898,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, "test_pipeline_new": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline_new", }, }, diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index ea0c94d66..53a986864 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,6 +21,11 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } + vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"}) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go index 0239bad18..d8de55bf0 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -15,8 +15,17 @@ import ( func TestConvertPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", + // This fields is not part of TF schema yet, but once we upgrade to TF version that supports it, this test will fail because run_as + // will be exposed which is expected and test will need to be updated. + RunAs: &pipelines.RunAs{ + UserName: "foo@bar.com", + }, + // We expect AllowDuplicateNames and DryRun to be ignored and not passed to the TF output. + // This is not supported by TF now, so we don't want to expose it. + AllowDuplicateNames: true, + DryRun: true, Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index c10f43b04..2d1a6a3d8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -414,6 +414,16 @@ github.com/databricks/cli/bundle/config/resources.Permission: "user_name": "description": |- The name of the user that has the permission set in level. +github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + PLACEHOLDER + "dry_run": + "description": |- + PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index d5a9bf69e..d9a0be50e 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -371,6 +371,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "description": |- Tags to be attached to the serving endpoint and automatically propagated to billing logs. github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + If false, deployment will fail if name conflicts with that of another pipeline. "budget_policy_id": "description": |- Budget policy of this pipeline. @@ -395,6 +398,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "development": "description": |- Whether the pipeline is in Development mode. Defaults to false. + "dry_run": {} "edition": "description": |- Pipeline product edition. @@ -425,6 +429,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "restart_window": "description": |- Restart window of this pipeline. + "run_as": {} "schema": "description": |- The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode. @@ -2624,6 +2629,18 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow: "description": |- Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. +github.com/databricks/databricks-sdk-go/service/pipelines.RunAs: + "_": + "description": |- + Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline. + + Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. + "service_principal_name": + "description": |- + Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": + "description": |- + The email of an active workspace user. Users can only set this field to their own email. github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec: "destination_catalog": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 585886313..be83af2d1 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -239,9 +239,15 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: - notebook: path: ./pipeline.py ``` + "dry_run": + "description": |- + PLACEHOLDER "permissions": "description": |- PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.QualityMonitor: "_": "markdown_description": |- diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 38e099ece..2e0120e62 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -109,6 +109,20 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { return s } +func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Pipeline{}): + // Even though DABs supports this field, TF provider does not. Thus, we + // should not expose it to the user. + delete(s.Properties, "dry_run") + delete(s.Properties, "allow_duplicate_names") + default: + // Do nothing + } + + return s +} + // While volume_type is required in the volume create API, DABs automatically sets // it's value to "MANAGED" if it's not provided. Thus, we make it optional // in the bundle schema. @@ -168,6 +182,7 @@ func generateSchema(workdir, outputFile string) { // Generate the JSON schema from the bundle Go struct. s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ removeJobsFields, + removePipelineFields, makeVolumeTypeOptional, a.addAnnotations, addInterpolationPatterns, diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index c48704a63..3e5f9c61b 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -38,8 +38,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, @@ -98,8 +98,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 506756f70..d092e77c8 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -530,12 +530,12 @@ func TestRenderSummary(t *testing.T) { "pipeline2": { ID: "4", // no URL - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"}, + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline2-name"}, }, "pipeline1": { - ID: "3", - URL: "https://url3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"}, + ID: "3", + URL: "https://url3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1-name"}, }, }, Schemas: map[string]*resources.Schema{ diff --git a/bundle/resources/completion_test.go b/bundle/resources/completion_test.go index 80412b6f1..56559f18c 100644 --- a/bundle/resources/completion_test.go +++ b/bundle/resources/completion_test.go @@ -25,7 +25,7 @@ func TestCompletions_SkipDuplicates(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -50,7 +50,7 @@ func TestCompletions_Filter(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/resources/lookup_test.go b/bundle/resources/lookup_test.go index 0ea5af7a2..d95da977a 100644 --- a/bundle/resources/lookup_test.go +++ b/bundle/resources/lookup_test.go @@ -56,7 +56,7 @@ func TestLookup_MultipleFound(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -107,7 +107,7 @@ func TestLookup_NominalWithFilters(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index bdcf0f142..1cd6e8743 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -79,10 +79,10 @@ type pipelineRunner struct { } func (r *pipelineRunner) Name() string { - if r.pipeline == nil || r.pipeline.PipelineSpec == nil { + if r.pipeline == nil || r.pipeline.CreatePipeline == nil { return "" } - return r.pipeline.PipelineSpec.Name + return r.pipeline.CreatePipeline.Name } func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 7c72c440e..9d4304cd8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -703,6 +703,9 @@ "description": "Restart window of this pipeline.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow" }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RunAs" + }, "schema": { "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", "$ref": "#/$defs/string" @@ -5385,6 +5388,29 @@ } ] }, + "pipelines.RunAs": { + "oneOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.SchemaSpec": { "oneOf": [ { diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 1d2c345d6..9bf9e9947 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -92,7 +92,7 @@ func NewGeneratePipelineCommand() *cobra.Command { } saver := yamlsaver.NewSaverWithStyle( - // Including all PipelineSpec and nested fields which are map[string]string type + // Including all CreatePipeline and nested fields which are map[string]string type map[string]yaml.Style{ "spark_conf": yaml.DoubleQuotedStyle, "custom_tags": yaml.DoubleQuotedStyle, diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go new file mode 100644 index 000000000..494f9b9cd --- /dev/null +++ b/libs/dyn/drop_keys.go @@ -0,0 +1,27 @@ +package dyn + +func DropKeys(v Value, drop []string) (Value, error) { + var err error + nv, err := Walk(v, func(p Path, v Value) (Value, error) { + if len(p) == 0 { + return v, nil + } + + // Check if this key should be dropped. + for _, key := range drop { + if p[0].Key() != key { + continue + } + + return InvalidValue, ErrDrop + } + + // Pass through all other values. + return v, ErrSkip + }) + if err != nil { + return InvalidValue, err + } + + return nv, nil +} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go new file mode 100644 index 000000000..83a9744ca --- /dev/null +++ b/libs/dyn/drop_keys_test.go @@ -0,0 +1,24 @@ +package dyn + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDropKeysTest(t *testing.T) { + v := V(map[string]Value{ + "key1": V("value1"), + "key2": V("value2"), + "key3": V("value3"), + }) + + vout, err := DropKeys(v, []string{"key1", "key3"}) + require.NoError(t, err) + + mv := vout.MustMap() + require.Equal(t, 1, mv.Len()) + v, ok := mv.GetByString("key2") + require.True(t, ok) + require.Equal(t, "value2", v.MustString()) +} From 989aabe5f1c4d91aa1f9b611f8b49c8241023744 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 18:42:35 +0100 Subject: [PATCH 069/105] acc: Make variable tests local-only (#2312) Makes use of #2294 --- acceptance/bundle/variables/env_overrides/test.toml | 2 -- acceptance/bundle/variables/resolve-builtin/test.toml | 2 -- .../bundle/variables/resolve-vars-in-root-path/test.toml | 2 -- acceptance/bundle/variables/test.toml | 3 +++ 4 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 acceptance/bundle/variables/env_overrides/test.toml delete mode 100644 acceptance/bundle/variables/resolve-builtin/test.toml delete mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/test.toml create mode 100644 acceptance/bundle/variables/test.toml diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml deleted file mode 100644 index 439c2fab1..000000000 --- a/acceptance/bundle/variables/env_overrides/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml deleted file mode 100644 index 085fab6c0..000000000 --- a/acceptance/bundle/variables/resolve-builtin/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml deleted file mode 100644 index d833bd848..000000000 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml new file mode 100644 index 000000000..32398e828 --- /dev/null +++ b/acceptance/bundle/variables/test.toml @@ -0,0 +1,3 @@ +# The tests here intend to test variable interpolation via "bundle validate". +# Even though "bundle validate" does a few API calls, that's not the focus there. +LocalOnly = true From 6d83ffd1090fc3b80f9b9957aa1402ea47cfb9ef Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 18:42:47 +0100 Subject: [PATCH 070/105] acc: enable bundle/scripts on cloud (#2313) --- acceptance/bundle/scripts/test.toml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 acceptance/bundle/scripts/test.toml diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml deleted file mode 100644 index 1dbd78681..000000000 --- a/acceptance/bundle/scripts/test.toml +++ /dev/null @@ -1 +0,0 @@ -LocalOnly = true # Deployment currently fails when run locally; once that is fixed, remove this setting From 2a97dcaa45d14981610619f334428b1a6858c332 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 17:55:16 +0000 Subject: [PATCH 071/105] Raise an error when there are multiple local libraries with the same basename used (#2297) ## Changes Raise an error when there are multiple local libraries with the same basename used Fixes #1674 ## Tests Added an unit test --- .../same_name_libraries/databricks.yml | 50 ++++++++ .../artifacts/same_name_libraries/output.txt | 14 ++ .../artifacts/same_name_libraries/script | 2 + .../artifacts/same_name_libraries/test.toml | 0 .../same_name_libraries/whl1/setup.py | 36 ++++++ .../whl1/src/my_default_python/__init__.py | 1 + .../whl1/src/my_default_python/main.py | 1 + .../same_name_libraries/whl2/setup.py | 36 ++++++ .../whl2/src/my_default_python/__init__.py | 1 + .../whl2/src/my_default_python/main.py | 1 + bundle/libraries/expand_glob_references.go | 2 +- bundle/libraries/same_name_libraries.go | 97 ++++++++++++++ bundle/libraries/same_name_libraries_test.go | 121 ++++++++++++++++++ bundle/phases/deploy.go | 5 + 14 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml create mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt create mode 100644 acceptance/bundle/artifacts/same_name_libraries/script create mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py create mode 100644 bundle/libraries/same_name_libraries.go create mode 100644 bundle/libraries/same_name_libraries_test.go diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml new file mode 100644 index 000000000..a065bae76 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml @@ -0,0 +1,50 @@ +bundle: + name: same_name_libraries + +variables: + cluster: + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode + +artifacts: + whl1: + type: whl + path: ./whl1 + whl2: + type: whl + path: ./whl2 + +resources: + jobs: + test: + name: "test" + tasks: + - task_key: task1 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl + - task_key: task2 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl2/dist/*.whl + - task_key: task3 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt new file mode 100644 index 000000000..38cdd43c4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt @@ -0,0 +1,14 @@ + +>>> errcode [CLI] bundle deploy +Building whl1... +Building whl2... +Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl + at resources.jobs.test.tasks[0].libraries[0].whl + resources.jobs.test.tasks[1].libraries[0].whl + in databricks.yml:36:15 + databricks.yml:43:15 + +Local library names must be unique + + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script new file mode 100644 index 000000000..6c899df07 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/script @@ -0,0 +1,2 @@ +trace errcode $CLI bundle deploy +rm -rf whl1 whl2 diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index bb1905045..7a808f627 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic for _, match := range matches { output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.V(match), + libType: dyn.NewValue(match, lib.Locations()), }, lib.Locations())) } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go new file mode 100644 index 000000000..88b96ab54 --- /dev/null +++ b/bundle/libraries/same_name_libraries.go @@ -0,0 +1,97 @@ +package libraries + +import ( + "context" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type checkForSameNameLibraries struct{} + +var patterns = []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + envDepsPattern.Append(dyn.AnyIndex()), +} + +type libData struct { + fullPath string + locations []dyn.Location + paths []dyn.Path +} + +func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + libs := make(map[string]*libData) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + libPath := lv.MustString() + // If not local library, skip the check + if !IsLibraryLocal(libPath) { + return lv, nil + } + + libFullPath := lv.MustString() + lib := filepath.Base(libFullPath) + // If the same basename was seen already but full path is different + // then it's a duplicate. Add the location to the location list. + lp, ok := libs[lib] + if !ok { + libs[lib] = &libData{ + fullPath: libFullPath, + locations: []dyn.Location{lv.Location()}, + paths: []dyn.Path{p}, + } + } else if lp.fullPath != libFullPath { + lp.locations = append(lp.locations, lv.Location()) + lp.paths = append(lp.paths, p) + } + + return lv, nil + }) + if err != nil { + return dyn.InvalidValue, err + } + } + + if err != nil { + return dyn.InvalidValue, err + } + + return v, nil + }) + + // Iterate over all the libraries and check if there are any duplicates. + // Duplicates will have more than one location. + // If there are duplicates, add a diagnostic. + for lib, lv := range libs { + if len(lv.locations) > 1 { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Duplicate local library name " + lib, + Detail: "Local library names must be unique", + Locations: lv.locations, + Paths: lv.paths, + }) + } + } + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (c checkForSameNameLibraries) Name() string { + return "CheckForSameNameLibraries" +} + +func CheckForSameNameLibraries() bundle.Mutator { + return checkForSameNameLibraries{} +} diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go new file mode 100644 index 000000000..42c38773b --- /dev/null +++ b/bundle/libraries/same_name_libraries_test.go @@ -0,0 +1,121 @@ +package libraries + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestSameNameLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test.whl", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + }) + bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{ + {File: "databricks.yml", Line: 20, Column: 1}, + }) + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Len(t, diags, 1) + require.Equal(t, diag.Error, diags[0].Severity) + require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary) + require.Equal(t, []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + {File: "databricks.yml", Line: 20, Column: 1}, + }, diags[0].Locations) + + paths := make([]string, 0) + for _, p := range diags[0].Paths { + paths = append(paths, p.String()) + } + require.Equal(t, []string{ + "resources.jobs.test.tasks[0].libraries[0].whl", + "resources.jobs.test.tasks[1].libraries[0].whl", + }, paths) +} + +func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test-0.1.0.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Empty(t, diags) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c6ec04962..2e9211a7e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -155,6 +155,11 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { mutator.ValidateGitDetails(), artifacts.CleanUp(), libraries.ExpandGlobReferences(), + // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we + // know what are the actual library paths. + // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this + // mutator is part of the deploy step rather than validate. + libraries.CheckForSameNameLibraries(), libraries.Upload(), trampoline.TransformWheelTask(), files.Upload(outputHandler), From f8aaa7fce337ac95dba354ddb22cc5a8fa5de046 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 18:37:03 +0000 Subject: [PATCH 072/105] Added support to generate Git based jobs (#2304) ## Changes This will generate bundle YAML configuration for Git based jobs but won't download any related files as they are in Git repo. Fixes #1423 ## Tests Added unit test --------- Co-authored-by: Pieter Noordhuis --- .../bundle/generate/git_job/databricks.yml | 2 + .../bundle/generate/git_job/out.job.yml | 17 +++++++ acceptance/bundle/generate/git_job/output.txt | 2 + acceptance/bundle/generate/git_job/script | 1 + acceptance/bundle/generate/git_job/test.toml | 33 +++++++++++++ bundle/config/generate/job.go | 1 - cmd/bundle/generate/job.go | 20 ++++++-- libs/dyn/yamlsaver/utils.go | 47 +++++++++++++++++- libs/dyn/yamlsaver/utils_test.go | 48 +++++++++++++++++++ 9 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/generate/git_job/databricks.yml create mode 100644 acceptance/bundle/generate/git_job/out.job.yml create mode 100644 acceptance/bundle/generate/git_job/output.txt create mode 100644 acceptance/bundle/generate/git_job/script create mode 100644 acceptance/bundle/generate/git_job/test.toml diff --git a/acceptance/bundle/generate/git_job/databricks.yml b/acceptance/bundle/generate/git_job/databricks.yml new file mode 100644 index 000000000..adaa7aab3 --- /dev/null +++ b/acceptance/bundle/generate/git_job/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: git_job diff --git a/acceptance/bundle/generate/git_job/out.job.yml b/acceptance/bundle/generate/git_job/out.job.yml new file mode 100644 index 000000000..0eb2a3fb1 --- /dev/null +++ b/acceptance/bundle/generate/git_job/out.job.yml @@ -0,0 +1,17 @@ +resources: + jobs: + out: + name: gitjob + tasks: + - task_key: test_task + notebook_task: + notebook_path: some/test/notebook.py + - task_key: test_task_2 + notebook_task: + notebook_path: /Workspace/Users/foo@bar.com/some/test/notebook.py + source: WORKSPACE + git_source: + git_branch: main + git_commit: abcdef + git_provider: github + git_url: https://git.databricks.com diff --git a/acceptance/bundle/generate/git_job/output.txt b/acceptance/bundle/generate/git_job/output.txt new file mode 100644 index 000000000..680c92ff9 --- /dev/null +++ b/acceptance/bundle/generate/git_job/output.txt @@ -0,0 +1,2 @@ +Job is using Git source, skipping downloading files +Job configuration successfully saved to out.job.yml diff --git a/acceptance/bundle/generate/git_job/script b/acceptance/bundle/generate/git_job/script new file mode 100644 index 000000000..7598966b0 --- /dev/null +++ b/acceptance/bundle/generate/git_job/script @@ -0,0 +1 @@ +$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml new file mode 100644 index 000000000..28b473245 --- /dev/null +++ b/acceptance/bundle/generate/git_job/test.toml @@ -0,0 +1,33 @@ +LocalOnly = true # This test needs to run against stubbed Databricks API + +[[Server]] +Pattern = "GET /api/2.1/jobs/get" +Response.Body = ''' +{ + "job_id": 11223344, + "settings": { + "name": "gitjob", + "git_source": { + "git_url": "https://git.databricks.com", + "git_provider": "github", + "git_branch": "main", + "git_commit": "abcdef" + }, + "tasks": [ + { + "task_key": "test_task", + "notebook_task": { + "notebook_path": "some/test/notebook.py" + } + }, + { + "task_key": "test_task_2", + "notebook_task": { + "source": "WORKSPACE", + "notebook_path": "/Workspace/Users/foo@bar.com/some/test/notebook.py" + } + } + ] + } +} +''' diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 0cdcbf3ad..934eda2cf 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -13,7 +13,6 @@ var ( func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { value := make(map[string]dyn.Value) - if job.Settings.Tasks != nil { tasks := make([]dyn.Value, 0) for _, task := range job.Settings.Tasks { diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index d97891cd5..438b235c9 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -50,10 +50,22 @@ func NewGenerateJobCommand() *cobra.Command { } downloader := newDownloader(w, sourceDir, configDir) - for _, task := range job.Settings.Tasks { - err := downloader.MarkTaskForDownload(ctx, &task) - if err != nil { - return err + + // Don't download files if the job is using Git source + // When Git source is used, the job will be using the files from the Git repository + // but specific tasks might override this behaviour by using `source: WORKSPACE` setting. + // In this case, we don't want to download the files as well for these specific tasks + // because it leads to confusion with relative paths between workspace and GIT files. + // Instead we keep these tasks as is and let the user handle the files manually. + // The configuration will be deployable as tasks paths for source: WORKSPACE tasks will be absolute workspace paths. + if job.Settings.GitSource != nil { + cmdio.LogString(ctx, "Job is using Git source, skipping downloading files") + } else { + for _, task := range job.Settings.Tasks { + err := downloader.MarkTaskForDownload(ctx, &task) + if err != nil { + return err + } } } diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index a162bf31f..c1b60b1b5 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -22,9 +22,50 @@ func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[str return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind()) } + mv, err = sortMapAlphabetically(mv) + if err != nil { + return dyn.InvalidValue, err + } + return skipAndOrder(mv, order, skipFields, dst) } +// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files. +func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { + sortedMap := dyn.NewMapping() + mapV := mv.MustMap() + keys := mapV.Keys() + slices.SortStableFunc(keys, func(i, j dyn.Value) int { + iKey := i.MustString() + jKey := j.MustString() + if iKey < jKey { + return -1 + } + + if iKey > jKey { + return 1 + } + return 0 + }) + + for _, key := range keys { + value, _ := mapV.Get(key) + var err error + if value.Kind() == dyn.KindMap { + value, err = sortMapAlphabetically(value) + if err != nil { + return dyn.InvalidValue, err + } + } + err = sortedMap.Set(key, value) + if err != nil { + return dyn.InvalidValue, err + } + } + + return dyn.V(sortedMap), nil +} + func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { for _, pair := range mv.MustMap().Pairs() { k := pair.Key.MustString() @@ -44,7 +85,11 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin continue } - dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + if order == nil { + dst[k] = v + } else { + dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + } } return dyn.V(dst), nil diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 1afab601a..f7ea3c96c 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -7,6 +7,54 @@ import ( assert "github.com/databricks/cli/libs/dyn/dynassert" ) +func TestConvertToMap(t *testing.T) { + type test struct { + Name string `json:"name"` + Map map[string]string `json:"map"` + List []string `json:"list"` + LongNameField string `json:"long_name_field"` + ForceSendFields []string `json:"-"` + Format string `json:"format"` + } + + v := &test{ + Name: "test", + Map: map[string]string{ + "key2": "value2", + "key1": "value1", + }, + List: []string{"a", "b", "c"}, + ForceSendFields: []string{ + "Name", + }, + LongNameField: "long name goes here", + } + result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) + assert.NoError(t, err) + assert.Equal(t, dyn.V(map[string]dyn.Value{ + "list": dyn.NewValue( + []dyn.Value{ + dyn.V("a"), + dyn.V("b"), + dyn.V("c"), + }, + []dyn.Location{}, + ), + "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{}), + "map": dyn.NewValue( + map[string]dyn.Value{ + "key1": dyn.V("value1"), + "key2": dyn.V("value2"), + }, + []dyn.Location{}, + ), + "name": dyn.NewValue( + "test", + []dyn.Location{}, + ), + }), result) +} + func TestConvertToMapValueWithOrder(t *testing.T) { type test struct { Name string `json:"name"` From 06e342afc58139264fabd34eec603d58f2b7f95a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 10:16:31 +0100 Subject: [PATCH 073/105] Silence a comment in Makefile (#2315) It was not indended to be printed. Follow up to #2298 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7da7e4789..fb3936184 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ lint: golangci-lint run --fix tidy: - # not part of golangci-lint, apparently + @# not part of golangci-lint, apparently go mod tidy lintcheck: From 2175dd24a4935963eea8ad1c95b72d309d4ea780 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:42:39 +0100 Subject: [PATCH 074/105] Do not gitignore .databricks and terraform (#2318) For acceptance/bundle/templates I'd like to run "bundle deploy". This would create .databricks directory inside materialized output. It might makes sense to commit some of this as part of golden files output. Even if we did not commit anything, the test runner will see those files and show the difference. Thus git should also see them. Also rename .gitignore to out.gitignore in those tests, since that includes .databricks as well. --- .gitignore | 4 ---- .../dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/dbt-sql/script | 3 +++ .../output/my_default_python/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/default-python/script | 3 +++ .../output/my_default_sql/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/default-sql/script | 3 +++ .../output/my_jobs_as_code/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/experimental-jobs-as-code/script | 3 +++ 9 files changed, 12 insertions(+), 4 deletions(-) rename acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/default-python/output/my_default_python/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/default-sql/output/my_default_sql/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/{.gitignore => out.gitignore} (100%) diff --git a/.gitignore b/.gitignore index 2060b6bac..35aef1764 100644 --- a/.gitignore +++ b/.gitignore @@ -25,11 +25,7 @@ coverage-acceptance.txt __pycache__ *.pyc -.terraform -.terraform.lock.hcl - .vscode/launch.json .vscode/tasks.json -.databricks .ruff_cache diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script index c4ca817fe..3a2660de5 100644 --- a/acceptance/bundle/templates/dbt-sql/script +++ b/acceptance/bundle/templates/dbt-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output cd output/my_dbt_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.gitignore rename to acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/script index b11a7ea21..e5fcb7741 100644 --- a/acceptance/bundle/templates/default-python/script +++ b/acceptance/bundle/templates/default-python/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir ou cd output/my_default_python trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore rename to acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index 66e7a14a2..7ea0d863c 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-sql --config-file ./input.json --output-dir outpu cd output/my_default_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore similarity index 100% rename from acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore rename to acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index 10188aabd..08e48fc5f 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -8,3 +8,6 @@ uv sync -q trace $CLI bundle validate -t dev --output json | jq ".resources" rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore From cc073801855d0e0a27f32b25f2a89c1fa391dc8d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:53:00 +0100 Subject: [PATCH 075/105] acc: Summarize unexpected files (#2320) ## Changes When there are many unexpected files, it's good to see them as a list rather than scattered throughout the output. ## Tests Manually, example output: ``` acceptance_test.go:363: Test produced unexpected files: output/my_default_sql/.databricks/bundle/dev/sync-snapshots/71c79ded90615dc7.json output/my_default_sql/.databricks/bundle/dev/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64 output/my_default_sql/.databricks/bundle/dev/terraform/plan output/my_default_sql/.databricks/bundle/prod/sync-snapshots/83e677e75259c93b.json output/my_default_sql/.databricks/bundle/prod/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64 ``` --- acceptance/acceptance_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 4c4404d55..241ab42be 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -343,6 +343,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // Make sure there are not unaccounted for new files files := ListDir(t, tmpDir) + unexpected := []string{} for _, relPath := range files { if _, ok := inputs[relPath]; ok { continue @@ -350,13 +351,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := outputs[relPath]; ok { continue } - t.Errorf("Unexpected output: %s", relPath) + unexpected = append(unexpected, relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } } + + if len(unexpected) > 0 { + t.Error("Test produced unexpected files:\n" + strings.Join(unexpected, "\n")) + } } func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { From 4ebc86282fbbaec9293af022a8d9f5b9b0b80a4a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:55:34 +0100 Subject: [PATCH 076/105] acc: Split bundle/templates and bundle/templates-machinery (#2317) The tests in acceptance/bundle/template focus on standard templates. The plan is to extend them with "bundle deploy" and enable them on the cloud. The tests in acceptance/bundle/template-machinery focus on specific aspects of template implementation. Most of them are expected to remain local-only. --- .../helpers-error/databricks_template_schema.json | 0 .../{templates => templates-machinery}/helpers-error/output.txt | 0 .../{templates => templates-machinery}/helpers-error/script | 0 .../helpers-error/template/helpers.txt.tmpl | 0 .../{templates => templates-machinery}/helpers-error/test.toml | 0 .../helpers/databricks_template_schema.json | 0 .../{templates => templates-machinery}/helpers/output.txt | 0 .../bundle/{templates => templates-machinery}/helpers/script | 0 .../helpers/template/helpers.txt.tmpl | 0 .../bundle/{templates => templates-machinery}/helpers/test.toml | 0 acceptance/bundle/templates-machinery/test.toml | 2 ++ .../{templates => templates-machinery}/wrong-path/output.txt | 0 .../bundle/{templates => templates-machinery}/wrong-path/script | 0 .../{templates => templates-machinery}/wrong-path/test.toml | 0 .../{templates => templates-machinery}/wrong-url/output.txt | 0 .../bundle/{templates => templates-machinery}/wrong-url/script | 0 .../{templates => templates-machinery}/wrong-url/test.toml | 0 17 files changed, 2 insertions(+) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/databricks_template_schema.json (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/script (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/template/helpers.txt.tmpl (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/test.toml (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/databricks_template_schema.json (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/script (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/template/helpers.txt.tmpl (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/test.toml (100%) create mode 100644 acceptance/bundle/templates-machinery/test.toml rename acceptance/bundle/{templates => templates-machinery}/wrong-path/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-path/script (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-path/test.toml (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/script (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/test.toml (100%) diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json similarity index 100% rename from acceptance/bundle/templates/helpers-error/databricks_template_schema.json rename to acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt similarity index 100% rename from acceptance/bundle/templates/helpers-error/output.txt rename to acceptance/bundle/templates-machinery/helpers-error/output.txt diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates-machinery/helpers-error/script similarity index 100% rename from acceptance/bundle/templates/helpers-error/script rename to acceptance/bundle/templates-machinery/helpers-error/script diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl similarity index 100% rename from acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl rename to acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml similarity index 100% rename from acceptance/bundle/templates/helpers-error/test.toml rename to acceptance/bundle/templates-machinery/helpers-error/test.toml diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json similarity index 100% rename from acceptance/bundle/templates/helpers/databricks_template_schema.json rename to acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates-machinery/helpers/output.txt similarity index 100% rename from acceptance/bundle/templates/helpers/output.txt rename to acceptance/bundle/templates-machinery/helpers/output.txt diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates-machinery/helpers/script similarity index 100% rename from acceptance/bundle/templates/helpers/script rename to acceptance/bundle/templates-machinery/helpers/script diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl similarity index 100% rename from acceptance/bundle/templates/helpers/template/helpers.txt.tmpl rename to acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml similarity index 100% rename from acceptance/bundle/templates/helpers/test.toml rename to acceptance/bundle/templates-machinery/helpers/test.toml diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml new file mode 100644 index 000000000..9083ecd1b --- /dev/null +++ b/acceptance/bundle/templates-machinery/test.toml @@ -0,0 +1,2 @@ +# Testing template machinery, by default there is no need to check against cloud. +LocalOnly = true diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates-machinery/wrong-path/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-path/output.txt rename to acceptance/bundle/templates-machinery/wrong-path/output.txt diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates-machinery/wrong-path/script similarity index 100% rename from acceptance/bundle/templates/wrong-path/script rename to acceptance/bundle/templates-machinery/wrong-path/script diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates-machinery/wrong-path/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-path/test.toml rename to acceptance/bundle/templates-machinery/wrong-path/test.toml diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates-machinery/wrong-url/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-url/output.txt rename to acceptance/bundle/templates-machinery/wrong-url/output.txt diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates-machinery/wrong-url/script similarity index 100% rename from acceptance/bundle/templates/wrong-url/script rename to acceptance/bundle/templates-machinery/wrong-url/script diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates-machinery/wrong-url/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-url/test.toml rename to acceptance/bundle/templates-machinery/wrong-url/test.toml From ee440e65fec623fbf3e4cba05ac21d67ee6306db Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:48:05 +0530 Subject: [PATCH 077/105] Serialize all header values in acceptance tests (#2311) ## Changes Based on feedback in https://github.com/databricks/cli/pull/2296#discussion_r1946660650. Previously we only serialized the first value for a header in the requests log. Now we serialise all values for a header key. ## Tests Existing test --- .../workspace/jobs/create/out.requests.txt | 2 +- libs/testserver/server.go | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 60977e3e3..2510762db 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 9ccf34be0..d0c340c12 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -31,10 +31,10 @@ type Server struct { } type Request struct { - Headers map[string]string `json:"headers,omitempty"` - Method string `json:"method"` - Path string `json:"path"` - Body any `json:"body"` + Headers http.Header `json:"headers,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -109,12 +109,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { body, err := io.ReadAll(r.Body) assert.NoError(s.t, err) - headers := make(map[string]string) + headers := make(http.Header) for k, v := range r.Header { - if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) { + if !slices.Contains(s.IncludeRequestHeaders, k) { continue } - headers[k] = v[0] + for _, vv := range v { + headers.Add(k, vv) + } } s.Requests = append(s.Requests, Request{ From 047691dd9148061610e50aa6c12e04cdb85e4022 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:50:52 +0000 Subject: [PATCH 078/105] Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0 (#2321) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.56.1 to 0.57.0.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.57.0

[Release] Release v0.57.0

New Features and Improvements

  • Add support for async OAuth token refreshes (#1135).

API Changes:

OpenAPI SHA: c72c58f97b950fcb924a90ef164bcb10cfcd5ece, Date: 2025-02-03

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.57.0

New Features and Improvements

  • Add support for async OAuth token refreshes (#1135).

API Changes:

OpenAPI SHA: c72c58f97b950fcb924a90ef164bcb10cfcd5ece, Date: 2025-02-03

Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.56.1&new-version=0.57.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- .gitattributes | 5 + .../internal/schema/annotations_openapi.yml | 14 + bundle/schema/jsonschema.json | 20 + cmd/account/budget-policy/budget-policy.go | 373 ++++++++++++++++++ cmd/account/cmd.go | 2 + .../custom-app-integration.go | 2 + .../enable-ip-access-lists.go | 218 ++++++++++ .../federation-policy/federation-policy.go | 3 - .../service-principal-federation-policy.go | 3 - cmd/account/settings/settings.go | 2 + cmd/workspace/alerts/alerts.go | 15 +- cmd/workspace/catalogs/catalogs.go | 1 + cmd/workspace/clean-rooms/clean-rooms.go | 5 +- cmd/workspace/cmd.go | 6 + cmd/workspace/jobs/jobs.go | 1 + .../lakeview-embedded/lakeview-embedded.go | 98 +++++ cmd/workspace/queries/queries.go | 15 +- .../query-execution/query-execution.go | 245 ++++++++++++ .../query-visualizations.go | 15 +- cmd/workspace/redash-config/redash-config.go | 80 ++++ .../serving-endpoints/serving-endpoints.go | 3 +- go.mod | 2 +- go.sum | 4 +- 24 files changed, 1109 insertions(+), 25 deletions(-) create mode 100755 cmd/account/budget-policy/budget-policy.go create mode 100755 cmd/account/enable-ip-access-lists/enable-ip-access-lists.go create mode 100755 cmd/workspace/lakeview-embedded/lakeview-embedded.go create mode 100755 cmd/workspace/query-execution/query-execution.go create mode 100755 cmd/workspace/redash-config/redash-config.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 588cf9d63..9a95107e8 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -0be1b914249781b5e903b7676fd02255755bc851 \ No newline at end of file +c72c58f97b950fcb924a90ef164bcb10cfcd5ece \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index ebe94ed8e..4b3715c93 100755 --- a/.gitattributes +++ b/.gitattributes @@ -1,11 +1,13 @@ cmd/account/access-control/access-control.go linguist-generated=true cmd/account/billable-usage/billable-usage.go linguist-generated=true +cmd/account/budget-policy/budget-policy.go linguist-generated=true cmd/account/budgets/budgets.go linguist-generated=true cmd/account/cmd.go linguist-generated=true cmd/account/credentials/credentials.go linguist-generated=true cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true +cmd/account/enable-ip-access-lists/enable-ip-access-lists.go linguist-generated=true cmd/account/encryption-keys/encryption-keys.go linguist-generated=true cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true cmd/account/federation-policy/federation-policy.go linguist-generated=true @@ -75,6 +77,7 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true cmd/workspace/jobs/jobs.go linguist-generated=true +cmd/workspace/lakeview-embedded/lakeview-embedded.go linguist-generated=true cmd/workspace/lakeview/lakeview.go linguist-generated=true cmd/workspace/libraries/libraries.go linguist-generated=true cmd/workspace/metastores/metastores.go linguist-generated=true @@ -99,11 +102,13 @@ cmd/workspace/providers/providers.go linguist-generated=true cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true cmd/workspace/queries-legacy/queries-legacy.go linguist-generated=true cmd/workspace/queries/queries.go linguist-generated=true +cmd/workspace/query-execution/query-execution.go linguist-generated=true cmd/workspace/query-history/query-history.go linguist-generated=true cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go linguist-generated=true cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true cmd/workspace/recipient-activation/recipient-activation.go linguist-generated=true cmd/workspace/recipients/recipients.go linguist-generated=true +cmd/workspace/redash-config/redash-config.go linguist-generated=true cmd/workspace/registered-models/registered-models.go linguist-generated=true cmd/workspace/repos/repos.go linguist-generated=true cmd/workspace/resource-quotas/resource-quotas.go linguist-generated=true diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index d9a0be50e..74cd06c66 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -281,6 +281,9 @@ github.com/databricks/cli/bundle/config/resources.Job: "parameters": "description": |- Job-level parameter definitions + "performance_target": + "description": |- + PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be. "queue": "description": |- The queue settings of the job. @@ -1818,6 +1821,17 @@ github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus: UNPAUSED - |- PAUSED +github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget: + "_": + "description": |- + PerformanceTarget defines how performant (lower latency) or cost efficient the execution of run on serverless compute should be. + The performance mode on the job or pipeline should map to a performance setting that is passed to Cluster Manager + (see cluster-common PerformanceTarget). + "enum": + - |- + PERFORMANCE_OPTIMIZED + - |- + COST_OPTIMIZED github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration: "interval": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9d4304cd8..c3c31b58c 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -409,6 +409,10 @@ "description": "Job-level parameter definitions", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" }, + "performance_target": { + "description": "PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget" + }, "permissions": { "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, @@ -3904,6 +3908,22 @@ } ] }, + "jobs.PerformanceTarget": { + "oneOf": [ + { + "type": "string", + "description": "PerformanceTarget defines how performant (lower latency) or cost efficient the execution of run on serverless compute should be.\nThe performance mode on the job or pipeline should map to a performance setting that is passed to Cluster Manager\n(see cluster-common PerformanceTarget).", + "enum": [ + "PERFORMANCE_OPTIMIZED", + "COST_OPTIMIZED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.PeriodicTriggerConfiguration": { "oneOf": [ { diff --git a/cmd/account/budget-policy/budget-policy.go b/cmd/account/budget-policy/budget-policy.go new file mode 100755 index 000000000..28b14ea91 --- /dev/null +++ b/cmd/account/budget-policy/budget-policy.go @@ -0,0 +1,373 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package budget_policy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/billing" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "budget-policy", + Short: `A service serves REST API about Budget policies.`, + Long: `A service serves REST API about Budget policies`, + GroupID: "billing", + Annotations: map[string]string{ + "package": "billing", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *billing.CreateBudgetPolicyRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq billing.CreateBudgetPolicyRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: custom_tags + cmd.Flags().StringVar(&createReq.PolicyName, "policy-name", createReq.PolicyName, `The name of the policy.`) + cmd.Flags().StringVar(&createReq.RequestId, "request-id", createReq.RequestId, `A unique identifier for this request.`) + + cmd.Use = "create" + cmd.Short = `Create a budget policy.` + cmd.Long = `Create a budget policy. + + Creates a new policy.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + + response, err := a.BudgetPolicy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *billing.DeleteBudgetPolicyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq billing.DeleteBudgetPolicyRequest + + // TODO: short flags + + cmd.Use = "delete POLICY_ID" + cmd.Short = `Delete a budget policy.` + cmd.Long = `Delete a budget policy. + + Deletes a policy + + Arguments: + POLICY_ID: The Id of the policy.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + deleteReq.PolicyId = args[0] + + err = a.BudgetPolicy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *billing.GetBudgetPolicyRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq billing.GetBudgetPolicyRequest + + // TODO: short flags + + cmd.Use = "get POLICY_ID" + cmd.Short = `Get a budget policy.` + cmd.Long = `Get a budget policy. + + Retrieves a policy by it's ID. + + Arguments: + POLICY_ID: The Id of the policy.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + getReq.PolicyId = args[0] + + response, err := a.BudgetPolicy.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, + *billing.ListBudgetPoliciesRequest, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + var listReq billing.ListBudgetPoliciesRequest + + // TODO: short flags + + // TODO: complex arg: filter_by + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The maximum number of budget policies to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token, received from a previous ListServerlessPolicies call.`) + // TODO: complex arg: sort_spec + + cmd.Use = "list" + cmd.Short = `List policies.` + cmd.Long = `List policies. + + Lists all policies. Policies are returned in the alphabetically ascending + order of their names.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response := a.BudgetPolicy.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd, &listReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *billing.UpdateBudgetPolicyRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq billing.UpdateBudgetPolicyRequest + updateReq.Policy = &billing.BudgetPolicy{} + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: custom_tags + cmd.Flags().StringVar(&updateReq.Policy.PolicyName, "policy-name", updateReq.Policy.PolicyName, `The name of the policy.`) + + cmd.Use = "update POLICY_ID" + cmd.Short = `Update a budget policy.` + cmd.Long = `Update a budget policy. + + Updates a policy + + Arguments: + POLICY_ID: The Id of the policy. This field is generated by Databricks and globally + unique.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq.Policy) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + updateReq.PolicyId = args[0] + + response, err := a.BudgetPolicy.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service BudgetPolicy diff --git a/cmd/account/cmd.go b/cmd/account/cmd.go index f34966fd9..758e2af5e 100644 --- a/cmd/account/cmd.go +++ b/cmd/account/cmd.go @@ -7,6 +7,7 @@ import ( account_access_control "github.com/databricks/cli/cmd/account/access-control" billable_usage "github.com/databricks/cli/cmd/account/billable-usage" + budget_policy "github.com/databricks/cli/cmd/account/budget-policy" budgets "github.com/databricks/cli/cmd/account/budgets" credentials "github.com/databricks/cli/cmd/account/credentials" custom_app_integration "github.com/databricks/cli/cmd/account/custom-app-integration" @@ -43,6 +44,7 @@ func New() *cobra.Command { cmd.AddCommand(account_access_control.New()) cmd.AddCommand(billable_usage.New()) + cmd.AddCommand(budget_policy.New()) cmd.AddCommand(credentials.New()) cmd.AddCommand(custom_app_integration.New()) cmd.AddCommand(encryption_keys.New()) diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index 43e458bc6..61cfe0a09 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -65,6 +65,7 @@ func newCreate() *cobra.Command { // TODO: array: redirect_urls // TODO: array: scopes // TODO: complex arg: token_access_policy + // TODO: array: user_authorized_scopes cmd.Use = "create" cmd.Short = `Create Custom OAuth App Integration.` @@ -309,6 +310,7 @@ func newUpdate() *cobra.Command { // TODO: array: redirect_urls // TODO: array: scopes // TODO: complex arg: token_access_policy + // TODO: array: user_authorized_scopes cmd.Use = "update INTEGRATION_ID" cmd.Short = `Updates Custom OAuth App Integration.` diff --git a/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go b/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go new file mode 100755 index 000000000..24d30c9c6 --- /dev/null +++ b/cmd/account/enable-ip-access-lists/enable-ip-access-lists.go @@ -0,0 +1,218 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package enable_ip_access_lists + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "enable-ip-access-lists", + Short: `Controls the enforcement of IP access lists for accessing the account console.`, + Long: `Controls the enforcement of IP access lists for accessing the account console. + Allowing you to enable or disable restricted access based on IP addresses.`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteAccountIpAccessEnableRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteAccountIpAccessEnableRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the account IP access toggle setting.` + cmd.Long = `Delete the account IP access toggle setting. + + Reverts the value of the account IP access toggle setting to default (ON)` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.EnableIpAccessLists().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetAccountIpAccessEnableRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetAccountIpAccessEnableRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the account IP access toggle setting.` + cmd.Long = `Get the account IP access toggle setting. + + Gets the value of the account IP access toggle setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.EnableIpAccessLists().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateAccountIpAccessEnableRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateAccountIpAccessEnableRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the account IP access toggle setting.` + cmd.Long = `Update the account IP access toggle setting. + + Updates the value of the account IP access toggle setting.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := a.Settings.EnableIpAccessLists().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service EnableIpAccessLists diff --git a/cmd/account/federation-policy/federation-policy.go b/cmd/account/federation-policy/federation-policy.go index e47bf8324..ad45c0405 100755 --- a/cmd/account/federation-policy/federation-policy.go +++ b/cmd/account/federation-policy/federation-policy.go @@ -71,9 +71,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "oauth2", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go index df36de239..451523b7e 100755 --- a/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go +++ b/cmd/account/service-principal-federation-policy/service-principal-federation-policy.go @@ -78,9 +78,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "oauth2", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/account/settings/settings.go b/cmd/account/settings/settings.go index 9a9cd44bf..cd30743f7 100755 --- a/cmd/account/settings/settings.go +++ b/cmd/account/settings/settings.go @@ -7,6 +7,7 @@ import ( csp_enablement_account "github.com/databricks/cli/cmd/account/csp-enablement-account" disable_legacy_features "github.com/databricks/cli/cmd/account/disable-legacy-features" + enable_ip_access_lists "github.com/databricks/cli/cmd/account/enable-ip-access-lists" esm_enablement_account "github.com/databricks/cli/cmd/account/esm-enablement-account" personal_compute "github.com/databricks/cli/cmd/account/personal-compute" ) @@ -29,6 +30,7 @@ func New() *cobra.Command { // Add subservices cmd.AddCommand(csp_enablement_account.New()) cmd.AddCommand(disable_legacy_features.New()) + cmd.AddCommand(enable_ip_access_lists.New()) cmd.AddCommand(esm_enablement_account.New()) cmd.AddCommand(personal_compute.New()) diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go index fcf18652b..79467c405 100755 --- a/cmd/workspace/alerts/alerts.go +++ b/cmd/workspace/alerts/alerts.go @@ -335,10 +335,17 @@ func newUpdate() *cobra.Command { Arguments: ID: - UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask - specifies which fields of the setting payload will be updated. The field - mask needs to be supplied as single string. To specify multiple fields in - the field mask, use comma as the separator (no space).` + UPDATE_MASK: The field mask must be a single string, with multiple fields separated by + commas (no spaces). The field path is relative to the resource object, + using a dot (.) to navigate sub-fields (e.g., author.given_name). + Specification of elements in sequence or map fields is not allowed, as + only the entire collection field can be specified. Field names must + exactly match the resource field names. + + A field mask of * indicates full replacement. It’s recommended to + always explicitly list the fields being updated and avoid using * + wildcards, as it can lead to unintended results if the API changes in the + future.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/catalogs/catalogs.go b/cmd/workspace/catalogs/catalogs.go index 9294c192b..ce37b6d54 100755 --- a/cmd/workspace/catalogs/catalogs.go +++ b/cmd/workspace/catalogs/catalogs.go @@ -342,6 +342,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateReq.EnablePredictiveOptimization, "enable-predictive-optimization", `Whether predictive optimization should be enabled for this object and objects under it. Supported values: [DISABLE, ENABLE, INHERIT]`) cmd.Flags().Var(&updateReq.IsolationMode, "isolation-mode", `Whether the current securable is accessible from all workspaces or a specific set of workspaces. Supported values: [ISOLATED, OPEN]`) cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the catalog.`) + // TODO: map via StringToStringVar: options cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of catalog.`) // TODO: map via StringToStringVar: properties diff --git a/cmd/workspace/clean-rooms/clean-rooms.go b/cmd/workspace/clean-rooms/clean-rooms.go index 053e41e8a..4fe61d56b 100755 --- a/cmd/workspace/clean-rooms/clean-rooms.go +++ b/cmd/workspace/clean-rooms/clean-rooms.go @@ -75,8 +75,9 @@ func newCreate() *cobra.Command { Create a new clean room with the specified collaborators. This method is asynchronous; the returned name field inside the clean_room field can be used to poll the clean room status, using the :method:cleanrooms/get method. When - this method returns, the cluster will be in a PROVISIONING state. The cluster - will be usable once it enters an ACTIVE state. + this method returns, the clean room will be in a PROVISIONING state, with only + name, owner, comment, created_at and status populated. The clean room will be + usable once it enters an ACTIVE state. The caller must be a metastore admin or have the **CREATE_CLEAN_ROOM** privilege on the metastore.` diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index c447bd736..2bd3c59a5 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -39,6 +39,7 @@ import ( ip_access_lists "github.com/databricks/cli/cmd/workspace/ip-access-lists" jobs "github.com/databricks/cli/cmd/workspace/jobs" lakeview "github.com/databricks/cli/cmd/workspace/lakeview" + lakeview_embedded "github.com/databricks/cli/cmd/workspace/lakeview-embedded" libraries "github.com/databricks/cli/cmd/workspace/libraries" metastores "github.com/databricks/cli/cmd/workspace/metastores" model_registry "github.com/databricks/cli/cmd/workspace/model-registry" @@ -62,11 +63,13 @@ import ( quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors" queries "github.com/databricks/cli/cmd/workspace/queries" queries_legacy "github.com/databricks/cli/cmd/workspace/queries-legacy" + query_execution "github.com/databricks/cli/cmd/workspace/query-execution" query_history "github.com/databricks/cli/cmd/workspace/query-history" query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations" query_visualizations_legacy "github.com/databricks/cli/cmd/workspace/query-visualizations-legacy" recipient_activation "github.com/databricks/cli/cmd/workspace/recipient-activation" recipients "github.com/databricks/cli/cmd/workspace/recipients" + redash_config "github.com/databricks/cli/cmd/workspace/redash-config" registered_models "github.com/databricks/cli/cmd/workspace/registered-models" repos "github.com/databricks/cli/cmd/workspace/repos" resource_quotas "github.com/databricks/cli/cmd/workspace/resource-quotas" @@ -133,6 +136,7 @@ func All() []*cobra.Command { out = append(out, ip_access_lists.New()) out = append(out, jobs.New()) out = append(out, lakeview.New()) + out = append(out, lakeview_embedded.New()) out = append(out, libraries.New()) out = append(out, metastores.New()) out = append(out, model_registry.New()) @@ -156,11 +160,13 @@ func All() []*cobra.Command { out = append(out, quality_monitors.New()) out = append(out, queries.New()) out = append(out, queries_legacy.New()) + out = append(out, query_execution.New()) out = append(out, query_history.New()) out = append(out, query_visualizations.New()) out = append(out, query_visualizations_legacy.New()) out = append(out, recipient_activation.New()) out = append(out, recipients.New()) + out = append(out, redash_config.New()) out = append(out, registered_models.New()) out = append(out, repos.New()) out = append(out, resource_quotas.New()) diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 38a88f014..0f911d400 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -1354,6 +1354,7 @@ func newRunNow() *cobra.Command { // TODO: map via StringToStringVar: job_parameters // TODO: map via StringToStringVar: notebook_params // TODO: array: only + cmd.Flags().Var(&runNowReq.PerformanceTarget, "performance-target", `PerformanceTarget defines how performant or cost efficient the execution of run on serverless compute should be. Supported values: [COST_OPTIMIZED, PERFORMANCE_OPTIMIZED]`) // TODO: complex arg: pipeline_params // TODO: map via StringToStringVar: python_named_params // TODO: array: python_params diff --git a/cmd/workspace/lakeview-embedded/lakeview-embedded.go b/cmd/workspace/lakeview-embedded/lakeview-embedded.go new file mode 100755 index 000000000..ef04c2c13 --- /dev/null +++ b/cmd/workspace/lakeview-embedded/lakeview-embedded.go @@ -0,0 +1,98 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package lakeview_embedded + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "lakeview-embedded", + Short: `Token-based Lakeview APIs for embedding dashboards in external applications.`, + Long: `Token-based Lakeview APIs for embedding dashboards in external applications.`, + GroupID: "dashboards", + Annotations: map[string]string{ + "package": "dashboards", + }, + } + + // Add methods + cmd.AddCommand(newGetPublishedDashboardEmbedded()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start get-published-dashboard-embedded command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getPublishedDashboardEmbeddedOverrides []func( + *cobra.Command, + *dashboards.GetPublishedDashboardEmbeddedRequest, +) + +func newGetPublishedDashboardEmbedded() *cobra.Command { + cmd := &cobra.Command{} + + var getPublishedDashboardEmbeddedReq dashboards.GetPublishedDashboardEmbeddedRequest + + // TODO: short flags + + cmd.Use = "get-published-dashboard-embedded DASHBOARD_ID" + cmd.Short = `Read a published dashboard in an embedded ui.` + cmd.Long = `Read a published dashboard in an embedded ui. + + Get the current published dashboard within an embedded context. + + Arguments: + DASHBOARD_ID: UUID identifying the published dashboard.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getPublishedDashboardEmbeddedReq.DashboardId = args[0] + + err = w.LakeviewEmbedded.GetPublishedDashboardEmbedded(ctx, getPublishedDashboardEmbeddedReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getPublishedDashboardEmbeddedOverrides { + fn(cmd, &getPublishedDashboardEmbeddedReq) + } + + return cmd +} + +// end service LakeviewEmbedded diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index 208f887da..bf74bb3f5 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -406,10 +406,17 @@ func newUpdate() *cobra.Command { Arguments: ID: - UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask - specifies which fields of the setting payload will be updated. The field - mask needs to be supplied as single string. To specify multiple fields in - the field mask, use comma as the separator (no space).` + UPDATE_MASK: The field mask must be a single string, with multiple fields separated by + commas (no spaces). The field path is relative to the resource object, + using a dot (.) to navigate sub-fields (e.g., author.given_name). + Specification of elements in sequence or map fields is not allowed, as + only the entire collection field can be specified. Field names must + exactly match the resource field names. + + A field mask of * indicates full replacement. It’s recommended to + always explicitly list the fields being updated and avoid using * + wildcards, as it can lead to unintended results if the API changes in the + future.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/query-execution/query-execution.go b/cmd/workspace/query-execution/query-execution.go new file mode 100755 index 000000000..ebbb90f89 --- /dev/null +++ b/cmd/workspace/query-execution/query-execution.go @@ -0,0 +1,245 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package query_execution + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "query-execution", + Short: `Query execution APIs for AI / BI Dashboards.`, + Long: `Query execution APIs for AI / BI Dashboards`, + GroupID: "dashboards", + Annotations: map[string]string{ + "package": "dashboards", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCancelPublishedQueryExecution()) + cmd.AddCommand(newExecutePublishedDashboardQuery()) + cmd.AddCommand(newPollPublishedQueryStatus()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start cancel-published-query-execution command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cancelPublishedQueryExecutionOverrides []func( + *cobra.Command, + *dashboards.CancelPublishedQueryExecutionRequest, +) + +func newCancelPublishedQueryExecution() *cobra.Command { + cmd := &cobra.Command{} + + var cancelPublishedQueryExecutionReq dashboards.CancelPublishedQueryExecutionRequest + + // TODO: short flags + + // TODO: array: tokens + + cmd.Use = "cancel-published-query-execution DASHBOARD_NAME DASHBOARD_REVISION_ID" + cmd.Short = `Cancel the results for the a query for a published, embedded dashboard.` + cmd.Long = `Cancel the results for the a query for a published, embedded dashboard.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + cancelPublishedQueryExecutionReq.DashboardName = args[0] + cancelPublishedQueryExecutionReq.DashboardRevisionId = args[1] + + response, err := w.QueryExecution.CancelPublishedQueryExecution(ctx, cancelPublishedQueryExecutionReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range cancelPublishedQueryExecutionOverrides { + fn(cmd, &cancelPublishedQueryExecutionReq) + } + + return cmd +} + +// start execute-published-dashboard-query command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var executePublishedDashboardQueryOverrides []func( + *cobra.Command, + *dashboards.ExecutePublishedDashboardQueryRequest, +) + +func newExecutePublishedDashboardQuery() *cobra.Command { + cmd := &cobra.Command{} + + var executePublishedDashboardQueryReq dashboards.ExecutePublishedDashboardQueryRequest + var executePublishedDashboardQueryJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&executePublishedDashboardQueryJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&executePublishedDashboardQueryReq.OverrideWarehouseId, "override-warehouse-id", executePublishedDashboardQueryReq.OverrideWarehouseId, `A dashboard schedule can override the warehouse used as compute for processing the published dashboard queries.`) + + cmd.Use = "execute-published-dashboard-query DASHBOARD_NAME DASHBOARD_REVISION_ID" + cmd.Short = `Execute a query for a published dashboard.` + cmd.Long = `Execute a query for a published dashboard. + + Arguments: + DASHBOARD_NAME: Dashboard name and revision_id is required to retrieve + PublishedDatasetDataModel which contains the list of datasets, + warehouse_id, and embedded_credentials + DASHBOARD_REVISION_ID: ` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'dashboard_name', 'dashboard_revision_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := executePublishedDashboardQueryJson.Unmarshal(&executePublishedDashboardQueryReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + executePublishedDashboardQueryReq.DashboardName = args[0] + } + if !cmd.Flags().Changed("json") { + executePublishedDashboardQueryReq.DashboardRevisionId = args[1] + } + + err = w.QueryExecution.ExecutePublishedDashboardQuery(ctx, executePublishedDashboardQueryReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range executePublishedDashboardQueryOverrides { + fn(cmd, &executePublishedDashboardQueryReq) + } + + return cmd +} + +// start poll-published-query-status command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var pollPublishedQueryStatusOverrides []func( + *cobra.Command, + *dashboards.PollPublishedQueryStatusRequest, +) + +func newPollPublishedQueryStatus() *cobra.Command { + cmd := &cobra.Command{} + + var pollPublishedQueryStatusReq dashboards.PollPublishedQueryStatusRequest + + // TODO: short flags + + // TODO: array: tokens + + cmd.Use = "poll-published-query-status DASHBOARD_NAME DASHBOARD_REVISION_ID" + cmd.Short = `Poll the results for the a query for a published, embedded dashboard.` + cmd.Long = `Poll the results for the a query for a published, embedded dashboard.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + pollPublishedQueryStatusReq.DashboardName = args[0] + pollPublishedQueryStatusReq.DashboardRevisionId = args[1] + + response, err := w.QueryExecution.PollPublishedQueryStatus(ctx, pollPublishedQueryStatusReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range pollPublishedQueryStatusOverrides { + fn(cmd, &pollPublishedQueryStatusReq) + } + + return cmd +} + +// end service QueryExecution diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go index 621661952..2d50229ba 100755 --- a/cmd/workspace/query-visualizations/query-visualizations.go +++ b/cmd/workspace/query-visualizations/query-visualizations.go @@ -198,10 +198,17 @@ func newUpdate() *cobra.Command { Arguments: ID: - UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask - specifies which fields of the setting payload will be updated. The field - mask needs to be supplied as single string. To specify multiple fields in - the field mask, use comma as the separator (no space).` + UPDATE_MASK: The field mask must be a single string, with multiple fields separated by + commas (no spaces). The field path is relative to the resource object, + using a dot (.) to navigate sub-fields (e.g., author.given_name). + Specification of elements in sequence or map fields is not allowed, as + only the entire collection field can be specified. Field names must + exactly match the resource field names. + + A field mask of * indicates full replacement. It’s recommended to + always explicitly list the fields being updated and avoid using * + wildcards, as it can lead to unintended results if the API changes in the + future.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/redash-config/redash-config.go b/cmd/workspace/redash-config/redash-config.go new file mode 100755 index 000000000..1a0f37759 --- /dev/null +++ b/cmd/workspace/redash-config/redash-config.go @@ -0,0 +1,80 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package redash_config + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "redash-config", + Short: `Redash V2 service for workspace configurations (internal).`, + Long: `Redash V2 service for workspace configurations (internal)`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newGetConfig()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start get-config command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getConfigOverrides []func( + *cobra.Command, +) + +func newGetConfig() *cobra.Command { + cmd := &cobra.Command{} + + cmd.Use = "get-config" + cmd.Short = `Read workspace configuration for Redash-v2.` + cmd.Long = `Read workspace configuration for Redash-v2.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + response, err := w.RedashConfig.GetConfig(ctx) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getConfigOverrides { + fn(cmd) + } + + return cmd +} + +// end service RedashConfig diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index 034133623..645111646 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -642,7 +642,8 @@ func newHttpRequest() *cobra.Command { if err != nil { return err } - return cmdio.Render(ctx, response) + defer response.Contents.Close() + return cmdio.Render(ctx, response.Contents) } // Disable completions since they are not applicable. diff --git a/go.mod b/go.mod index 662fcd40b..db3261beb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.56.1 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.57.0 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 diff --git a/go.sum b/go.sum index bffc3b53d..d6102a0d1 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.56.1 h1:sgweTRvAQaI8EPrfDnVdAB0lNX6L5uTT720SlMMQI2U= -github.com/databricks/databricks-sdk-go v0.56.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= +github.com/databricks/databricks-sdk-go v0.57.0 h1:Vs3a+Zmg403er4+xpD7ZTQWm7e51d2q3yYEyIIgvtYw= +github.com/databricks/databricks-sdk-go v0.57.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From d282f33a22072525db102ba5b2f5952f001f70ed Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 14:00:49 +0100 Subject: [PATCH 079/105] Append newline to "-o json" for validate/summary/run (#2326) ## Changes - Insert newline after rendering indented JSON in bundle validate/summary/run. - This prevents "No newline at end of file" message in various cases, for example when switching between recording raw output of the command to output processed by jq, since jq does add a newline or when running diff in acceptance tests. ## Tests Manually running validate: ``` ~/work/dabs_cuj_brickfood % ../cli/cli-main bundle validate -o json | tail -n 2 # without change Error: root_path must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development' } }% ~/work/dabs_cuj_brickfood % ../cli/cli bundle validate -o json | tail -n 2 # with change Error: root_path must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development' } } ~/work/dabs_cuj_brickfood % ``` Via #2316 -- see cleaner output there. --- acceptance/bundle/variables/host/output.txt | 1 + cmd/bundle/run.go | 1 + cmd/bundle/summary.go | 1 + cmd/bundle/validate.go | 4 +++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/variables/host/output.txt b/acceptance/bundle/variables/host/output.txt index 63c41426a..df0a4527a 100644 --- a/acceptance/bundle/variables/host/output.txt +++ b/acceptance/bundle/variables/host/output.txt @@ -23,6 +23,7 @@ Error: failed during request visitor: parse "https://${var.host}": invalid chara "host": "${var.host}" } } + Exit code: 1 >>> errcode [CLI] bundle validate diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index df35d7222..ffb9c1b88 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -173,6 +173,7 @@ task or a Python wheel task, the second example applies. if err != nil { return err } + _, _ = cmd.OutOrStdout().Write([]byte{'\n'}) default: return fmt.Errorf("unknown output type %s", root.OutputType(cmd)) } diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index 7c669c845..2871c82ff 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -74,6 +74,7 @@ func newSummaryCommand() *cobra.Command { return err } _, _ = cmd.OutOrStdout().Write(buf) + _, _ = cmd.OutOrStdout().Write([]byte{'\n'}) default: return fmt.Errorf("unknown output type %s", root.OutputType(cmd)) } diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 41fa87f30..c45453af6 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -20,7 +20,9 @@ func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle) error { if err != nil { return err } - _, _ = cmd.OutOrStdout().Write(buf) + out := cmd.OutOrStdout() + _, _ = out.Write(buf) + _, _ = out.Write([]byte{'\n'}) return nil } From ddedc4272d0f07eb49b2d23bea0dfba5f0773d33 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 10 Feb 2025 18:35:12 +0530 Subject: [PATCH 080/105] Return 501 status code when API stub is not implemented (#2327) ## Changes Addresses feedback from https://github.com/databricks/cli/pull/2292#discussion_r1946846865 ## Tests Manually, confirmed that unstubbed API calls still cause acceptance tests to fail. --- libs/testserver/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/testserver/server.go b/libs/testserver/server.go index d0c340c12..d5877e90e 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -76,7 +76,7 @@ Response.StatusCode = return apierr.APIError{ Message: "No stub found for pattern: " + pattern, - }, http.StatusNotFound + }, http.StatusNotImplemented }) return s From 6953a84db6d85fc72e7b5bb6347acb59d68bbfa5 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 10 Feb 2025 19:03:27 +0530 Subject: [PATCH 081/105] Serialize recorded requests with indentation in acceptance tests (#2329) ## Changes This PR indents the recorded requests to make them easier to review. They can still be parsed using jq. ## Tests Existing tests. --- acceptance/acceptance_test.go | 2 +- .../workspace/jobs/create-error/out.requests.txt | 8 +++++++- .../workspace/jobs/create/out.requests.txt | 16 +++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 241ab42be..b05b10f47 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -318,7 +318,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont require.NoError(t, err) for _, req := range server.Requests { - reqJson, err := json.Marshal(req) + reqJson, err := json.MarshalIndent(req, "", " ") require.NoError(t, err) reqJsonWithRepls := repls.Replace(string(reqJson)) diff --git a/acceptance/workspace/jobs/create-error/out.requests.txt b/acceptance/workspace/jobs/create-error/out.requests.txt index b22876b70..30f104fd1 100644 --- a/acceptance/workspace/jobs/create-error/out.requests.txt +++ b/acceptance/workspace/jobs/create-error/out.requests.txt @@ -1 +1,7 @@ -{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{ + "method": "POST", + "path": "/api/2.1/jobs/create", + "body": { + "name": "abc" + } +} diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 2510762db..1d200a547 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1,15 @@ -{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ], + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.1/jobs/create", + "body": { + "name": "abc" + } +} From 4bc231ad4f0d0bead1c1c26e11dedd1480d4e892 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:58:18 +0100 Subject: [PATCH 082/105] Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0 (#2322) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.25.0 to 0.26.0.
Commits
  • b9c813b google: add warning about externally-provided credentials
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.25.0&new-version=0.26.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db3261beb..665e258bb 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/wI2L/jsondiff v0.6.1 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.22.0 - golang.org/x/oauth2 v0.25.0 + golang.org/x/oauth2 v0.26.0 golang.org/x/sync v0.10.0 golang.org/x/term v0.28.0 golang.org/x/text v0.21.0 diff --git a/go.sum b/go.sum index d6102a0d1..87118cf0c 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= +golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From f7a45d0c7ef2e3322509e3c214d7927bba61afa9 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 10 Feb 2025 14:06:02 +0000 Subject: [PATCH 083/105] Upgrade to TF provider 1.65.1 (#2328) ## Changes Upgrade to TF provider 1.65.1 Notable changes: - Now it's possible to use `run_as` field in `pipelines` definition - Added support for `performance_target` for `jobs` --- acceptance/terraform/main.tf | 2 +- acceptance/terraform/output.txt | 6 +- .../terraform/tfdyn/convert_pipeline_test.go | 3 + bundle/internal/tf/codegen/README.md | 3 + bundle/internal/tf/codegen/schema/version.go | 2 +- .../internal/tf/schema/data_source_catalog.go | 1 - ...shboard_embedding_access_policy_setting.go | 14 ++ ...oard_embedding_approved_domains_setting.go | 14 ++ .../schema/resource_custom_app_integration.go | 25 +- bundle/internal/tf/schema/resource_job.go | 1 + .../internal/tf/schema/resource_pipeline.go | 6 + bundle/internal/tf/schema/resources.go | 214 +++++++++--------- bundle/internal/tf/schema/root.go | 2 +- 13 files changed, 169 insertions(+), 124 deletions(-) create mode 100644 bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go create mode 100644 bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf index 93f665ff4..674b41a3c 100644 --- a/acceptance/terraform/main.tf +++ b/acceptance/terraform/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { databricks = { source = "databricks/databricks" - version = "1.64.1" + version = "1.65.1" } } diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt index 6bdc809f6..851785827 100644 --- a/acceptance/terraform/output.txt +++ b/acceptance/terraform/output.txt @@ -4,9 +4,9 @@ Initializing the backend... Initializing provider plugins... -- Finding databricks/databricks versions matching "1.64.1"... -- Installing databricks/databricks v1.64.1... -- Installed databricks/databricks v1.64.1 (unauthenticated) +- Finding databricks/databricks versions matching "1.65.1"... +- Installing databricks/databricks v1.65.1... +- Installed databricks/databricks v1.65.1 (unauthenticated) Terraform has created a lock file .terraform.lock.hcl to record the provider selections it made above. Include this file in your version control repository diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go index d8de55bf0..63d023c43 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -122,6 +122,9 @@ func TestConvertPipeline(t *testing.T) { "num_workers": int64(1), }, }, + "run_as": map[string]any{ + "user_name": "foo@bar.com", + }, }, out.Pipeline["my_pipeline"]) // Assert equality on the permissions diff --git a/bundle/internal/tf/codegen/README.md b/bundle/internal/tf/codegen/README.md index b1f8a33a8..968bf29ed 100644 --- a/bundle/internal/tf/codegen/README.md +++ b/bundle/internal/tf/codegen/README.md @@ -19,3 +19,6 @@ How to regenerate Go structs from an updated terraform provider? 2. Delete `./tmp` if it exists 3. Run `go run .` 4. Run `gofmt -s -w ../schema` +5. Go back to the root of the repo. +6. Update `/acceptance/terraform/main.tf` file to use new version of TF provider +7. Run `go test ./acceptance -v -update -run TestAccept/terraform` to update test output with a new version of TF provider diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 393afd6ed..46548f3e8 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.64.1" +const ProviderVersion = "1.65.1" diff --git a/bundle/internal/tf/schema/data_source_catalog.go b/bundle/internal/tf/schema/data_source_catalog.go index 6f9237cfa..4b8c6df97 100644 --- a/bundle/internal/tf/schema/data_source_catalog.go +++ b/bundle/internal/tf/schema/data_source_catalog.go @@ -28,7 +28,6 @@ type DataSourceCatalogCatalogInfo struct { Owner string `json:"owner,omitempty"` Properties map[string]string `json:"properties,omitempty"` ProviderName string `json:"provider_name,omitempty"` - SecurableKind string `json:"securable_kind,omitempty"` SecurableType string `json:"securable_type,omitempty"` ShareName string `json:"share_name,omitempty"` StorageLocation string `json:"storage_location,omitempty"` diff --git a/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go new file mode 100644 index 000000000..d816b235d --- /dev/null +++ b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_access_policy_setting.go @@ -0,0 +1,14 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceAibiDashboardEmbeddingAccessPolicySettingAibiDashboardEmbeddingAccessPolicy struct { + AccessPolicyType string `json:"access_policy_type"` +} + +type ResourceAibiDashboardEmbeddingAccessPolicySetting struct { + Etag string `json:"etag,omitempty"` + Id string `json:"id,omitempty"` + SettingName string `json:"setting_name,omitempty"` + AibiDashboardEmbeddingAccessPolicy *ResourceAibiDashboardEmbeddingAccessPolicySettingAibiDashboardEmbeddingAccessPolicy `json:"aibi_dashboard_embedding_access_policy,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go new file mode 100644 index 000000000..690b334cd --- /dev/null +++ b/bundle/internal/tf/schema/resource_aibi_dashboard_embedding_approved_domains_setting.go @@ -0,0 +1,14 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceAibiDashboardEmbeddingApprovedDomainsSettingAibiDashboardEmbeddingApprovedDomains struct { + ApprovedDomains []string `json:"approved_domains"` +} + +type ResourceAibiDashboardEmbeddingApprovedDomainsSetting struct { + Etag string `json:"etag,omitempty"` + Id string `json:"id,omitempty"` + SettingName string `json:"setting_name,omitempty"` + AibiDashboardEmbeddingApprovedDomains *ResourceAibiDashboardEmbeddingApprovedDomainsSettingAibiDashboardEmbeddingApprovedDomains `json:"aibi_dashboard_embedding_approved_domains,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_custom_app_integration.go b/bundle/internal/tf/schema/resource_custom_app_integration.go index e89eb7fe5..0a964f6ab 100644 --- a/bundle/internal/tf/schema/resource_custom_app_integration.go +++ b/bundle/internal/tf/schema/resource_custom_app_integration.go @@ -8,16 +8,17 @@ type ResourceCustomAppIntegrationTokenAccessPolicy struct { } type ResourceCustomAppIntegration struct { - ClientId string `json:"client_id,omitempty"` - ClientSecret string `json:"client_secret,omitempty"` - Confidential bool `json:"confidential,omitempty"` - CreateTime string `json:"create_time,omitempty"` - CreatedBy int `json:"created_by,omitempty"` - CreatorUsername string `json:"creator_username,omitempty"` - Id string `json:"id,omitempty"` - IntegrationId string `json:"integration_id,omitempty"` - Name string `json:"name,omitempty"` - RedirectUrls []string `json:"redirect_urls,omitempty"` - Scopes []string `json:"scopes,omitempty"` - TokenAccessPolicy *ResourceCustomAppIntegrationTokenAccessPolicy `json:"token_access_policy,omitempty"` + ClientId string `json:"client_id,omitempty"` + ClientSecret string `json:"client_secret,omitempty"` + Confidential bool `json:"confidential,omitempty"` + CreateTime string `json:"create_time,omitempty"` + CreatedBy int `json:"created_by,omitempty"` + CreatorUsername string `json:"creator_username,omitempty"` + Id string `json:"id,omitempty"` + IntegrationId string `json:"integration_id,omitempty"` + Name string `json:"name,omitempty"` + RedirectUrls []string `json:"redirect_urls,omitempty"` + Scopes []string `json:"scopes,omitempty"` + UserAuthorizedScopes []string `json:"user_authorized_scopes,omitempty"` + TokenAccessPolicy *ResourceCustomAppIntegrationTokenAccessPolicy `json:"token_access_policy,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index da277b5c1..2c27f0be7 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -1489,6 +1489,7 @@ type ResourceJob struct { MaxRetries int `json:"max_retries,omitempty"` MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` Name string `json:"name,omitempty"` + PerformanceTarget string `json:"performance_target,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` Tags map[string]string `json:"tags,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index ebdb85027..8e260e65c 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -249,6 +249,11 @@ type ResourcePipelineRestartWindow struct { TimeZoneId string `json:"time_zone_id,omitempty"` } +type ResourcePipelineRunAs struct { + ServicePrincipalName string `json:"service_principal_name,omitempty"` + UserName string `json:"user_name,omitempty"` +} + type ResourcePipelineTriggerCron struct { QuartzCronSchedule string `json:"quartz_cron_schedule,omitempty"` TimezoneId string `json:"timezone_id,omitempty"` @@ -296,5 +301,6 @@ type ResourcePipeline struct { Library []ResourcePipelineLibrary `json:"library,omitempty"` Notification []ResourcePipelineNotification `json:"notification,omitempty"` RestartWindow *ResourcePipelineRestartWindow `json:"restart_window,omitempty"` + RunAs *ResourcePipelineRunAs `json:"run_as,omitempty"` Trigger *ResourcePipelineTrigger `json:"trigger,omitempty"` } diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index b57c2711a..c6eaa5b21 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -3,115 +3,119 @@ package schema type Resources struct { - AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"` - Alert map[string]any `json:"databricks_alert,omitempty"` - App map[string]any `json:"databricks_app,omitempty"` - ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"` - AutomaticClusterUpdateWorkspaceSetting map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"` - AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"` - AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` - AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` - AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` - Budget map[string]any `json:"databricks_budget,omitempty"` - Catalog map[string]any `json:"databricks_catalog,omitempty"` - CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` - Cluster map[string]any `json:"databricks_cluster,omitempty"` - ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` - ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` - Connection map[string]any `json:"databricks_connection,omitempty"` - Credential map[string]any `json:"databricks_credential,omitempty"` - CustomAppIntegration map[string]any `json:"databricks_custom_app_integration,omitempty"` - Dashboard map[string]any `json:"databricks_dashboard,omitempty"` - DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` - DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` - Directory map[string]any `json:"databricks_directory,omitempty"` - EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"` - Entitlements map[string]any `json:"databricks_entitlements,omitempty"` - ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` - File map[string]any `json:"databricks_file,omitempty"` - GitCredential map[string]any `json:"databricks_git_credential,omitempty"` - GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"` - Grant map[string]any `json:"databricks_grant,omitempty"` - Grants map[string]any `json:"databricks_grants,omitempty"` - Group map[string]any `json:"databricks_group,omitempty"` - GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"` - GroupMember map[string]any `json:"databricks_group_member,omitempty"` - GroupRole map[string]any `json:"databricks_group_role,omitempty"` - InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` - InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"` - IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"` - Job map[string]any `json:"databricks_job,omitempty"` - LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"` - Library map[string]any `json:"databricks_library,omitempty"` - Metastore map[string]any `json:"databricks_metastore,omitempty"` - MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"` - MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"` - MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` - MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` - MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"` - ModelServing map[string]any `json:"databricks_model_serving,omitempty"` - Mount map[string]any `json:"databricks_mount,omitempty"` - MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` - MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"` - MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"` - MwsNccBinding map[string]any `json:"databricks_mws_ncc_binding,omitempty"` - MwsNccPrivateEndpointRule map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"` - MwsNetworkConnectivityConfig map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"` - MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"` - MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"` - MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"` - MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"` - MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` - MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` - Notebook map[string]any `json:"databricks_notebook,omitempty"` - NotificationDestination map[string]any `json:"databricks_notification_destination,omitempty"` - OboToken map[string]any `json:"databricks_obo_token,omitempty"` - OnlineTable map[string]any `json:"databricks_online_table,omitempty"` - PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` - Permissions map[string]any `json:"databricks_permissions,omitempty"` - Pipeline map[string]any `json:"databricks_pipeline,omitempty"` - Provider map[string]any `json:"databricks_provider,omitempty"` - QualityMonitor map[string]any `json:"databricks_quality_monitor,omitempty"` - Query map[string]any `json:"databricks_query,omitempty"` - Recipient map[string]any `json:"databricks_recipient,omitempty"` - RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` - Repo map[string]any `json:"databricks_repo,omitempty"` - RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"` - Schema map[string]any `json:"databricks_schema,omitempty"` - Secret map[string]any `json:"databricks_secret,omitempty"` - SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"` - SecretScope map[string]any `json:"databricks_secret_scope,omitempty"` - ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` - ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"` - ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"` - Share map[string]any `json:"databricks_share,omitempty"` - SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"` - SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"` - SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"` - SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"` - SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"` - SqlQuery map[string]any `json:"databricks_sql_query,omitempty"` - SqlTable map[string]any `json:"databricks_sql_table,omitempty"` - SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"` - SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"` - StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` - SystemSchema map[string]any `json:"databricks_system_schema,omitempty"` - Table map[string]any `json:"databricks_table,omitempty"` - Token map[string]any `json:"databricks_token,omitempty"` - User map[string]any `json:"databricks_user,omitempty"` - UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"` - UserRole map[string]any `json:"databricks_user_role,omitempty"` - VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` - VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` - Volume map[string]any `json:"databricks_volume,omitempty"` - WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` - WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` - WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` + AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"` + AibiDashboardEmbeddingAccessPolicySetting map[string]any `json:"databricks_aibi_dashboard_embedding_access_policy_setting,omitempty"` + AibiDashboardEmbeddingApprovedDomainsSetting map[string]any `json:"databricks_aibi_dashboard_embedding_approved_domains_setting,omitempty"` + Alert map[string]any `json:"databricks_alert,omitempty"` + App map[string]any `json:"databricks_app,omitempty"` + ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"` + AutomaticClusterUpdateWorkspaceSetting map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"` + AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"` + AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` + AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` + AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` + Budget map[string]any `json:"databricks_budget,omitempty"` + Catalog map[string]any `json:"databricks_catalog,omitempty"` + CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` + Cluster map[string]any `json:"databricks_cluster,omitempty"` + ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` + ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` + Connection map[string]any `json:"databricks_connection,omitempty"` + Credential map[string]any `json:"databricks_credential,omitempty"` + CustomAppIntegration map[string]any `json:"databricks_custom_app_integration,omitempty"` + Dashboard map[string]any `json:"databricks_dashboard,omitempty"` + DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` + DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` + Directory map[string]any `json:"databricks_directory,omitempty"` + EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"` + Entitlements map[string]any `json:"databricks_entitlements,omitempty"` + ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` + File map[string]any `json:"databricks_file,omitempty"` + GitCredential map[string]any `json:"databricks_git_credential,omitempty"` + GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"` + Grant map[string]any `json:"databricks_grant,omitempty"` + Grants map[string]any `json:"databricks_grants,omitempty"` + Group map[string]any `json:"databricks_group,omitempty"` + GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"` + GroupMember map[string]any `json:"databricks_group_member,omitempty"` + GroupRole map[string]any `json:"databricks_group_role,omitempty"` + InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` + InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"` + IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"` + Job map[string]any `json:"databricks_job,omitempty"` + LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"` + Library map[string]any `json:"databricks_library,omitempty"` + Metastore map[string]any `json:"databricks_metastore,omitempty"` + MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"` + MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"` + MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` + MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"` + ModelServing map[string]any `json:"databricks_model_serving,omitempty"` + Mount map[string]any `json:"databricks_mount,omitempty"` + MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` + MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"` + MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"` + MwsNccBinding map[string]any `json:"databricks_mws_ncc_binding,omitempty"` + MwsNccPrivateEndpointRule map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"` + MwsNetworkConnectivityConfig map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"` + MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"` + MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"` + MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"` + MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"` + MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` + MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` + Notebook map[string]any `json:"databricks_notebook,omitempty"` + NotificationDestination map[string]any `json:"databricks_notification_destination,omitempty"` + OboToken map[string]any `json:"databricks_obo_token,omitempty"` + OnlineTable map[string]any `json:"databricks_online_table,omitempty"` + PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` + Permissions map[string]any `json:"databricks_permissions,omitempty"` + Pipeline map[string]any `json:"databricks_pipeline,omitempty"` + Provider map[string]any `json:"databricks_provider,omitempty"` + QualityMonitor map[string]any `json:"databricks_quality_monitor,omitempty"` + Query map[string]any `json:"databricks_query,omitempty"` + Recipient map[string]any `json:"databricks_recipient,omitempty"` + RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` + Repo map[string]any `json:"databricks_repo,omitempty"` + RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` + Secret map[string]any `json:"databricks_secret,omitempty"` + SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"` + SecretScope map[string]any `json:"databricks_secret_scope,omitempty"` + ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` + ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"` + ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"` + Share map[string]any `json:"databricks_share,omitempty"` + SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"` + SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"` + SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"` + SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"` + SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"` + SqlQuery map[string]any `json:"databricks_sql_query,omitempty"` + SqlTable map[string]any `json:"databricks_sql_table,omitempty"` + SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"` + SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"` + StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` + SystemSchema map[string]any `json:"databricks_system_schema,omitempty"` + Table map[string]any `json:"databricks_table,omitempty"` + Token map[string]any `json:"databricks_token,omitempty"` + User map[string]any `json:"databricks_user,omitempty"` + UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"` + UserRole map[string]any `json:"databricks_user_role,omitempty"` + VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` + VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` + WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` + WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } func NewResources() *Resources { return &Resources{ - AccessControlRuleSet: make(map[string]any), + AccessControlRuleSet: make(map[string]any), + AibiDashboardEmbeddingAccessPolicySetting: make(map[string]any), + AibiDashboardEmbeddingApprovedDomainsSetting: make(map[string]any), Alert: make(map[string]any), App: make(map[string]any), ArtifactAllowlist: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 2ac852355..816e8e6aa 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.64.1" +const ProviderVersion = "1.65.1" func NewRoot() *Root { return &Root{ From f6c50a631801f0a08a549b358eb129283d1e6bc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:49:52 +0100 Subject: [PATCH 084/105] Bump golang.org/x/term from 0.28.0 to 0.29.0 (#2325) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.28.0 to 0.29.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.28.0&new-version=0.29.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 665e258bb..11bc1f87f 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( golang.org/x/mod v0.22.0 golang.org/x/oauth2 v0.26.0 golang.org/x/sync v0.10.0 - golang.org/x/term v0.28.0 + golang.org/x/term v0.29.0 golang.org/x/text v0.21.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -71,7 +71,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/sys v0.30.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 87118cf0c..523a44abd 100644 --- a/go.sum +++ b/go.sum @@ -227,10 +227,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= From 6f3dbaec4ccb97cb83b4807070da43662457f262 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 16:13:38 +0100 Subject: [PATCH 085/105] Bump golang.org/x/text from 0.21.0 to 0.22.0 (#2323) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.21.0 to 0.22.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 11bc1f87f..16242a09e 100644 --- a/go.mod +++ b/go.mod @@ -29,9 +29,9 @@ require ( golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.22.0 golang.org/x/oauth2 v0.26.0 - golang.org/x/sync v0.10.0 + golang.org/x/sync v0.11.0 golang.org/x/term v0.29.0 - golang.org/x/text v0.21.0 + golang.org/x/text v0.22.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 523a44abd..13e51beec 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,8 +233,8 @@ golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From e81ec4ee237bd6066f1d45a0c3a789c22e626104 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:13:49 +0100 Subject: [PATCH 086/105] Bump golang.org/x/mod from 0.22.0 to 0.23.0 (#2324) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.22.0 to 0.23.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.22.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 16242a09e..c8b209edd 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/stretchr/testify v1.10.0 // MIT github.com/wI2L/jsondiff v0.6.1 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.22.0 + golang.org/x/mod v0.23.0 golang.org/x/oauth2 v0.26.0 golang.org/x/sync v0.11.0 golang.org/x/term v0.29.0 diff --git a/go.sum b/go.sum index 13e51beec..0369fc2d9 100644 --- a/go.sum +++ b/go.sum @@ -199,8 +199,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From f2096eddcc7e586216c36911430fa67bd40a427c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Feb 2025 10:38:53 +0100 Subject: [PATCH 087/105] acc: Do not show all replacements on every failure (#2331) ## Changes - Only print replacements if VERBOSE_TEST flag is set. - This is set on CI but not when you do "go test" or "make test". Note, env var is used, so that it can be set in Makefile. ## Tests Manually. --- Makefile | 4 ++-- acceptance/acceptance_test.go | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index fb3936184..0c3860e29 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test: cover: rm -fr ./acceptance/build/cover/ - CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES} + VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES} rm -fr ./acceptance/build/cover-merged/ mkdir -p acceptance/build/cover-merged/ go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/ @@ -61,6 +61,6 @@ integration: vendor $(INTEGRATION) integration-short: vendor - $(INTEGRATION) -short + VERBOSE_TEST=1 $(INTEGRATION) -short .PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index b05b10f47..94db3232d 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -30,8 +30,9 @@ import ( ) var ( - KeepTmp bool - NoRepl bool + KeepTmp bool + NoRepl bool + VerboseTest bool = os.Getenv("VERBOSE_TEST") != "" ) // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" @@ -412,7 +413,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN testutil.WriteFile(t, pathRef, valueNew) } - if !equal && printedRepls != nil && !*printedRepls { + if VerboseTest && !equal && printedRepls != nil && !*printedRepls { *printedRepls = true var items []string for _, item := range repls.Repls { From 8d849fe868ed2807cb47d037323847a0b1bfc4b4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Feb 2025 11:37:48 +0100 Subject: [PATCH 088/105] acc: Disable custom server on CLOUD_ENV (#2332) We're not using local server when CLOUD_ENV is enabled, no need to set up a custom one. --- acceptance/acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 94db3232d..320948fda 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -262,7 +262,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // 2. The test is configured to record requests and assert on them. We need // a duplicate of the default server to record requests because the default // server otherwise is a shared resource. - if len(config.Server) > 0 || config.RecordRequests { + if cloudEnv == "" && (len(config.Server) > 0 || config.RecordRequests) { server = testserver.New(t) server.RecordRequests = config.RecordRequests server.IncludeRequestHeaders = config.IncludeRequestHeaders From 878fa803224a99176dcf2e4b4e94342389cb8a5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Feb 2025 11:50:52 +0100 Subject: [PATCH 089/105] acc: Fix RecordRequests to support requests without body (#2333) ## Changes Do not paste request body into output if it's not a valid JSON. ## Tests While working on #2334 I found that if I try to record a test that calls /api/2.0/preview/scim/v2/Me which has no request body, it crashes. --- libs/testserver/server.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libs/testserver/server.go b/libs/testserver/server.go index d5877e90e..577ef082c 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -34,7 +34,8 @@ type Request struct { Headers http.Header `json:"headers,omitempty"` Method string `json:"method"` Path string `json:"path"` - Body any `json:"body"` + Body any `json:"body,omitempty"` + RawBody string `json:"raw_body,omitempty"` } func New(t testutil.TestingT) *Server { @@ -119,13 +120,19 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } } - s.Requests = append(s.Requests, Request{ + req := Request{ Headers: headers, Method: r.Method, Path: r.URL.Path, - Body: json.RawMessage(body), - }) + } + if json.Valid(body) { + req.Body = json.RawMessage(body) + } else { + req.RawBody = string(body) + } + + s.Requests = append(s.Requests, req) } w.Header().Set("Content-Type", "application/json") From 272ce6130272c28f6c8e0a9113a3727b952282fe Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Feb 2025 15:26:46 +0100 Subject: [PATCH 090/105] acc: Fix singleTest option to support forward slashes (#2336) The filtering of tests needs to see forward slashes otherwise it is OS-dependent. I've also switched to filepath.ToSlash but it should be a no-op. --- acceptance/acceptance_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 320948fda..fce508498 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -180,8 +180,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { } for _, dir := range testDirs { - testName := strings.ReplaceAll(dir, "\\", "/") - t.Run(testName, func(t *testing.T) { + t.Run(dir, func(t *testing.T) { if !InprocessMode { t.Parallel() } @@ -203,7 +202,8 @@ func getTests(t *testing.T) []string { name := filepath.Base(path) if name == EntryPointScript { // Presence of 'script' marks a test case in this directory - testDirs = append(testDirs, filepath.Dir(path)) + testName := filepath.ToSlash(filepath.Dir(path)) + testDirs = append(testDirs, testName) } return nil }) From 5d392acbef7873d3dcb54b314181381da89dda03 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Feb 2025 16:03:41 +0100 Subject: [PATCH 091/105] acc: Allow mixing custom stubs with default server impl (#2334) ## Changes - Currently if you define [[Server]] block, you disable the default server implementation. With this change [[Server]] block takes precedence over default server but default server remains. - Switched mux implementation to [gorilla/mux](https://github.com/gorilla/mux) -- unlike built-in it does not panic if you set two handlers on the same part (instead the earliest one wins). It also does not have any dependencies. - Move acceptance/selftest into acceptance/selftest/basic and added acceptance/selftest/server that demoes server override. - Rewrite server set up to ensure that env vars and replacements are set up correctly. Previously replacements for DATABRICKS_HOST referred to default server, not to the custom server. - Avoid calling CurrentUser.Me() in the local case. This allows overriding /api/2.0/preview/scim/v2/Me, which we use in some tests (e.g. bundle/templates-machinery/helpers-error). Previously the test passed because CurrentUser.Me() was calling default server which is incorrect but it happened to make the tests pass. - The default server is now available on DATABRICKS_DEFAULT_HOST env var. - Rewrite "not found" handler in local test to handle error better (do not raise http500 when header is already written). ## Tests New acceptance test selftest/server specifically tests that both custom and default handlers are available in a single test. --- NOTICE | 4 + acceptance/acceptance_test.go | 123 ++++++++++-------- acceptance/auth/bundle_and_profile/output.txt | 4 +- acceptance/auth/bundle_and_profile/test.toml | 6 +- acceptance/cmd_server_test.go | 6 +- acceptance/selftest/{ => basic}/out.hello.txt | 0 acceptance/selftest/{ => basic}/output.txt | 0 acceptance/selftest/{ => basic}/script | 0 acceptance/selftest/{ => basic}/test.toml | 0 acceptance/selftest/server/out.requests.txt | 8 ++ acceptance/selftest/server/output.txt | 15 +++ acceptance/selftest/server/script | 2 + acceptance/selftest/server/test.toml | 18 +++ acceptance/server_test.go | 45 ++++--- go.mod | 1 + go.sum | 2 + libs/testserver/server.go | 39 ++++-- 17 files changed, 179 insertions(+), 94 deletions(-) rename acceptance/selftest/{ => basic}/out.hello.txt (100%) rename acceptance/selftest/{ => basic}/output.txt (100%) rename acceptance/selftest/{ => basic}/script (100%) rename acceptance/selftest/{ => basic}/test.toml (100%) create mode 100644 acceptance/selftest/server/out.requests.txt create mode 100644 acceptance/selftest/server/output.txt create mode 100644 acceptance/selftest/server/script create mode 100644 acceptance/selftest/server/test.toml diff --git a/NOTICE b/NOTICE index 4331a2a32..0b1d2da04 100644 --- a/NOTICE +++ b/NOTICE @@ -114,3 +114,7 @@ dario.cat/mergo Copyright (c) 2013 Dario Castañé. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. https://github.com/darccio/mergo/blob/master/LICENSE + +https://github.com/gorilla/mux +Copyright (c) 2023 The Gorilla Authors. All rights reserved. +https://github.com/gorilla/mux/blob/main/LICENSE diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index fce508498..117172f60 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "runtime" "slices" "sort" @@ -26,6 +27,7 @@ import ( "github.com/databricks/cli/libs/testdiff" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/require" ) @@ -72,7 +74,8 @@ func TestInprocessMode(t *testing.T) { if InprocessMode { t.Skip("Already tested by TestAccept") } - require.Equal(t, 1, testAccept(t, true, "selftest")) + require.Equal(t, 1, testAccept(t, true, "selftest/basic")) + require.Equal(t, 1, testAccept(t, true, "selftest/server")) } func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { @@ -118,14 +121,12 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { uvCache := getUVDefaultCacheDir(t) t.Setenv("UV_CACHE_DIR", uvCache) - ctx := context.Background() cloudEnv := os.Getenv("CLOUD_ENV") if cloudEnv == "" { defaultServer := testserver.New(t) AddHandlers(defaultServer) - // Redirect API access to local server: - t.Setenv("DATABRICKS_HOST", defaultServer.URL) + t.Setenv("DATABRICKS_DEFAULT_HOST", defaultServer.URL) homeDir := t.TempDir() // Do not read user's ~/.databrickscfg @@ -148,27 +149,12 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") - var config databricks.Config - if cloudEnv == "" { - // use fake token for local tests - config = databricks.Config{Token: "dbapi1234"} - } else { - // non-local tests rely on environment variables - config = databricks.Config{} - } - workspaceClient, err := databricks.NewWorkspaceClient(&config) - require.NoError(t, err) - - user, err := workspaceClient.CurrentUser.Me(ctx) - require.NoError(t, err) - require.NotNil(t, user) - testdiff.PrepareReplacementsUser(t, &repls, *user) - testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient) - testdiff.PrepareReplacementsUUID(t, &repls) testdiff.PrepareReplacementsDevVersion(t, &repls) testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) + repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) + testDirs := getTests(t) require.NotEmpty(t, testDirs) @@ -239,7 +225,6 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } repls.SetPathWithParents(tmpDir, "[TMPDIR]") - repls.Repls = append(repls.Repls, config.Repls...) scriptContents := readMergedScriptContents(t, dir) testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents) @@ -253,38 +238,79 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd := exec.Command(args[0], args[1:]...) cmd.Env = os.Environ() + var workspaceClient *databricks.WorkspaceClient + var user iam.User + // Start a new server with a custom configuration if the acceptance test // specifies a custom server stubs. var server *testserver.Server - // Start a new server for this test if either: - // 1. A custom server spec is defined in the test configuration. - // 2. The test is configured to record requests and assert on them. We need - // a duplicate of the default server to record requests because the default - // server otherwise is a shared resource. - if cloudEnv == "" && (len(config.Server) > 0 || config.RecordRequests) { - server = testserver.New(t) - server.RecordRequests = config.RecordRequests - server.IncludeRequestHeaders = config.IncludeRequestHeaders + if cloudEnv == "" { + // Start a new server for this test if either: + // 1. A custom server spec is defined in the test configuration. + // 2. The test is configured to record requests and assert on them. We need + // a duplicate of the default server to record requests because the default + // server otherwise is a shared resource. - // If no custom server stubs are defined, add the default handlers. - if len(config.Server) == 0 { + databricksLocalHost := os.Getenv("DATABRICKS_DEFAULT_HOST") + + if len(config.Server) > 0 || config.RecordRequests { + server = testserver.New(t) + server.RecordRequests = config.RecordRequests + server.IncludeRequestHeaders = config.IncludeRequestHeaders + + for _, stub := range config.Server { + require.NotEmpty(t, stub.Pattern) + items := strings.Split(stub.Pattern, " ") + require.Len(t, items, 2) + server.Handle(items[0], items[1], func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { + statusCode := http.StatusOK + if stub.Response.StatusCode != 0 { + statusCode = stub.Response.StatusCode + } + return stub.Response.Body, statusCode + }) + } + + // The earliest handlers take precedence, add default handlers last AddHandlers(server) + databricksLocalHost = server.URL } - for _, stub := range config.Server { - require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { - statusCode := http.StatusOK - if stub.Response.StatusCode != 0 { - statusCode = stub.Response.StatusCode - } - return stub.Response.Body, statusCode - }) + // Each local test should use a new token that will result into a new fake workspace, + // so that test don't interfere with each other. + tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "") + config := databricks.Config{ + Host: databricksLocalHost, + Token: "dbapi" + tokenSuffix, } - cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL) + workspaceClient, err = databricks.NewWorkspaceClient(&config) + require.NoError(t, err) + + cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+config.Host) + cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+config.Token) + + // For the purposes of replacements, use testUser. + // Note, users might have overriden /api/2.0/preview/scim/v2/Me but that should not affect the replacement: + user = testUser + } else { + // Use whatever authentication mechanism is configured by the test runner. + workspaceClient, err = databricks.NewWorkspaceClient(&databricks.Config{}) + require.NoError(t, err) + pUser, err := workspaceClient.CurrentUser.Me(context.Background()) + require.NoError(t, err, "Failed to get current user") + user = *pUser } + testdiff.PrepareReplacementsUser(t, &repls, user) + testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient) + + // Must be added PrepareReplacementsUser, otherwise conflicts with [USERNAME] + testdiff.PrepareReplacementsUUID(t, &repls) + + // User replacements come last: + repls.Repls = append(repls.Repls, config.Repls...) + if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one // results in sporadic failures like this one (only if tests are running in parallel): @@ -295,15 +321,6 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } - // Each local test should use a new token that will result into a new fake workspace, - // so that test don't interfere with each other. - if cloudEnv == "" { - tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "") - token := "dbapi" + tokenSuffix - cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token) - repls.Set(token, "[DATABRICKS_TOKEN]") - } - // Write combined output to a file out, err := os.Create(filepath.Join(tmpDir, "output.txt")) require.NoError(t, err) @@ -320,7 +337,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, req := range server.Requests { reqJson, err := json.MarshalIndent(req, "", " ") - require.NoError(t, err) + require.NoErrorf(t, err, "Failed to indent: %#v", req) reqJsonWithRepls := repls.Replace(string(reqJson)) _, err = f.WriteString(reqJsonWithRepls + "\n") diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt index 022b3148d..8d2584622 100644 --- a/acceptance/auth/bundle_and_profile/output.txt +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -13,13 +13,13 @@ === Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail >>> errcode [CLI] current-user me -p profile_name -Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET] Exit code: 1 === Inside the bundle, target and not matching profile >>> errcode [CLI] current-user me -t dev -p profile_name -Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET] Exit code: 1 diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml index b20190ca5..1a611ed95 100644 --- a/acceptance/auth/bundle_and_profile/test.toml +++ b/acceptance/auth/bundle_and_profile/test.toml @@ -5,4 +5,8 @@ Badness = "When -p flag is used inside the bundle folder for any CLI commands, C # This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST [[Repls]] Old='DATABRICKS_HOST' -New='DATABRICKS_URL' +New='DATABRICKS_TARGET' + +[[Repls]] +Old='DATABRICKS_URL' +New='DATABRICKS_TARGET' diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index c8a52f4cd..d3db06003 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -14,11 +14,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - - // {$} is a wildcard that only matches the end of the URL. We explicitly use - // /{$} to disambiguate it from the generic handler for '/' which is used to - // identify unhandled API endpoints in the test server. - server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/", func(_ *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/selftest/out.hello.txt b/acceptance/selftest/basic/out.hello.txt similarity index 100% rename from acceptance/selftest/out.hello.txt rename to acceptance/selftest/basic/out.hello.txt diff --git a/acceptance/selftest/output.txt b/acceptance/selftest/basic/output.txt similarity index 100% rename from acceptance/selftest/output.txt rename to acceptance/selftest/basic/output.txt diff --git a/acceptance/selftest/script b/acceptance/selftest/basic/script similarity index 100% rename from acceptance/selftest/script rename to acceptance/selftest/basic/script diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/basic/test.toml similarity index 100% rename from acceptance/selftest/test.toml rename to acceptance/selftest/basic/test.toml diff --git a/acceptance/selftest/server/out.requests.txt b/acceptance/selftest/server/out.requests.txt new file mode 100644 index 000000000..2cb8708ac --- /dev/null +++ b/acceptance/selftest/server/out.requests.txt @@ -0,0 +1,8 @@ +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/custom/endpoint" +} diff --git a/acceptance/selftest/server/output.txt b/acceptance/selftest/server/output.txt new file mode 100644 index 000000000..f9e51caa9 --- /dev/null +++ b/acceptance/selftest/server/output.txt @@ -0,0 +1,15 @@ + +>>> curl -s [DATABRICKS_URL]/api/2.0/preview/scim/v2/Me +{ + "id": "[USERID]", + "userName": "[USERNAME]" +} +>>> curl -sD - [DATABRICKS_URL]/custom/endpoint?query=param +HTTP/1.1 201 Created +Content-Type: application/json +Date: (redacted) +Content-Length: (redacted) + +custom +--- +response diff --git a/acceptance/selftest/server/script b/acceptance/selftest/server/script new file mode 100644 index 000000000..53e2c4b8a --- /dev/null +++ b/acceptance/selftest/server/script @@ -0,0 +1,2 @@ +trace curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me +trace curl -sD - $DATABRICKS_HOST/custom/endpoint?query=param diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml new file mode 100644 index 000000000..2531fb910 --- /dev/null +++ b/acceptance/selftest/server/test.toml @@ -0,0 +1,18 @@ +LocalOnly = true +RecordRequests = true + +[[Server]] +Pattern = "GET /custom/endpoint" +Response.Body = '''custom +--- +response +''' +Response.StatusCode = 201 + +[[Repls]] +Old = 'Date: .*' +New = 'Date: (redacted)' + +[[Repls]] +Old = 'Content-Length: [0-9]*' +New = 'Content-Length: (redacted)' diff --git a/acceptance/server_test.go b/acceptance/server_test.go index d21ab66e8..11d03c30b 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/gorilla/mux" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -16,8 +17,13 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +var testUser = iam.User{ + Id: "1000012345", + UserName: "tester@databricks.com", +} + func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -32,7 +38,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -43,7 +49,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -58,20 +64,17 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - return iam.User{ - Id: "1000012345", - UserName: "tester@databricks.com", - }, http.StatusOK + server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return testUser, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { path := r.URL.Query().Get("path") return fakeWorkspace.WorkspaceGetStatus(path) }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("POST", "/api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { request := workspace.Mkdirs{} decoder := json.NewDecoder(r.Body) @@ -83,13 +86,13 @@ func AddHandlers(server *testserver.Server) { return fakeWorkspace.WorkspaceMkdirs(request) }) - server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { path := r.URL.Query().Get("path") return fakeWorkspace.WorkspaceExport(path) }) - server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("POST", "/api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { path := r.URL.Query().Get("path") recursiveStr := r.URL.Query().Get("recursive") var recursive bool @@ -103,8 +106,9 @@ func AddHandlers(server *testserver.Server) { return fakeWorkspace.WorkspaceDelete(path, recursive) }) - server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - path := r.PathValue("path") + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + vars := mux.Vars(r) + path := vars["path"] body := new(bytes.Buffer) _, err := body.ReadFrom(r.Body) @@ -115,14 +119,15 @@ func AddHandlers(server *testserver.Server) { return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes()) }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - objectId := r.PathValue("objectId") + server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + vars := mux.Vars(r) + objectId := vars["objectId"] return workspace.WorkspaceObjectPermissions{ ObjectId: objectId, @@ -140,7 +145,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("POST", "/api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { request := jobs.CreateJob{} decoder := json.NewDecoder(r.Body) @@ -152,13 +157,13 @@ func AddHandlers(server *testserver.Server) { return fakeWorkspace.JobsCreate(request) }) - server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { jobId := r.URL.Query().Get("job_id") return fakeWorkspace.JobsGet(jobId) }) - server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return fakeWorkspace.JobsList() }) } diff --git a/go.mod b/go.mod index c8b209edd..2e2505361 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/databricks/databricks-sdk-go v0.57.0 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause + github.com/gorilla/mux v1.8.1 // BSD 3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 github.com/hashicorp/hc-install v0.9.1 // MPL 2.0 github.com/hashicorp/terraform-exec v0.22.0 // MPL 2.0 diff --git a/go.sum b/go.sum index 0369fc2d9..fbf942148 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 577ef082c..cf4d5aca2 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -9,6 +9,8 @@ import ( "strings" "sync" + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" "github.com/databricks/cli/internal/testutil" @@ -17,7 +19,7 @@ import ( type Server struct { *httptest.Server - Mux *http.ServeMux + Router *mux.Router t testutil.TestingT @@ -39,22 +41,20 @@ type Request struct { } func New(t testutil.TestingT) *Server { - mux := http.NewServeMux() - server := httptest.NewServer(mux) + router := mux.NewRouter() + server := httptest.NewServer(router) t.Cleanup(server.Close) s := &Server{ Server: server, - Mux: mux, + Router: router, t: t, mu: &sync.Mutex{}, fakeWorkspaces: map[string]*FakeWorkspace{}, } - // The server resolves conflicting handlers by using the one with higher - // specificity. This handler is the least specific, so it will be used as a - // fallback when no other handlers match. - s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) { + // Set up the not found handler as fallback + router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { pattern := r.Method + " " + r.URL.Path t.Errorf(` @@ -75,9 +75,22 @@ Response.StatusCode = `, pattern, pattern) - return apierr.APIError{ + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotImplemented) + + resp := apierr.APIError{ Message: "No stub found for pattern: " + pattern, - }, http.StatusNotImplemented + } + + respBytes, err := json.Marshal(resp) + if err != nil { + t.Errorf("JSON encoding error: %s", err) + respBytes = []byte("{\"message\": \"JSON encoding error\"}") + } + + if _, err := w.Write(respBytes); err != nil { + t.Errorf("Response write error: %s", err) + } }) return s @@ -85,8 +98,8 @@ Response.StatusCode = type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) -func (s *Server) Handle(pattern string, handler HandlerFunc) { - s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { +func (s *Server) Handle(method, path string, handler HandlerFunc) { + s.Router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { // For simplicity we process requests sequentially. It's fast enough because // we don't do any IO except reading and writing request/response bodies. s.mu.Lock() @@ -156,7 +169,7 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - }) + }).Methods(method) } func getToken(r *http.Request) string { From 24ac8d8d595df531a2aa7b9faa72f8be7132f7b1 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 11 Feb 2025 21:20:03 +0530 Subject: [PATCH 092/105] Add acceptance tests for auth resolution (#2285) ## Changes This PR adds acceptance tests for native Databricks auth methods: basic, oauth, and pat. In the future we could compare this with auth credentials used by downstream tools like TF or the telemetry process to ensure consistent auth credentials are picked up and used. Note: We do not add acceptance tests for other auth methods like Azure because they communicate with external endpoints. To test them locally, we would need to set up a reverse proxy server, which is out of scope for this change. ## Tests N/A --- .../auth/credentials/basic/out.requests.txt | 12 +++++++ acceptance/auth/credentials/basic/output.txt | 4 +++ acceptance/auth/credentials/basic/script | 8 +++++ acceptance/auth/credentials/basic/test.toml | 4 +++ .../auth/credentials/oauth/out.requests.txt | 34 +++++++++++++++++++ acceptance/auth/credentials/oauth/output.txt | 4 +++ acceptance/auth/credentials/oauth/script | 8 +++++ acceptance/auth/credentials/oauth/test.toml | 5 +++ .../auth/credentials/pat/out.requests.txt | 12 +++++++ acceptance/auth/credentials/pat/output.txt | 4 +++ acceptance/auth/credentials/pat/script | 3 ++ acceptance/auth/credentials/test.toml | 20 +++++++++++ acceptance/server_test.go | 16 +++++++++ 13 files changed, 134 insertions(+) create mode 100644 acceptance/auth/credentials/basic/out.requests.txt create mode 100644 acceptance/auth/credentials/basic/output.txt create mode 100644 acceptance/auth/credentials/basic/script create mode 100644 acceptance/auth/credentials/basic/test.toml create mode 100644 acceptance/auth/credentials/oauth/out.requests.txt create mode 100644 acceptance/auth/credentials/oauth/output.txt create mode 100644 acceptance/auth/credentials/oauth/script create mode 100644 acceptance/auth/credentials/oauth/test.toml create mode 100644 acceptance/auth/credentials/pat/out.requests.txt create mode 100644 acceptance/auth/credentials/pat/output.txt create mode 100644 acceptance/auth/credentials/pat/script create mode 100644 acceptance/auth/credentials/test.toml diff --git a/acceptance/auth/credentials/basic/out.requests.txt b/acceptance/auth/credentials/basic/out.requests.txt new file mode 100644 index 000000000..b549c7423 --- /dev/null +++ b/acceptance/auth/credentials/basic/out.requests.txt @@ -0,0 +1,12 @@ +{ + "headers": { + "Authorization": [ + "Basic [ENCODED_AUTH]" + ], + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/basic" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/auth/credentials/basic/output.txt b/acceptance/auth/credentials/basic/output.txt new file mode 100644 index 000000000..c5747c9e4 --- /dev/null +++ b/acceptance/auth/credentials/basic/output.txt @@ -0,0 +1,4 @@ +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/auth/credentials/basic/script b/acceptance/auth/credentials/basic/script new file mode 100644 index 000000000..aae249083 --- /dev/null +++ b/acceptance/auth/credentials/basic/script @@ -0,0 +1,8 @@ +# Unset the token which is configured by default +# in acceptance tests +export DATABRICKS_TOKEN="" + +export DATABRICKS_USERNAME=username +export DATABRICKS_PASSWORD=password + +$CLI current-user me diff --git a/acceptance/auth/credentials/basic/test.toml b/acceptance/auth/credentials/basic/test.toml new file mode 100644 index 000000000..4998d81d7 --- /dev/null +++ b/acceptance/auth/credentials/basic/test.toml @@ -0,0 +1,4 @@ +# "username:password" in base64 is dXNlcm5hbWU6cGFzc3dvcmQ=, expect to see this in Authorization header +[[Repls]] +Old = "dXNlcm5hbWU6cGFzc3dvcmQ=" +New = "[ENCODED_AUTH]" diff --git a/acceptance/auth/credentials/oauth/out.requests.txt b/acceptance/auth/credentials/oauth/out.requests.txt new file mode 100644 index 000000000..525e148d8 --- /dev/null +++ b/acceptance/auth/credentials/oauth/out.requests.txt @@ -0,0 +1,34 @@ +{ + "headers": { + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]" + ] + }, + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic [ENCODED_AUTH]" + ], + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS]" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ], + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/oauth-m2m" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/auth/credentials/oauth/output.txt b/acceptance/auth/credentials/oauth/output.txt new file mode 100644 index 000000000..c5747c9e4 --- /dev/null +++ b/acceptance/auth/credentials/oauth/output.txt @@ -0,0 +1,4 @@ +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/auth/credentials/oauth/script b/acceptance/auth/credentials/oauth/script new file mode 100644 index 000000000..e4519e41b --- /dev/null +++ b/acceptance/auth/credentials/oauth/script @@ -0,0 +1,8 @@ +# Unset the token which is configured by default +# in acceptance tests +export DATABRICKS_TOKEN="" + +export DATABRICKS_CLIENT_ID=client_id +export DATABRICKS_CLIENT_SECRET=client_secret + +$CLI current-user me diff --git a/acceptance/auth/credentials/oauth/test.toml b/acceptance/auth/credentials/oauth/test.toml new file mode 100644 index 000000000..2adade96a --- /dev/null +++ b/acceptance/auth/credentials/oauth/test.toml @@ -0,0 +1,5 @@ +# "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to +# see this in Authorization header +[[Repls]] +Old = "Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=" +New = "[ENCODED_AUTH]" diff --git a/acceptance/auth/credentials/pat/out.requests.txt b/acceptance/auth/credentials/pat/out.requests.txt new file mode 100644 index 000000000..73c448c2f --- /dev/null +++ b/acceptance/auth/credentials/pat/out.requests.txt @@ -0,0 +1,12 @@ +{ + "headers": { + "Authorization": [ + "Bearer dapi1234" + ], + "User-Agent": [ + "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/current-user_me cmd-exec-id/[UUID] auth/pat" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/auth/credentials/pat/output.txt b/acceptance/auth/credentials/pat/output.txt new file mode 100644 index 000000000..c5747c9e4 --- /dev/null +++ b/acceptance/auth/credentials/pat/output.txt @@ -0,0 +1,4 @@ +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/auth/credentials/pat/script b/acceptance/auth/credentials/pat/script new file mode 100644 index 000000000..ccf1098e7 --- /dev/null +++ b/acceptance/auth/credentials/pat/script @@ -0,0 +1,3 @@ +export DATABRICKS_TOKEN=dapi1234 + +$CLI current-user me diff --git a/acceptance/auth/credentials/test.toml b/acceptance/auth/credentials/test.toml new file mode 100644 index 000000000..89438f43a --- /dev/null +++ b/acceptance/auth/credentials/test.toml @@ -0,0 +1,20 @@ +LocalOnly = true + +RecordRequests = true +IncludeRequestHeaders = ["Authorization", "User-Agent"] + +[[Repls]] +Old = '(linux|darwin|windows)' +New = '[OS]' + +[[Repls]] +Old = " upstream/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " upstream-version/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " cicd/[A-Za-z0-9.-]+" +New = "" diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 11d03c30b..fd8006b8f 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -166,6 +166,22 @@ func AddHandlers(server *testserver.Server) { server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return fakeWorkspace.JobsList() }) + + server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return map[string]string{ + "authorization_endpoint": server.URL + "oidc/v1/authorize", + "token_endpoint": server.URL + "/oidc/v1/token", + }, http.StatusOK + }) + + server.Handle("POST", "/oidc/v1/token", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return map[string]string{ + "access_token": "oauth-token", + "expires_in": "3600", + "scope": "all-apis", + "token_type": "Bearer", + }, http.StatusOK + }) } func internalError(err error) (any, int) { From bfde3585b9df9eb5f38a4bed63d2f85426ad1119 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Feb 2025 12:45:01 +0100 Subject: [PATCH 093/105] acc: Fix priority of stubs in test.toml (#2339) ## Changes Reverse the order of stubs to match expectation (leaf configuration takes precedence over parent configuration). Follow up to #2334 . ## Tests acceptance/selftest/server is extended with duplicate handler --- acceptance/acceptance_test.go | 4 ++++ acceptance/selftest/server/test.toml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 117172f60..e61166c31 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -259,6 +259,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont server.RecordRequests = config.RecordRequests server.IncludeRequestHeaders = config.IncludeRequestHeaders + // We want later stubs takes precedence, because then leaf configs take precedence over parent directory configs + // In gorilla/mux earlier handlers take precedence, so we need to reverse the order + slices.Reverse(config.Server) + for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) items := strings.Split(stub.Pattern, " ") diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml index 2531fb910..fca41bf02 100644 --- a/acceptance/selftest/server/test.toml +++ b/acceptance/selftest/server/test.toml @@ -1,6 +1,10 @@ LocalOnly = true RecordRequests = true +[[Server]] +Pattern = "GET /custom/endpoint" +Response.Body = '''should not see this response, latter response takes precedence''' + [[Server]] Pattern = "GET /custom/endpoint" Response.Body = '''custom From 4034766c939620c062a0dbad2207eaad6821c54b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Feb 2025 14:00:57 +0100 Subject: [PATCH 094/105] acc: Simplify writing handlers; support headers in responses (#2338) ## Changes Handlers now receive testserver.Request and return any which could be - string or []byte (returns it as is but sets content-type to json or plain text depending on content) - struct (encodes it as json and sets content-type to json) - testserver.Response (full control over status and headers) Note if testserver.Response is returned from the handler, it's Body attribute can still be an object. In that case, it'll be serialized and appropriate content-type header will be added. The config is now using the same testserver.Response struct, the same logic applies both configured responses and responses returned from handlers. As a result, one can set headers both in Golang handlers and in test.toml. This also fixes a bug with RecordRequest not seeing the body if it was already consumed by the handler. ## Tests - Existing rests. - acceptance/selftest/server is extended to set response header. --- acceptance/acceptance_test.go | 9 +- .../bundle/debug/out.stderr.parallel.txt | 6 +- acceptance/bundle/debug/out.stderr.txt | 3 +- acceptance/cmd_server_test.go | 8 +- acceptance/config_test.go | 6 +- acceptance/selftest/server/out.requests.txt | 4 + acceptance/selftest/server/output.txt | 8 +- acceptance/selftest/server/script | 2 + acceptance/selftest/server/test.toml | 2 + acceptance/server_test.go | 132 ++++----- libs/testserver/fake_workspace.go | 98 +++---- libs/testserver/server.go | 251 ++++++++++++++---- 12 files changed, 336 insertions(+), 193 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e61166c31..85c345032 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "net/http" "os" "os/exec" "path/filepath" @@ -267,12 +266,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont require.NotEmpty(t, stub.Pattern) items := strings.Split(stub.Pattern, " ") require.Len(t, items, 2) - server.Handle(items[0], items[1], func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { - statusCode := http.StatusOK - if stub.Response.StatusCode != 0 { - statusCode = stub.Response.StatusCode - } - return stub.Response.Body, statusCode + server.Handle(items[0], items[1], func(req testserver.Request) any { + return stub.Response }) } diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt index 7dd770068..13c81c511 100644 --- a/acceptance/bundle/debug/out.stderr.parallel.txt +++ b/acceptance/bundle/debug/out.stderr.parallel.txt @@ -9,7 +9,7 @@ 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns 10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync -10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true -< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true -< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< HTTP/0.0 000 OK pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt index 9cac8bb2b..e5867e008 100644 --- a/acceptance/bundle/debug/out.stderr.txt +++ b/acceptance/bundle/debug/out.stderr.txt @@ -79,11 +79,12 @@ 10:07:59 Debug: Apply pid=12345 mutator=validate 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files < HTTP/1.1 404 Not Found +< { +< "message": "Workspace path not found" 10:07:59 Debug: POST /api/2.0/workspace/mkdirs > { > "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" > } -< HTTP/1.1 200 OK 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files < HTTP/1.1 200 OK < { diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index d3db06003..dc48a85d7 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -1,8 +1,8 @@ package acceptance_test import ( + "context" "encoding/json" - "net/http" "os" "strings" "testing" @@ -14,7 +14,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("GET", "/", func(_ *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/", func(r testserver.Request) any { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") @@ -27,7 +27,7 @@ func StartCmdServer(t *testing.T) *testserver.Server { defer Chdir(t, q.Get("cwd"))() - c := testcli.NewRunner(t, r.Context(), args...) + c := testcli.NewRunner(t, context.Background(), args...) c.Verbose = false stdout, stderr, err := c.Run() result := map[string]any{ @@ -39,7 +39,7 @@ func StartCmdServer(t *testing.T) *testserver.Server { exitcode = 1 } result["exitcode"] = exitcode - return result, http.StatusOK + return result }) return server } diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 920e713a1..ec0d1baee 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -10,6 +10,7 @@ import ( "dario.cat/mergo" "github.com/BurntSushi/toml" "github.com/databricks/cli/libs/testdiff" + "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/require" ) @@ -56,10 +57,7 @@ type ServerStub struct { Pattern string // The response body to return. - Response struct { - Body string - StatusCode int - } + Response testserver.Response } // FindConfigs finds all the config relevant for this test, diff --git a/acceptance/selftest/server/out.requests.txt b/acceptance/selftest/server/out.requests.txt index 2cb8708ac..34f4c4899 100644 --- a/acceptance/selftest/server/out.requests.txt +++ b/acceptance/selftest/server/out.requests.txt @@ -6,3 +6,7 @@ "method": "GET", "path": "/custom/endpoint" } +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} diff --git a/acceptance/selftest/server/output.txt b/acceptance/selftest/server/output.txt index f9e51caa9..7147f9c9b 100644 --- a/acceptance/selftest/server/output.txt +++ b/acceptance/selftest/server/output.txt @@ -6,10 +6,16 @@ } >>> curl -sD - [DATABRICKS_URL]/custom/endpoint?query=param HTTP/1.1 201 Created -Content-Type: application/json +X-Custom-Header: hello Date: (redacted) Content-Length: (redacted) +Content-Type: text/plain; charset=utf-8 custom --- response + +>>> errcode [CLI] workspace get-status /a/b/c +Error: Workspace path not found + +Exit code: 1 diff --git a/acceptance/selftest/server/script b/acceptance/selftest/server/script index 53e2c4b8a..810ea64b6 100644 --- a/acceptance/selftest/server/script +++ b/acceptance/selftest/server/script @@ -1,2 +1,4 @@ trace curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me trace curl -sD - $DATABRICKS_HOST/custom/endpoint?query=param + +trace errcode $CLI workspace get-status /a/b/c diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml index fca41bf02..43ad1e85b 100644 --- a/acceptance/selftest/server/test.toml +++ b/acceptance/selftest/server/test.toml @@ -12,6 +12,8 @@ Response.Body = '''custom response ''' Response.StatusCode = 201 +[Server.Response.Headers] +"X-Custom-Header" = ["hello"] [[Repls]] Old = 'Date: .*' diff --git a/acceptance/server_test.go b/acceptance/server_test.go index fd8006b8f..f73872e0b 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -1,14 +1,12 @@ package acceptance_test import ( - "bytes" "encoding/json" "fmt" "net/http" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/gorilla/mux" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -23,7 +21,7 @@ var testUser = iam.User{ } func AddHandlers(server *testserver.Server) { - server.Handle("GET", "/api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/policies/clusters/list", func(req testserver.Request) any { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -35,10 +33,10 @@ func AddHandlers(server *testserver.Server) { Name: "some-test-cluster-policy", }, }, - }, http.StatusOK + } }) - server.Handle("GET", "/api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.0/instance-pools/list", func(req testserver.Request) any { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -46,10 +44,10 @@ func AddHandlers(server *testserver.Server) { InstancePoolId: "1234", }, }, - }, http.StatusOK + } }) - server.Handle("GET", "/api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/clusters/list", func(req testserver.Request) any { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -61,74 +59,57 @@ func AddHandlers(server *testserver.Server) { ClusterId: "9876", }, }, - }, http.StatusOK + } }) - server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - return testUser, http.StatusOK + server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any { + return testUser }) - server.Handle("GET", "/api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - path := r.URL.Query().Get("path") - - return fakeWorkspace.WorkspaceGetStatus(path) + server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any { + path := req.URL.Query().Get("path") + return req.Workspace.WorkspaceGetStatus(path) }) - server.Handle("POST", "/api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - request := workspace.Mkdirs{} - decoder := json.NewDecoder(r.Body) - - err := decoder.Decode(&request) - if err != nil { - return internalError(err) + server.Handle("POST", "/api/2.0/workspace/mkdirs", func(req testserver.Request) any { + var request workspace.Mkdirs + if err := json.Unmarshal(req.Body, &request); err != nil { + return testserver.Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } } - return fakeWorkspace.WorkspaceMkdirs(request) + req.Workspace.WorkspaceMkdirs(request) + return "" }) - server.Handle("GET", "/api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - path := r.URL.Query().Get("path") - - return fakeWorkspace.WorkspaceExport(path) + server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any { + path := req.URL.Query().Get("path") + return req.Workspace.WorkspaceExport(path) }) - server.Handle("POST", "/api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - path := r.URL.Query().Get("path") - recursiveStr := r.URL.Query().Get("recursive") - var recursive bool - - if recursiveStr == "true" { - recursive = true - } else { - recursive = false - } - - return fakeWorkspace.WorkspaceDelete(path, recursive) + server.Handle("POST", "/api/2.0/workspace/delete", func(req testserver.Request) any { + path := req.URL.Query().Get("path") + recursive := req.URL.Query().Get("recursive") == "true" + req.Workspace.WorkspaceDelete(path, recursive) + return "" }) - server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - vars := mux.Vars(r) - path := vars["path"] - - body := new(bytes.Buffer) - _, err := body.ReadFrom(r.Body) - if err != nil { - return internalError(err) - } - - return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes()) + server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(req testserver.Request) any { + path := req.Vars["path"] + req.Workspace.WorkspaceFilesImportFile(path, req.Body) + return "" }) - server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(req testserver.Request) any { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", - }, http.StatusOK + } }) - server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - vars := mux.Vars(r) - objectId := vars["objectId"] - + server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(req testserver.Request) any { + objectId := req.Vars["objectId"] return workspace.WorkspaceObjectPermissions{ ObjectId: objectId, ObjectType: "DIRECTORY", @@ -142,48 +123,43 @@ func AddHandlers(server *testserver.Server) { }, }, }, - }, http.StatusOK + } }) - server.Handle("POST", "/api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - request := jobs.CreateJob{} - decoder := json.NewDecoder(r.Body) - - err := decoder.Decode(&request) - if err != nil { - return internalError(err) + server.Handle("POST", "/api/2.1/jobs/create", func(req testserver.Request) any { + var request jobs.CreateJob + if err := json.Unmarshal(req.Body, &request); err != nil { + return testserver.Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: 500, + } } - return fakeWorkspace.JobsCreate(request) + return req.Workspace.JobsCreate(request) }) - server.Handle("GET", "/api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - jobId := r.URL.Query().Get("job_id") - - return fakeWorkspace.JobsGet(jobId) + server.Handle("GET", "/api/2.1/jobs/get", func(req testserver.Request) any { + jobId := req.URL.Query().Get("job_id") + return req.Workspace.JobsGet(jobId) }) - server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { - return fakeWorkspace.JobsList() + server.Handle("GET", "/api/2.1/jobs/list", func(req testserver.Request) any { + return req.Workspace.JobsList() }) - server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any { return map[string]string{ "authorization_endpoint": server.URL + "oidc/v1/authorize", "token_endpoint": server.URL + "/oidc/v1/token", - }, http.StatusOK + } }) - server.Handle("POST", "/oidc/v1/token", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + server.Handle("POST", "/oidc/v1/token", func(_ testserver.Request) any { return map[string]string{ "access_token": "oauth-token", "expires_in": "3600", "scope": "all-apis", "token_type": "Bearer", - }, http.StatusOK + } }) } - -func internalError(err error) (any, int) { - return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c3e4f9a71..4e943f828 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "net/http" "sort" "strconv" "strings" @@ -33,40 +32,39 @@ func NewFakeWorkspace() *FakeWorkspace { } } -func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) { +func (s *FakeWorkspace) WorkspaceGetStatus(path string) Response { if s.directories[path] { - return workspace.ObjectInfo{ - ObjectType: "DIRECTORY", - Path: path, - }, http.StatusOK + return Response{ + Body: &workspace.ObjectInfo{ + ObjectType: "DIRECTORY", + Path: path, + }, + } } else if _, ok := s.files[path]; ok { - return workspace.ObjectInfo{ - ObjectType: "FILE", - Path: path, - Language: "SCALA", - }, http.StatusOK + return Response{ + Body: &workspace.ObjectInfo{ + ObjectType: "FILE", + Path: path, + Language: "SCALA", + }, + } } else { - return workspace.ObjectInfo{}, http.StatusNotFound + return Response{ + StatusCode: 404, + Body: map[string]string{"message": "Workspace path not found"}, + } } } -func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) { +func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { s.directories[request.Path] = true - - return "{}", http.StatusOK } -func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) { - file := s.files[path] - - if file == nil { - return nil, http.StatusNotFound - } - - return file, http.StatusOK +func (s *FakeWorkspace) WorkspaceExport(path string) []byte { + return s.files[path] } -func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) { +func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) { if !recursive { s.files[path] = nil } else { @@ -76,28 +74,26 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, in } } } - - return "{}", http.StatusOK } -func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) { +func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) { if !strings.HasPrefix(path, "/") { path = "/" + path } - s.files[path] = body - - return "{}", http.StatusOK } -func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) { +func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) Response { jobId := s.nextJobId s.nextJobId++ jobSettings := jobs.JobSettings{} err := jsonConvert(request, &jobSettings) if err != nil { - return internalError(err) + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("Cannot convert request to jobSettings: %s", err), + } } s.jobs[jobId] = jobs.Job{ @@ -105,32 +101,44 @@ func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) { Settings: &jobSettings, } - return jobs.CreateResponse{JobId: jobId}, http.StatusOK + return Response{ + Body: jobs.CreateResponse{JobId: jobId}, + } } -func (s *FakeWorkspace) JobsGet(jobId string) (any, int) { +func (s *FakeWorkspace) JobsGet(jobId string) Response { id := jobId jobIdInt, err := strconv.ParseInt(id, 10, 64) if err != nil { - return internalError(fmt.Errorf("failed to parse job id: %s", err)) + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("Failed to parse job id: %s: %v", err, id), + } } job, ok := s.jobs[jobIdInt] if !ok { - return jobs.Job{}, http.StatusNotFound + return Response{ + StatusCode: 404, + } } - return job, http.StatusOK + return Response{ + Body: job, + } } -func (s *FakeWorkspace) JobsList() (any, int) { +func (s *FakeWorkspace) JobsList() Response { list := make([]jobs.BaseJob, 0, len(s.jobs)) for _, job := range s.jobs { baseJob := jobs.BaseJob{} err := jsonConvert(job, &baseJob) if err != nil { - return internalError(fmt.Errorf("failed to convert job to base job: %w", err)) + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("failed to convert job to base job: %s", err), + } } list = append(list, baseJob) @@ -141,9 +149,11 @@ func (s *FakeWorkspace) JobsList() (any, int) { return list[i].JobId < list[j].JobId }) - return jobs.ListJobsResponse{ - Jobs: list, - }, http.StatusOK + return Response{ + Body: jobs.ListJobsResponse{ + Jobs: list, + }, + } } // jsonConvert saves input to a value pointed by output @@ -163,7 +173,3 @@ func jsonConvert(input, output any) error { return nil } - -func internalError(err error) (string, int) { - return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError -} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index cf4d5aca2..fa15973d7 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -5,14 +5,14 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" + "reflect" "slices" "strings" "sync" "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - "github.com/databricks/cli/internal/testutil" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -29,10 +29,10 @@ type Server struct { RecordRequests bool IncludeRequestHeaders []string - Requests []Request + Requests []LoggedRequest } -type Request struct { +type LoggedRequest struct { Headers http.Header `json:"headers,omitempty"` Method string `json:"method"` Path string `json:"path"` @@ -40,6 +40,153 @@ type Request struct { RawBody string `json:"raw_body,omitempty"` } +type Request struct { + Method string + URL *url.URL + Headers http.Header + Body []byte + Vars map[string]string + Workspace *FakeWorkspace +} + +type Response struct { + StatusCode int + Headers http.Header + Body any +} + +type encodedResponse struct { + StatusCode int + Headers http.Header + Body []byte +} + +func NewRequest(t testutil.TestingT, r *http.Request, fakeWorkspace *FakeWorkspace) Request { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("Failed to read request body: %s", err) + } + + return Request{ + Method: r.Method, + URL: r.URL, + Headers: r.Header, + Body: body, + Vars: mux.Vars(r), + Workspace: fakeWorkspace, + } +} + +func normalizeResponse(t testutil.TestingT, resp any) encodedResponse { + result := normalizeResponseBody(t, resp) + if result.StatusCode == 0 { + result.StatusCode = 200 + } + return result +} + +func normalizeResponseBody(t testutil.TestingT, resp any) encodedResponse { + if isNil(resp) { + t.Errorf("Handler must not return nil") + return encodedResponse{StatusCode: 500} + } + + respBytes, ok := resp.([]byte) + if ok { + return encodedResponse{ + Body: respBytes, + Headers: getHeaders(respBytes), + } + } + + respString, ok := resp.(string) + if ok { + return encodedResponse{ + Body: []byte(respString), + Headers: getHeaders([]byte(respString)), + } + } + + respStruct, ok := resp.(Response) + if ok { + if isNil(respStruct.Body) { + return encodedResponse{ + StatusCode: respStruct.StatusCode, + Headers: respStruct.Headers, + Body: []byte{}, + } + } + + bytesVal, isBytes := respStruct.Body.([]byte) + if isBytes { + return encodedResponse{ + StatusCode: respStruct.StatusCode, + Headers: respStruct.Headers, + Body: bytesVal, + } + } + + stringVal, isString := respStruct.Body.(string) + if isString { + return encodedResponse{ + StatusCode: respStruct.StatusCode, + Headers: respStruct.Headers, + Body: []byte(stringVal), + } + } + + respBytes, err := json.MarshalIndent(respStruct.Body, "", " ") + if err != nil { + t.Errorf("JSON encoding error: %s", err) + return encodedResponse{ + StatusCode: 500, + Body: []byte("internal error"), + } + } + + headers := respStruct.Headers + if headers == nil { + headers = getJsonHeaders() + } + + return encodedResponse{ + StatusCode: respStruct.StatusCode, + Headers: headers, + Body: respBytes, + } + } + + respBytes, err := json.MarshalIndent(resp, "", " ") + if err != nil { + t.Errorf("JSON encoding error: %s", err) + return encodedResponse{ + StatusCode: 500, + Body: []byte("internal error"), + } + } + + return encodedResponse{ + Body: respBytes, + Headers: getJsonHeaders(), + } +} + +func getJsonHeaders() http.Header { + return map[string][]string{ + "Content-Type": {"application/json"}, + } +} + +func getHeaders(value []byte) http.Header { + if json.Valid(value) { + return getJsonHeaders() + } else { + return map[string][]string{ + "Content-Type": {"text/plain"}, + } + } +} + func New(t testutil.TestingT) *Server { router := mux.NewRouter() server := httptest.NewServer(router) @@ -96,7 +243,7 @@ Response.StatusCode = return s } -type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) +type HandlerFunc func(req Request) any func (s *Server) Handle(method, path string, handler HandlerFunc) { s.Router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { @@ -117,56 +264,22 @@ func (s *Server) Handle(method, path string, handler HandlerFunc) { fakeWorkspace = s.fakeWorkspaces[token] } - resp, statusCode := handler(fakeWorkspace, r) - + request := NewRequest(s.t, r, fakeWorkspace) if s.RecordRequests { - body, err := io.ReadAll(r.Body) - assert.NoError(s.t, err) - - headers := make(http.Header) - for k, v := range r.Header { - if !slices.Contains(s.IncludeRequestHeaders, k) { - continue - } - for _, vv := range v { - headers.Add(k, vv) - } - } - - req := Request{ - Headers: headers, - Method: r.Method, - Path: r.URL.Path, - } - - if json.Valid(body) { - req.Body = json.RawMessage(body) - } else { - req.RawBody = string(body) - } - - s.Requests = append(s.Requests, req) + s.Requests = append(s.Requests, getLoggedRequest(request, s.IncludeRequestHeaders)) } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(statusCode) + respAny := handler(request) + resp := normalizeResponse(s.t, respAny) - var respBytes []byte - var err error - if respString, ok := resp.(string); ok { - respBytes = []byte(respString) - } else if respBytes0, ok := resp.([]byte); ok { - respBytes = respBytes0 - } else { - respBytes, err = json.MarshalIndent(resp, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + for k, v := range resp.Headers { + w.Header()[k] = v } - if _, err := w.Write(respBytes); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + w.WriteHeader(resp.StatusCode) + + if _, err := w.Write(resp.Body); err != nil { + s.t.Errorf("Failed to write response: %s", err) return } }).Methods(method) @@ -182,3 +295,43 @@ func getToken(r *http.Request) string { return header[len(prefix):] } + +func getLoggedRequest(req Request, includedHeaders []string) LoggedRequest { + result := LoggedRequest{ + Method: req.Method, + Path: req.URL.Path, + Headers: filterHeaders(req.Headers, includedHeaders), + } + + if json.Valid(req.Body) { + result.Body = json.RawMessage(req.Body) + } else { + result.RawBody = string(req.Body) + } + + return result +} + +func filterHeaders(h http.Header, includedHeaders []string) http.Header { + headers := make(http.Header) + for k, v := range h { + if !slices.Contains(includedHeaders, k) { + continue + } + headers[k] = v + } + return headers +} + +func isNil(i any) bool { + if i == nil { + return true + } + v := reflect.ValueOf(i) + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Interface, reflect.Slice: + return v.IsNil() + default: + return false + } +} From 1dadc794f57b98a833949557f2a61cb44e33146f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 12 Feb 2025 14:29:23 +0000 Subject: [PATCH 095/105] [Release] Release v0.241.0 (#2340) Bundles: * Added support to generate Git based jobs ([#2304](https://github.com/databricks/cli/pull/2304)). * Added support for run_as in pipelines ([#2287](https://github.com/databricks/cli/pull/2287)). * Raise an error when there are multiple local libraries with the same basename used ([#2297](https://github.com/databricks/cli/pull/2297)). * Fix env variable for AzureCli local config ([#2248](https://github.com/databricks/cli/pull/2248)). * Accept JSON files in includes section ([#2265](https://github.com/databricks/cli/pull/2265)). * Always print warnings and errors; clean up format ([#2213](https://github.com/databricks/cli/pull/2213)) API Changes: * Added `databricks account budget-policy` command group. * Added `databricks lakeview-embedded` command group. * Added `databricks query-execution` command group. * Added `databricks account enable-ip-access-lists` command group. * Added `databricks redash-config` command group. OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03) Dependency updates: * Upgrade to TF provider 1.65.1 ([#2328](https://github.com/databricks/cli/pull/2328)). * Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0 ([#2237](https://github.com/databricks/cli/pull/2237)). * Bump github.com/spf13/pflag from 1.0.5 to 1.0.6 ([#2281](https://github.com/databricks/cli/pull/2281)). * Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0 ([#2321](https://github.com/databricks/cli/pull/2321)). * Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0 ([#2322](https://github.com/databricks/cli/pull/2322)). * Bump golang.org/x/term from 0.28.0 to 0.29.0 ([#2325](https://github.com/databricks/cli/pull/2325)). * Bump golang.org/x/text from 0.21.0 to 0.22.0 ([#2323](https://github.com/databricks/cli/pull/2323)). * Bump golang.org/x/mod from 0.22.0 to 0.23.0 ([#2324](https://github.com/databricks/cli/pull/2324)). --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 449c30288..c0202b6a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Version changelog +## [Release] Release v0.241.0 + +Bundles: + * Added support to generate Git based jobs ([#2304](https://github.com/databricks/cli/pull/2304)). + * Added support for run_as in pipelines ([#2287](https://github.com/databricks/cli/pull/2287)). + * Raise an error when there are multiple local libraries with the same basename used ([#2297](https://github.com/databricks/cli/pull/2297)). + * Fix env variable for AzureCli local config ([#2248](https://github.com/databricks/cli/pull/2248)). + * Accept JSON files in includes section ([#2265](https://github.com/databricks/cli/pull/2265)). + * Always print warnings and errors; clean up format ([#2213](https://github.com/databricks/cli/pull/2213)) + +API Changes: + * Added `databricks account budget-policy` command group. + * Added `databricks lakeview-embedded` command group. + * Added `databricks query-execution` command group. + * Added `databricks account enable-ip-access-lists` command group. + * Added `databricks redash-config` command group. + +OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03) +Dependency updates: + * Upgrade to TF provider 1.65.1 ([#2328](https://github.com/databricks/cli/pull/2328)). + * Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0 ([#2237](https://github.com/databricks/cli/pull/2237)). + * Bump github.com/spf13/pflag from 1.0.5 to 1.0.6 ([#2281](https://github.com/databricks/cli/pull/2281)). + * Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0 ([#2321](https://github.com/databricks/cli/pull/2321)). + * Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0 ([#2322](https://github.com/databricks/cli/pull/2322)). + * Bump golang.org/x/term from 0.28.0 to 0.29.0 ([#2325](https://github.com/databricks/cli/pull/2325)). + * Bump golang.org/x/text from 0.21.0 to 0.22.0 ([#2323](https://github.com/databricks/cli/pull/2323)). + * Bump golang.org/x/mod from 0.22.0 to 0.23.0 ([#2324](https://github.com/databricks/cli/pull/2324)). + ## [Release] Release v0.240.0 Bundles: From ac439f8c1a60f7b5dd046caa88ad3ed93d9e0c51 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 12 Feb 2025 16:14:30 +0000 Subject: [PATCH 096/105] Fix for regression deploying resources with PyPi and Maven library types (#2341) ## Changes The CheckForSameNameLibraries mutator incorrectly assumed all resource libraries define libraries as paths of the `string` type, but some libraries, such as PyPi and Maven, define them as objects. This PR addresses this issue. It was introduced in #2297. ## Tests Added regression test. --- .../artifacts/same_name_libraries/databricks.yml | 2 ++ .../bundle/artifacts/same_name_libraries/output.txt | 2 +- bundle/libraries/same_name_libraries.go | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml index a065bae76..837cd01e8 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml +++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml @@ -34,6 +34,8 @@ resources: package_name: my_default_python libraries: - whl: ./whl1/dist/*.whl + - pypi: + package: test_package - task_key: task2 new_cluster: ${var.cluster} python_wheel_task: diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt index 38cdd43c4..1253d9680 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/output.txt +++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt @@ -6,7 +6,7 @@ Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl at resources.jobs.test.tasks[0].libraries[0].whl resources.jobs.test.tasks[1].libraries[0].whl in databricks.yml:36:15 - databricks.yml:43:15 + databricks.yml:45:15 Local library names must be unique diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 88b96ab54..8de34cfec 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -31,13 +31,18 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) var err error for _, pattern := range patterns { v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { - libPath := lv.MustString() + libFullPath, ok := lv.AsString() + // If the value is not a string, skip the check because it's not whl or jar type which defines the library + // as a string versus PyPi or Maven which defines the library as a map. + if !ok { + return v, nil + } + // If not local library, skip the check - if !IsLibraryLocal(libPath) { + if !IsLibraryLocal(libFullPath) { return lv, nil } - libFullPath := lv.MustString() lib := filepath.Base(libFullPath) // If the same basename was seen already but full path is different // then it's a duplicate. Add the location to the location list. From 9c90984688060d9c1c889055ab0336536d2a6732 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 12 Feb 2025 16:17:38 +0000 Subject: [PATCH 097/105] [Release] Release v0.241.1 (#2342) Bundles: * Fix for regression deploying resources with PyPi and Maven library types ([#2341](https://github.com/databricks/cli/pull/2341)). --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0202b6a2..d3df510b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Version changelog +## [Release] Release v0.241.1 + +Bundles: + * Fix for regression deploying resources with PyPi and Maven library types ([#2341](https://github.com/databricks/cli/pull/2341)). + ## [Release] Release v0.241.0 Bundles: From 96302c7415331e925fc9d6056b3e34923e84f64f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 12 Feb 2025 20:05:49 +0100 Subject: [PATCH 098/105] Revert changes related to basename check for local libraries (#2345) ## Changes These changes break the use of non-local libraries (such as PyPI libraries). This reverts the set so we can cut a patch release and take a closer look later. Original PRs are #2297 and #2341. Issue reported in #2343. ## Tests Manually confirmed that a bundle with PyPI package in libraries now deploys fine. --- .../same_name_libraries/databricks.yml | 52 -------- .../artifacts/same_name_libraries/output.txt | 14 -- .../artifacts/same_name_libraries/script | 2 - .../artifacts/same_name_libraries/test.toml | 0 .../same_name_libraries/whl1/setup.py | 36 ------ .../whl1/src/my_default_python/__init__.py | 1 - .../whl1/src/my_default_python/main.py | 1 - .../same_name_libraries/whl2/setup.py | 36 ------ .../whl2/src/my_default_python/__init__.py | 1 - .../whl2/src/my_default_python/main.py | 1 - bundle/libraries/expand_glob_references.go | 2 +- bundle/libraries/same_name_libraries.go | 102 --------------- bundle/libraries/same_name_libraries_test.go | 121 ------------------ bundle/phases/deploy.go | 5 - 14 files changed, 1 insertion(+), 373 deletions(-) delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/script delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py delete mode 100644 bundle/libraries/same_name_libraries.go delete mode 100644 bundle/libraries/same_name_libraries_test.go diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml deleted file mode 100644 index 837cd01e8..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml +++ /dev/null @@ -1,52 +0,0 @@ -bundle: - name: same_name_libraries - -variables: - cluster: - default: - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - data_security_mode: SINGLE_USER - num_workers: 0 - spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode - custom_tags: - ResourceClass: SingleNode - -artifacts: - whl1: - type: whl - path: ./whl1 - whl2: - type: whl - path: ./whl2 - -resources: - jobs: - test: - name: "test" - tasks: - - task_key: task1 - new_cluster: ${var.cluster} - python_wheel_task: - entry_point: main - package_name: my_default_python - libraries: - - whl: ./whl1/dist/*.whl - - pypi: - package: test_package - - task_key: task2 - new_cluster: ${var.cluster} - python_wheel_task: - entry_point: main - package_name: my_default_python - libraries: - - whl: ./whl2/dist/*.whl - - task_key: task3 - new_cluster: ${var.cluster} - python_wheel_task: - entry_point: main - package_name: my_default_python - libraries: - - whl: ./whl1/dist/*.whl diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt deleted file mode 100644 index 1253d9680..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/output.txt +++ /dev/null @@ -1,14 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Building whl1... -Building whl2... -Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl - at resources.jobs.test.tasks[0].libraries[0].whl - resources.jobs.test.tasks[1].libraries[0].whl - in databricks.yml:36:15 - databricks.yml:45:15 - -Local library names must be unique - - -Exit code: 1 diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script deleted file mode 100644 index 6c899df07..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/script +++ /dev/null @@ -1,2 +0,0 @@ -trace errcode $CLI bundle deploy -rm -rf whl1 whl2 diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml deleted file mode 100644 index e69de29bb..000000000 diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py deleted file mode 100644 index 1afaf3a4f..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -setup.py configuration script describing how to build and package this project. - -This file is primarily used by the setuptools library and typically should not -be executed directly. See README.md for how to deploy, test, and run -the my_default_python project. -""" - -from setuptools import setup, find_packages - -import sys - -sys.path.append("./src") - -import my_default_python - -setup( - name="my_default_python", - version=my_default_python.__version__, - url="https://databricks.com", - author="[USERNAME]", - description="wheel file based on my_default_python/src", - packages=find_packages(where="./src"), - package_dir={"": "src"}, - entry_points={ - "packages": [ - "main=my_default_python.main:main", - ], - }, - install_requires=[ - # Dependencies in case the output wheel file is used as a library dependency. - # For defining dependencies, when this package is used in Databricks, see: - # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html - "setuptools" - ], -) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py deleted file mode 100644 index f102a9cad..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py deleted file mode 100644 index 11b15b1a4..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py +++ /dev/null @@ -1 +0,0 @@ -print("hello") diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py deleted file mode 100644 index 1afaf3a4f..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -setup.py configuration script describing how to build and package this project. - -This file is primarily used by the setuptools library and typically should not -be executed directly. See README.md for how to deploy, test, and run -the my_default_python project. -""" - -from setuptools import setup, find_packages - -import sys - -sys.path.append("./src") - -import my_default_python - -setup( - name="my_default_python", - version=my_default_python.__version__, - url="https://databricks.com", - author="[USERNAME]", - description="wheel file based on my_default_python/src", - packages=find_packages(where="./src"), - package_dir={"": "src"}, - entry_points={ - "packages": [ - "main=my_default_python.main:main", - ], - }, - install_requires=[ - # Dependencies in case the output wheel file is used as a library dependency. - # For defining dependencies, when this package is used in Databricks, see: - # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html - "setuptools" - ], -) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py deleted file mode 100644 index f102a9cad..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py deleted file mode 100644 index 11b15b1a4..000000000 --- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py +++ /dev/null @@ -1 +0,0 @@ -print("hello") diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 7a808f627..bb1905045 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic for _, match := range matches { output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.NewValue(match, lib.Locations()), + libType: dyn.V(match), }, lib.Locations())) } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go deleted file mode 100644 index 8de34cfec..000000000 --- a/bundle/libraries/same_name_libraries.go +++ /dev/null @@ -1,102 +0,0 @@ -package libraries - -import ( - "context" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -type checkForSameNameLibraries struct{} - -var patterns = []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), - envDepsPattern.Append(dyn.AnyIndex()), -} - -type libData struct { - fullPath string - locations []dyn.Location - paths []dyn.Path -} - -func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - libs := make(map[string]*libData) - - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - var err error - for _, pattern := range patterns { - v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { - libFullPath, ok := lv.AsString() - // If the value is not a string, skip the check because it's not whl or jar type which defines the library - // as a string versus PyPi or Maven which defines the library as a map. - if !ok { - return v, nil - } - - // If not local library, skip the check - if !IsLibraryLocal(libFullPath) { - return lv, nil - } - - lib := filepath.Base(libFullPath) - // If the same basename was seen already but full path is different - // then it's a duplicate. Add the location to the location list. - lp, ok := libs[lib] - if !ok { - libs[lib] = &libData{ - fullPath: libFullPath, - locations: []dyn.Location{lv.Location()}, - paths: []dyn.Path{p}, - } - } else if lp.fullPath != libFullPath { - lp.locations = append(lp.locations, lv.Location()) - lp.paths = append(lp.paths, p) - } - - return lv, nil - }) - if err != nil { - return dyn.InvalidValue, err - } - } - - if err != nil { - return dyn.InvalidValue, err - } - - return v, nil - }) - - // Iterate over all the libraries and check if there are any duplicates. - // Duplicates will have more than one location. - // If there are duplicates, add a diagnostic. - for lib, lv := range libs { - if len(lv.locations) > 1 { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: "Duplicate local library name " + lib, - Detail: "Local library names must be unique", - Locations: lv.locations, - Paths: lv.paths, - }) - } - } - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - - return diags -} - -func (c checkForSameNameLibraries) Name() string { - return "CheckForSameNameLibraries" -} - -func CheckForSameNameLibraries() bundle.Mutator { - return checkForSameNameLibraries{} -} diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go deleted file mode 100644 index 42c38773b..000000000 --- a/bundle/libraries/same_name_libraries_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package libraries - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/require" -) - -func TestSameNameLibraries(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: "full/path/test.whl", - }, - }, - }, - { - Libraries: []compute.Library{ - { - Whl: "other/path/test.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{ - {File: "databricks.yml", Line: 10, Column: 1}, - }) - bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{ - {File: "databricks.yml", Line: 20, Column: 1}, - }) - - diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) - require.Len(t, diags, 1) - require.Equal(t, diag.Error, diags[0].Severity) - require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary) - require.Equal(t, []dyn.Location{ - {File: "databricks.yml", Line: 10, Column: 1}, - {File: "databricks.yml", Line: 20, Column: 1}, - }, diags[0].Locations) - - paths := make([]string, 0) - for _, p := range diags[0].Paths { - paths = append(paths, p.String()) - } - require.Equal(t, []string{ - "resources.jobs.test.tasks[0].libraries[0].whl", - "resources.jobs.test.tasks[1].libraries[0].whl", - }, paths) -} - -func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: "full/path/test-0.1.1.whl", - }, - - { - Whl: "cowsay", - }, - }, - }, - { - Libraries: []compute.Library{ - { - Whl: "other/path/test-0.1.0.whl", - }, - - { - Whl: "cowsay", - }, - }, - }, - { - Libraries: []compute.Library{ - { - Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) - require.Empty(t, diags) -} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 2e9211a7e..c6ec04962 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -155,11 +155,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { mutator.ValidateGitDetails(), artifacts.CleanUp(), libraries.ExpandGlobReferences(), - // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we - // know what are the actual library paths. - // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this - // mutator is part of the deploy step rather than validate. - libraries.CheckForSameNameLibraries(), libraries.Upload(), trampoline.TransformWheelTask(), files.Upload(outputHandler), From a20894b1f2e5569f2dd73df13f973806d8700f93 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 12 Feb 2025 20:21:48 +0100 Subject: [PATCH 099/105] [Release] Release v0.241.2 (#2346) This is a bugfix release to address an issue where jobs with tasks with a libraries section with PyPI packages could not be deployed. Bundles: * Revert changes related to basename check for local libraries ([#2345](https://github.com/databricks/cli/pull/2345)). --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3df510b7..23c696ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Version changelog +## [Release] Release v0.241.2 + +This is a bugfix release to address an issue where jobs with tasks with a +libraries section with PyPI packages could not be deployed. + +Bundles: + * Revert changes related to basename check for local libraries ([#2345](https://github.com/databricks/cli/pull/2345)). + ## [Release] Release v0.241.1 Bundles: From fac9bcf1afe6365dfe49c81ae9ea92135d660de9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 13 Feb 2025 08:26:22 +0100 Subject: [PATCH 100/105] acc: Set X-Databricks-Org-Id on scim/v2/Me endpoint (#2349) This is needed for b.WorkspaceClient().CurrentWorkspaceID(ctx) which is used by initialize_urls.go mutator ("bundle summary") #2316 It also also needed for to call serverless detection endpoint #2348 Builds on top of #2338 --- acceptance/server_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/server_test.go b/acceptance/server_test.go index f73872e0b..4fc3108d2 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -63,7 +63,10 @@ func AddHandlers(server *testserver.Server) { }) server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any { - return testUser + return testserver.Response{ + Headers: map[string][]string{"X-Databricks-Org-Id": {"900800700600"}}, + Body: testUser, + } }) server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any { From 2d0963661136d5901f6acf8fa7c296ff90b9ca8d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 13 Feb 2025 08:31:04 +0100 Subject: [PATCH 101/105] acc: do not show diff for missing output file (#2350) It's not interesting since it just dumps what is in the repo. This is especially annoying with bundle/templates tests with a lot of files. --- acceptance/acceptance_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 85c345032..d99ad2991 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -402,8 +402,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // The test did not produce an expected output file. if okRef && !okNew { - t.Errorf("Missing output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew) - testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew) + t.Errorf("Missing output file: %s", relPath) if testdiff.OverwriteMode { t.Logf("Removing output file: %s", relPath) require.NoError(t, os.Remove(pathRef)) From c0a56a93fb5ad472d9ba88c6e4fd1cf14973ec70 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 14 Feb 2025 12:02:12 +0100 Subject: [PATCH 102/105] acc: add a helper to diff with replacements (#2352) ## Changes diff.py is like "diff -r -U2" but it applies replacements first to the argument. This allows comparing different output files and directories but ignore differences that are going to be replaced by placeholders. This is useful for tests that record large amount of files, specifically "bundle init" with standard templates. In those tests, changing one parameter results in a small diff so recording the full directory is not helpful, because it's hard to see what changed there. I'm using it in implementation of serverless mode for templates that need it: #2348 The serverless templates are slightly different from classic, capturing the diff helps to see exactly where. Related small changes: - Add [TESTROOT] replacement for absolute path to acceptance directory in git repo. - Add $TESTDIR env var for absolute path to a given test in git repo. ## Tests - New test acceptance/selftest/diff to test the helper. - Via #2348 which makes use of this feature. --- acceptance/acceptance_test.go | 20 +++++++ acceptance/bin/diff.py | 56 +++++++++++++++++++ acceptance/selftest/diff/out_dir_a/output.txt | 7 +++ acceptance/selftest/diff/out_dir_b/output.txt | 7 +++ acceptance/selftest/diff/output.txt | 13 +++++ acceptance/selftest/diff/script | 17 ++++++ acceptance/selftest/test.toml | 1 + 7 files changed, 121 insertions(+) create mode 100755 acceptance/bin/diff.py create mode 100644 acceptance/selftest/diff/out_dir_a/output.txt create mode 100644 acceptance/selftest/diff/out_dir_b/output.txt create mode 100644 acceptance/selftest/diff/output.txt create mode 100644 acceptance/selftest/diff/script create mode 100644 acceptance/selftest/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index d99ad2991..c7b1151ab 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -57,6 +57,8 @@ const ( CleanupScript = "script.cleanup" PrepareScript = "script.prepare" MaxFileSize = 100_000 + // Filename to save replacements to (used by diff.py) + ReplsFile = "repls.json" ) var Scripts = map[string]bool{ @@ -65,6 +67,10 @@ var Scripts = map[string]bool{ PrepareScript: true, } +var Ignored = map[string]bool{ + ReplsFile: true, +} + func TestAccept(t *testing.T) { testAccept(t, InprocessMode, SingleTest) } @@ -152,6 +158,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) + repls.SetPath(cwd, "[TESTROOT]") + repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) testDirs := getTests(t) @@ -310,6 +318,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // User replacements come last: repls.Repls = append(repls.Repls, config.Repls...) + // Save replacements to temp test directory so that it can be read by diff.py + replsJson, err := json.MarshalIndent(repls.Repls, "", " ") + require.NoError(t, err) + testutil.WriteFile(t, filepath.Join(tmpDir, ReplsFile), string(replsJson)) + if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one // results in sporadic failures like this one (only if tests are running in parallel): @@ -320,6 +333,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } + absDir, err := filepath.Abs(dir) + require.NoError(t, err) + cmd.Env = append(cmd.Env, "TESTDIR="+absDir) + // Write combined output to a file out, err := os.Create(filepath.Join(tmpDir, "output.txt")) require.NoError(t, err) @@ -368,6 +385,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := outputs[relPath]; ok { continue } + if _, ok := Ignored[relPath]; ok { + continue + } unexpected = append(unexpected, relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" diff --git a/acceptance/bin/diff.py b/acceptance/bin/diff.py new file mode 100755 index 000000000..0a91d57ce --- /dev/null +++ b/acceptance/bin/diff.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""This script implements "diff -r -U2 dir1 dir2" but applies replacements first""" + +import sys +import difflib +import json +import re +from pathlib import Path + + +def replaceAll(patterns, s): + for comp, new in patterns: + s = comp.sub(new, s) + return s + + +def main(): + d1, d2 = sys.argv[1:] + d1, d2 = Path(d1), Path(d2) + + with open("repls.json") as f: + repls = json.load(f) + + patterns = [] + for r in repls: + try: + c = re.compile(r["Old"]) + patterns.append((c, r["New"])) + except re.error as e: + print(f"Regex error for pattern {r}: {e}", file=sys.stderr) + + files1 = [str(p.relative_to(d1)) for p in d1.rglob("*") if p.is_file()] + files2 = [str(p.relative_to(d2)) for p in d2.rglob("*") if p.is_file()] + + set1 = set(files1) + set2 = set(files2) + + for f in sorted(set1 | set2): + p1 = d1 / f + p2 = d2 / f + if f not in set2: + print(f"Only in {d1}: {f}") + elif f not in set1: + print(f"Only in {d2}: {f}") + else: + a = [replaceAll(patterns, x) for x in p1.read_text().splitlines(True)] + b = [replaceAll(patterns, x) for x in p2.read_text().splitlines(True)] + if a != b: + p1_str = p1.as_posix() + p2_str = p2.as_posix() + for line in difflib.unified_diff(a, b, p1_str, p2_str, "", "", 2): + print(line, end="") + + +if __name__ == "__main__": + main() diff --git a/acceptance/selftest/diff/out_dir_a/output.txt b/acceptance/selftest/diff/out_dir_a/output.txt new file mode 100644 index 000000000..303c1867b --- /dev/null +++ b/acceptance/selftest/diff/out_dir_a/output.txt @@ -0,0 +1,7 @@ +Hello! +{ + "id": "[USERID]", + "userName": "[USERNAME]" +} + +Footer \ No newline at end of file diff --git a/acceptance/selftest/diff/out_dir_b/output.txt b/acceptance/selftest/diff/out_dir_b/output.txt new file mode 100644 index 000000000..f4f01af13 --- /dev/null +++ b/acceptance/selftest/diff/out_dir_b/output.txt @@ -0,0 +1,7 @@ +Hello! +{ + "id": "[UUID]", + "userName": "[USERNAME]" +} + +Footer \ No newline at end of file diff --git a/acceptance/selftest/diff/output.txt b/acceptance/selftest/diff/output.txt new file mode 100644 index 000000000..aef99f1e3 --- /dev/null +++ b/acceptance/selftest/diff/output.txt @@ -0,0 +1,13 @@ + +>>> diff.py out_dir_a out_dir_b +Only in out_dir_a: only_in_a +Only in out_dir_b: only_in_b +--- out_dir_a/output.txt ++++ out_dir_b/output.txt +@@ -1,5 +1,5 @@ + Hello! + { +- "id": "[USERID]", ++ "id": "[UUID]", + "userName": "[USERNAME]" + } diff --git a/acceptance/selftest/diff/script b/acceptance/selftest/diff/script new file mode 100644 index 000000000..a7b8706e6 --- /dev/null +++ b/acceptance/selftest/diff/script @@ -0,0 +1,17 @@ +mkdir out_dir_a +mkdir out_dir_b + +touch out_dir_a/only_in_a +touch out_dir_b/only_in_b + +echo Hello! >> out_dir_a/output.txt +echo Hello! >> out_dir_b/output.txt + +curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me >> out_dir_a/output.txt +printf "\n\nFooter" >> out_dir_a/output.txt +printf '{\n "id": "7d639bad-ac6d-4e6f-abd7-9522a86b0239",\n "userName": "[USERNAME]"\n}\n\nFooter' >> out_dir_b/output.txt + +# Unlike regular diff, diff.py will apply replacements first before doing the comparison +errcode trace diff.py out_dir_a out_dir_b + +rm out_dir_a/only_in_a out_dir_b/only_in_b diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml new file mode 100644 index 000000000..b76e712fb --- /dev/null +++ b/acceptance/selftest/test.toml @@ -0,0 +1 @@ +LocalOnly = true From bc30d440972efe1b280d54d3afe5d7593115c978 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 17 Feb 2025 13:38:03 +0100 Subject: [PATCH 103/105] Provide instructions for testing in the default-python template (#2355) ## Changes Adds instructions for testing to the default-python template. ## Tests - Unit & acceptance tests. --- .../default-python/output/my_default_python/README.md | 10 ++++++---- .../template/{{.project_name}}/README.md.tmpl | 10 ++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/README.md index 97d7d7949..10f570bf4 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/README.md +++ b/acceptance/bundle/templates/default-python/output/my_default_python/README.md @@ -37,10 +37,12 @@ The 'my_default_python' project was generated by using the default-python templa ``` $ databricks bundle run ``` - -6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from - https://docs.databricks.com/dev-tools/vscode-ext.html. Or read the "getting started" documentation for - **Databricks Connect** for instructions on running the included Python code from a different IDE. +6. Optionally, install the Databricks extension for Visual Studio code for local development from + https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your + virtual environment and setup Databricks Connect for running unit tests locally. + When not using these tools, consult your development environment's documentation + and/or the documentation for Databricks Connect for manually setting up your environment + (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html). 7. For documentation on the Databricks asset bundles format used for this project, and for CI/CD configuration, see diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl index 53847a9c9..b8811fa3e 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl @@ -38,10 +38,16 @@ The '{{.project_name}}' project was generated by using the default-python templa $ databricks bundle run ``` +{{- if (eq .include_python "no") }} 6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from https://docs.databricks.com/dev-tools/vscode-ext.html. -{{- if (eq .include_python "yes") }} Or read the "getting started" documentation for - **Databricks Connect** for instructions on running the included Python code from a different IDE. +{{- else }} +6. Optionally, install the Databricks extension for Visual Studio code for local development from + https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your + virtual environment and setup Databricks Connect for running unit tests locally. + When not using these tools, consult your development environment's documentation + and/or the documentation for Databricks Connect for manually setting up your environment + (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html). {{- end}} 7. For documentation on the Databricks asset bundles format used From 874a05a27b750f2f732436d2b43d90997dc6b32c Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 18 Feb 2025 17:12:49 +0100 Subject: [PATCH 104/105] Add escaping for links and headers in docsgen (#2330) ## Changes To avoid build warnings and errors in docs build we need to escape symbols that are treated as syntax elements ## Tests --- bundle/docsgen/markdown.go | 23 +- bundle/docsgen/markdown_test.go | 42 ++ bundle/docsgen/nodes.go | 10 +- bundle/docsgen/output/reference.md | 90 +-- bundle/docsgen/output/resources.md | 940 +++++++++++++------------ bundle/docsgen/testdata/anchors.md | 28 + bundle/internal/schema/annotations.yml | 10 - bundle/internal/schema/main_test.go | 20 + 8 files changed, 651 insertions(+), 512 deletions(-) create mode 100644 bundle/docsgen/markdown_test.go create mode 100644 bundle/docsgen/testdata/anchors.md diff --git a/bundle/docsgen/markdown.go b/bundle/docsgen/markdown.go index 6e3b42b65..b711aa0e8 100644 --- a/bundle/docsgen/markdown.go +++ b/bundle/docsgen/markdown.go @@ -12,10 +12,11 @@ func buildMarkdown(nodes []rootNode, outputFile, header string) error { m = m.PlainText(header) for _, node := range nodes { m = m.LF() + title := escapeBrackets(node.Title) if node.TopLevel { - m = m.H2(node.Title) + m = m.H2(title) } else { - m = m.H3(node.Title) + m = m.H3(title) } m = m.LF() @@ -93,7 +94,23 @@ func formatDescription(a attributeNode) string { } else if s != "" { s += ". " } - s += fmt.Sprintf("See [_](#%s).", a.Link) + s += fmt.Sprintf("See [_](#%s).", cleanAnchor(a.Link)) } return s } + +// Docs framework does not allow special characters in anchor links and strip them out by default +// We need to clean them up to make sure the links pass the validation +func cleanAnchor(s string) string { + s = strings.ReplaceAll(s, "<", "") + s = strings.ReplaceAll(s, ">", "") + s = strings.ReplaceAll(s, ".", "") + + return s +} + +func escapeBrackets(s string) string { + s = strings.ReplaceAll(s, "<", "\\<") + s = strings.ReplaceAll(s, ">", "\\>") + return s +} diff --git a/bundle/docsgen/markdown_test.go b/bundle/docsgen/markdown_test.go new file mode 100644 index 000000000..d4f32230e --- /dev/null +++ b/bundle/docsgen/markdown_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/stretchr/testify/require" +) + +func TestBuildMarkdownAnchors(t *testing.T) { + nodes := []rootNode{ + { + Title: "some_field", + TopLevel: true, + Type: "Map", + Description: "This is a description", + Attributes: []attributeNode{ + { + Title: "my_attribute", + Type: "Map", + Description: "Desc with link", + Link: "some_field..my_attribute", + }, + }, + }, + { + Title: "some_field..my_attribute", + TopLevel: false, + Type: "Boolean", + Description: "Another description", + }, + } + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "output.md") + + err := buildMarkdown(nodes, path, "Header") + require.NoError(t, err) + + expected := testutil.ReadFile(t, "testdata/anchors.md") + testutil.AssertFileContents(t, path, expected) +} diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go index 61d2c21cc..6645e9ccc 100644 --- a/bundle/docsgen/nodes.go +++ b/bundle/docsgen/nodes.go @@ -65,7 +65,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel v = resolveRefs(v, refs) node := rootNode{ Title: k, - Description: getDescription(v, item.topLevel), + Description: getDescription(v), TopLevel: item.topLevel, Example: getExample(v), Type: getHumanReadableType(v.Type), @@ -78,7 +78,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel mapValueType := getMapValueType(v, refs) if mapValueType != nil { - d := getDescription(mapValueType, true) + d := getDescription(mapValueType) if d != "" { node.Description = d } @@ -174,7 +174,7 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri attributes = append(attributes, attributeNode{ Title: k, Type: typeString, - Description: getDescription(v, true), + Description: getDescription(v), Link: reference, }) } @@ -184,8 +184,8 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri return attributes } -func getDescription(s *jsonschema.Schema, allowMarkdown bool) string { - if allowMarkdown && s.MarkdownDescription != "" { +func getDescription(s *jsonschema.Schema) string { + if s.MarkdownDescription != "" { return s.MarkdownDescription } return s.Description diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 8a89d354b..0de3c6f2b 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -43,7 +43,7 @@ artifacts: * - `files` - Sequence - - The source files for the artifact. See [_](#artifacts..files). + - The source files for the artifact. See [_](#artifactsnamefiles). * - `path` - String @@ -64,7 +64,7 @@ artifacts: path: . ``` -### artifacts..files +### artifacts.\.files **`Type: Sequence`** @@ -113,11 +113,11 @@ The bundle attributes when deploying to this target, * - `deployment` - Map - - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundle.deployment). + - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundledeployment). * - `git` - Map - - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundle.git). + - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundlegit). * - `name` - String @@ -132,7 +132,7 @@ The bundle attributes when deploying to this target, **`Type: Map`** -The definition of the bundle deployment +The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). @@ -149,7 +149,7 @@ The definition of the bundle deployment * - `lock` - Map - - The deployment lock attributes. See [_](#bundle.deployment.lock). + - The deployment lock attributes. See [_](#bundledeploymentlock). ### bundle.deployment.lock @@ -180,7 +180,7 @@ The deployment lock attributes. **`Type: Map`** -The Git version control details that are associated with your bundle. +The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). @@ -217,11 +217,11 @@ Defines attributes for experimental features. * - `pydabs` - Map - - The PyDABs configuration. See [_](#experimental.pydabs). + - The PyDABs configuration. See [_](#experimentalpydabs). * - `python` - Map - - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimental.python). + - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimentalpython). * - `python_wheel_wrapper` - Boolean @@ -530,11 +530,11 @@ targets: * - `artifacts` - Map - - The artifacts to include in the target deployment. See [_](#targets..artifacts). + - The artifacts to include in the target deployment. See [_](#targetsnameartifacts). * - `bundle` - Map - - The bundle attributes when deploying to this target. See [_](#targets..bundle). + - The bundle attributes when deploying to this target. See [_](#targetsnamebundle). * - `cluster_id` - String @@ -550,7 +550,7 @@ targets: * - `git` - Map - - The Git version control settings for the target. See [_](#targets..git). + - The Git version control settings for the target. See [_](#targetsnamegit). * - `mode` - String @@ -558,34 +558,34 @@ targets: * - `permissions` - Sequence - - The permissions for deploying and running the bundle in the target. See [_](#targets..permissions). + - The permissions for deploying and running the bundle in the target. See [_](#targetsnamepermissions). * - `presets` - Map - - The deployment presets for the target. See [_](#targets..presets). + - The deployment presets for the target. See [_](#targetsnamepresets). * - `resources` - Map - - The resource definitions for the target. See [_](#targets..resources). + - The resource definitions for the target. See [_](#targetsnameresources). * - `run_as` - Map - - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targets..run_as). + - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targetsnamerun_as). * - `sync` - Map - - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targets..sync). + - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targetsnamesync). * - `variables` - Map - - The custom variable definitions for the target. See [_](#targets..variables). + - The custom variable definitions for the target. See [_](#targetsnamevariables). * - `workspace` - Map - - The Databricks workspace for the target. See [_](#targets..workspace). + - The Databricks workspace for the target. See [_](#targetsnameworkspace). -### targets..artifacts +### targets.\.artifacts **`Type: Map`** @@ -615,7 +615,7 @@ artifacts: * - `files` - Sequence - - The source files for the artifact. See [_](#targets..artifacts..files). + - The source files for the artifact. See [_](#targetsnameartifactsnamefiles). * - `path` - String @@ -626,7 +626,7 @@ artifacts: - Required. The type of the artifact. Valid values are `whl`. -### targets..artifacts..files +### targets.\.artifacts.\.files **`Type: Sequence`** @@ -646,7 +646,7 @@ The source files for the artifact. - Required. The path of the files used to build the artifact. -### targets..bundle +### targets.\.bundle **`Type: Map`** @@ -675,11 +675,11 @@ The bundle attributes when deploying to this target. * - `deployment` - Map - - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targets..bundle.deployment). + - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targetsnamebundledeployment). * - `git` - Map - - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targets..bundle.git). + - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targetsnamebundlegit). * - `name` - String @@ -690,11 +690,11 @@ The bundle attributes when deploying to this target. - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command). -### targets..bundle.deployment +### targets.\.bundle.deployment **`Type: Map`** -The definition of the bundle deployment +The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). @@ -711,10 +711,10 @@ The definition of the bundle deployment * - `lock` - Map - - The deployment lock attributes. See [_](#targets..bundle.deployment.lock). + - The deployment lock attributes. See [_](#targetsnamebundledeploymentlock). -### targets..bundle.deployment.lock +### targets.\.bundle.deployment.lock **`Type: Map`** @@ -738,11 +738,11 @@ The deployment lock attributes. - Whether to force this lock if it is enabled. -### targets..bundle.git +### targets.\.bundle.git **`Type: Map`** -The Git version control details that are associated with your bundle. +The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). @@ -762,7 +762,7 @@ The Git version control details that are associated with your bundle. - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git). -### targets..git +### targets.\.git **`Type: Map`** @@ -786,7 +786,7 @@ The Git version control settings for the target. - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git). -### targets..permissions +### targets.\.permissions **`Type: Sequence`** @@ -818,7 +818,7 @@ The permissions for deploying and running the bundle in the target. - The name of the user that has the permission set in level. -### targets..presets +### targets.\.presets **`Type: Map`** @@ -858,7 +858,7 @@ The deployment presets for the target. - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED. -### targets..resources +### targets.\.resources **`Type: Map`** @@ -922,11 +922,11 @@ The resource definitions for the target. - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes) -### targets..run_as +### targets.\.run_as **`Type: Map`** -The identity to use to run the bundle. +The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). @@ -946,7 +946,7 @@ The identity to use to run the bundle. - The email of an active workspace user. Non-admin users can only set this field to their own email. -### targets..sync +### targets.\.sync **`Type: Map`** @@ -974,7 +974,7 @@ The local paths to sync to the target workspace when a bundle is run or deployed - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed. -### targets..variables +### targets.\.variables **`Type: Map`** @@ -1004,14 +1004,14 @@ variables: * - `lookup` - Map - - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targets..variables..lookup). + - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targetsnamevariablesnamelookup). * - `type` - String - The type of the variable. -### targets..variables..lookup +### targets.\.variables.\.lookup **`Type: Map`** @@ -1075,7 +1075,7 @@ The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, m - -### targets..workspace +### targets.\.workspace **`Type: Map`** @@ -1185,18 +1185,18 @@ variables: * - `lookup` - Map - - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variables..lookup). + - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variablesnamelookup). * - `type` - String - The type of the variable. -### variables..lookup +### variables.\.lookup **`Type: Map`** -The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. +The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index df7578c73..e1bbc9672 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -93,15 +93,15 @@ apps: * - `active_deployment` - Map - - See [_](#apps..active_deployment). + - See [_](#appsnameactive_deployment). * - `app_status` - Map - - See [_](#apps..app_status). + - See [_](#appsnameapp_status). * - `compute_status` - Map - - See [_](#apps..compute_status). + - See [_](#appsnamecompute_status). * - `config` - Map @@ -129,15 +129,15 @@ apps: * - `pending_deployment` - Map - - See [_](#apps..pending_deployment). + - See [_](#appsnamepending_deployment). * - `permissions` - Sequence - - See [_](#apps..permissions). + - See [_](#appsnamepermissions). * - `resources` - Sequence - - See [_](#apps..resources). + - See [_](#appsnameresources). * - `service_principal_client_id` - String @@ -168,7 +168,7 @@ apps: - -### apps..active_deployment +### apps.\.active_deployment **`Type: Map`** @@ -193,7 +193,7 @@ apps: * - `deployment_artifacts` - Map - - See [_](#apps..active_deployment.deployment_artifacts). + - See [_](#appsnameactive_deploymentdeployment_artifacts). * - `deployment_id` - String @@ -209,14 +209,14 @@ apps: * - `status` - Map - - See [_](#apps..active_deployment.status). + - See [_](#appsnameactive_deploymentstatus). * - `update_time` - String - -### apps..active_deployment.deployment_artifacts +### apps.\.active_deployment.deployment_artifacts **`Type: Map`** @@ -236,7 +236,7 @@ apps: - -### apps..active_deployment.status +### apps.\.active_deployment.status **`Type: Map`** @@ -260,7 +260,7 @@ apps: - -### apps..app_status +### apps.\.app_status **`Type: Map`** @@ -284,7 +284,7 @@ apps: - -### apps..compute_status +### apps.\.compute_status **`Type: Map`** @@ -308,7 +308,7 @@ apps: - State of the app compute. -### apps..pending_deployment +### apps.\.pending_deployment **`Type: Map`** @@ -333,7 +333,7 @@ apps: * - `deployment_artifacts` - Map - - See [_](#apps..pending_deployment.deployment_artifacts). + - See [_](#appsnamepending_deploymentdeployment_artifacts). * - `deployment_id` - String @@ -349,14 +349,14 @@ apps: * - `status` - Map - - See [_](#apps..pending_deployment.status). + - See [_](#appsnamepending_deploymentstatus). * - `update_time` - String - -### apps..pending_deployment.deployment_artifacts +### apps.\.pending_deployment.deployment_artifacts **`Type: Map`** @@ -376,7 +376,7 @@ apps: - -### apps..pending_deployment.status +### apps.\.pending_deployment.status **`Type: Map`** @@ -400,7 +400,7 @@ apps: - -### apps..permissions +### apps.\.permissions **`Type: Sequence`** @@ -432,7 +432,7 @@ apps: - The name of the user that has the permission set in level. -### apps..resources +### apps.\.resources **`Type: Sequence`** @@ -453,7 +453,7 @@ apps: * - `job` - Map - - See [_](#apps..resources.job). + - See [_](#appsnameresourcesjob). * - `name` - String @@ -461,18 +461,18 @@ apps: * - `secret` - Map - - See [_](#apps..resources.secret). + - See [_](#appsnameresourcessecret). * - `serving_endpoint` - Map - - See [_](#apps..resources.serving_endpoint). + - See [_](#appsnameresourcesserving_endpoint). * - `sql_warehouse` - Map - - See [_](#apps..resources.sql_warehouse). + - See [_](#appsnameresourcessql_warehouse). -### apps..resources.job +### apps.\.resources.job **`Type: Map`** @@ -496,7 +496,7 @@ apps: - -### apps..resources.secret +### apps.\.resources.secret **`Type: Map`** @@ -524,7 +524,7 @@ apps: - -### apps..resources.serving_endpoint +### apps.\.resources.serving_endpoint **`Type: Map`** @@ -548,7 +548,7 @@ apps: - -### apps..resources.sql_warehouse +### apps.\.resources.sql_warehouse **`Type: Map`** @@ -598,7 +598,7 @@ clusters: * - `autoscale` - Map - - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clusters..autoscale). + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#clustersnameautoscale). * - `autotermination_minutes` - Integer @@ -606,15 +606,15 @@ clusters: * - `aws_attributes` - Map - - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..aws_attributes). + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnameaws_attributes). * - `azure_attributes` - Map - - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..azure_attributes). + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnameazure_attributes). * - `cluster_log_conf` - Map - - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clusters..cluster_log_conf). + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#clustersnamecluster_log_conf). * - `cluster_name` - String @@ -630,7 +630,7 @@ clusters: * - `docker_image` - Map - - See [_](#clusters..docker_image). + - See [_](#clustersnamedocker_image). * - `driver_instance_pool_id` - String @@ -650,11 +650,11 @@ clusters: * - `gcp_attributes` - Map - - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clusters..gcp_attributes). + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#clustersnamegcp_attributes). * - `init_scripts` - Sequence - - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#clusters..init_scripts). + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#clustersnameinit_scripts). * - `instance_pool_id` - String @@ -678,7 +678,7 @@ clusters: * - `permissions` - Sequence - - See [_](#clusters..permissions). + - See [_](#clustersnamepermissions). * - `policy_id` - String @@ -714,7 +714,7 @@ clusters: * - `workload_type` - Map - - See [_](#clusters..workload_type). + - See [_](#clustersnameworkload_type). **Example** @@ -745,7 +745,7 @@ resources: notebook_path: "./src/my_notebook.py" ``` -### clusters..autoscale +### clusters.\.autoscale **`Type: Map`** @@ -770,7 +770,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later. - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. -### clusters..aws_attributes +### clusters.\.aws_attributes **`Type: Map`** @@ -827,7 +827,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. -### clusters..azure_attributes +### clusters.\.azure_attributes **`Type: Map`** @@ -853,14 +853,14 @@ If not specified at cluster creation, a set of default values will be used. * - `log_analytics_info` - Map - - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clusters..azure_attributes.log_analytics_info). + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#clustersnameazure_attributeslog_analytics_info). * - `spot_bid_max_price` - Any - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. -### clusters..azure_attributes.log_analytics_info +### clusters.\.azure_attributes.log_analytics_info **`Type: Map`** @@ -884,7 +884,7 @@ Defines values necessary to configure and run Azure Log Analytics agent - -### clusters..cluster_log_conf +### clusters.\.cluster_log_conf **`Type: Map`** @@ -905,14 +905,14 @@ the destination of executor logs is `$destination/$clusterId/executor`. * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters..cluster_log_conf.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clustersnamecluster_log_confdbfs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters..cluster_log_conf.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clustersnamecluster_log_confs3). -### clusters..cluster_log_conf.dbfs +### clusters.\.cluster_log_conf.dbfs **`Type: Map`** @@ -933,7 +933,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### clusters..cluster_log_conf.s3 +### clusters.\.cluster_log_conf.s3 **`Type: Map`** @@ -980,7 +980,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### clusters..docker_image +### clusters.\.docker_image **`Type: Map`** @@ -997,14 +997,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in * - `basic_auth` - Map - - See [_](#clusters..docker_image.basic_auth). + - See [_](#clustersnamedocker_imagebasic_auth). * - `url` - String - URL of the docker image. -### clusters..docker_image.basic_auth +### clusters.\.docker_image.basic_auth **`Type: Map`** @@ -1028,7 +1028,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - Name of the user -### clusters..gcp_attributes +### clusters.\.gcp_attributes **`Type: Map`** @@ -1069,7 +1069,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. -### clusters..init_scripts +### clusters.\.init_scripts **`Type: Sequence`** @@ -1086,34 +1086,34 @@ The configuration for storing init scripts. Any number of destinations can be sp * - `abfss` - Map - - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#clusters..init_scripts.abfss). + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#clustersnameinit_scriptsabfss). * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clusters..init_scripts.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#clustersnameinit_scriptsdbfs). * - `file` - Map - - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clusters..init_scripts.file). + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#clustersnameinit_scriptsfile). * - `gcs` - Map - - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clusters..init_scripts.gcs). + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#clustersnameinit_scriptsgcs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clusters..init_scripts.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#clustersnameinit_scriptss3). * - `volumes` - Map - - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clusters..init_scripts.volumes). + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#clustersnameinit_scriptsvolumes). * - `workspace` - Map - - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clusters..init_scripts.workspace). + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#clustersnameinit_scriptsworkspace). -### clusters..init_scripts.abfss +### clusters.\.init_scripts.abfss **`Type: Map`** @@ -1134,7 +1134,7 @@ destination needs to be provided. e.g. - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. -### clusters..init_scripts.dbfs +### clusters.\.init_scripts.dbfs **`Type: Map`** @@ -1155,7 +1155,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### clusters..init_scripts.file +### clusters.\.init_scripts.file **`Type: Map`** @@ -1176,7 +1176,7 @@ destination needs to be provided. e.g. - local file destination, e.g. `file:/my/local/file.sh` -### clusters..init_scripts.gcs +### clusters.\.init_scripts.gcs **`Type: Map`** @@ -1197,7 +1197,7 @@ destination needs to be provided. e.g. - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` -### clusters..init_scripts.s3 +### clusters.\.init_scripts.s3 **`Type: Map`** @@ -1244,7 +1244,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### clusters..init_scripts.volumes +### clusters.\.init_scripts.volumes **`Type: Map`** @@ -1265,7 +1265,7 @@ destination needs to be provided. e.g. - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` -### clusters..init_scripts.workspace +### clusters.\.init_scripts.workspace **`Type: Map`** @@ -1286,7 +1286,7 @@ destination needs to be provided. e.g. - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` -### clusters..permissions +### clusters.\.permissions **`Type: Sequence`** @@ -1318,7 +1318,7 @@ destination needs to be provided. e.g. - The name of the user that has the permission set in level. -### clusters..workload_type +### clusters.\.workload_type **`Type: Map`** @@ -1335,10 +1335,10 @@ destination needs to be provided. e.g. * - `clients` - Map - - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clusters..workload_type.clients). + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#clustersnameworkload_typeclients). -### clusters..workload_type.clients +### clusters.\.workload_type.clients **`Type: Map`** @@ -1420,7 +1420,7 @@ dashboards: * - `permissions` - Sequence - - See [_](#dashboards..permissions). + - See [_](#dashboardsnamepermissions). * - `serialized_dashboard` - Any @@ -1451,7 +1451,7 @@ If you use the UI to modify the dashboard, modifications made through the UI are In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). -### dashboards..permissions +### dashboards.\.permissions **`Type: Sequence`** @@ -1529,11 +1529,11 @@ experiments: * - `permissions` - Sequence - - See [_](#experiments..permissions). + - See [_](#experimentsnamepermissions). * - `tags` - Sequence - - Tags: Additional metadata key-value pairs. See [_](#experiments..tags). + - Tags: Additional metadata key-value pairs. See [_](#experimentsnametags). **Example** @@ -1551,7 +1551,7 @@ resources: description: MLflow experiment used to track runs ``` -### experiments..permissions +### experiments.\.permissions **`Type: Sequence`** @@ -1583,7 +1583,7 @@ resources: - The name of the user that has the permission set in level. -### experiments..tags +### experiments.\.tags **`Type: Sequence`** @@ -1633,11 +1633,11 @@ jobs: * - `continuous` - Map - - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobs..continuous). + - An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used. See [_](#jobsnamecontinuous). * - `deployment` - Map - - Deployment information for jobs managed by external sources. See [_](#jobs..deployment). + - Deployment information for jobs managed by external sources. See [_](#jobsnamedeployment). * - `description` - String @@ -1649,11 +1649,11 @@ jobs: * - `email_notifications` - Map - - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobs..email_notifications). + - An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted. See [_](#jobsnameemail_notifications). * - `environments` - Sequence - - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobs..environments). + - A list of task execution environment specifications that can be referenced by serverless tasks of this job. An environment is required to be present for serverless tasks. For serverless notebook tasks, the environment is accessible in the notebook environment panel. For other serverless tasks, the task environment is required to be specified using environment_key in the task settings. See [_](#jobsnameenvironments). * - `format` - String @@ -1661,15 +1661,15 @@ jobs: * - `git_source` - Map - - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobs..git_source). + - An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks. If `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task. Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job. See [_](#jobsnamegit_source). * - `health` - Map - - An optional set of health rules that can be defined for this job. See [_](#jobs..health). + - An optional set of health rules that can be defined for this job. See [_](#jobsnamehealth). * - `job_clusters` - Sequence - - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobs..job_clusters). + - A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings. If more than 100 job clusters are available, you can paginate through them using :method:jobs/get. See [_](#jobsnamejob_clusters). * - `max_concurrent_runs` - Integer @@ -1681,27 +1681,31 @@ jobs: * - `notification_settings` - Map - - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobs..notification_settings). + - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job. See [_](#jobsnamenotification_settings). * - `parameters` - Sequence - - Job-level parameter definitions. See [_](#jobs..parameters). + - Job-level parameter definitions. See [_](#jobsnameparameters). + + * - `performance_target` + - String + - PerformanceTarget defines how performant or cost efficient the execution of run on serverless should be. * - `permissions` - Sequence - - See [_](#jobs..permissions). + - See [_](#jobsnamepermissions). * - `queue` - Map - - The queue settings of the job. See [_](#jobs..queue). + - The queue settings of the job. See [_](#jobsnamequeue). * - `run_as` - Map - - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job. Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobs..run_as). + - Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job. Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown. See [_](#jobsnamerun_as). * - `schedule` - Map - - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs..schedule). + - An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobsnameschedule). * - `tags` - Map @@ -1709,7 +1713,7 @@ jobs: * - `tasks` - Sequence - - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobs..tasks). + - A list of task specifications to be executed by this job. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available. See [_](#jobsnametasks). * - `timeout_seconds` - Integer @@ -1717,11 +1721,11 @@ jobs: * - `trigger` - Map - - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobs..trigger). + - A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`. See [_](#jobsnametrigger). * - `webhook_notifications` - Map - - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobs..webhook_notifications). + - A collection of system notification IDs to notify when runs of this job begin or complete. See [_](#jobsnamewebhook_notifications). **Example** @@ -1741,7 +1745,7 @@ resources: For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md). -### jobs..continuous +### jobs.\.continuous **`Type: Map`** @@ -1761,7 +1765,7 @@ An optional continuous property for this job. The continuous property will ensur - Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED. -### jobs..deployment +### jobs.\.deployment **`Type: Map`** @@ -1785,7 +1789,7 @@ Deployment information for jobs managed by external sources. - Path of the file that contains deployment metadata. -### jobs..email_notifications +### jobs.\.email_notifications **`Type: Map`** @@ -1825,7 +1829,7 @@ An optional set of email addresses that is notified when runs of this job begin - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. -### jobs..environments +### jobs.\.environments **`Type: Sequence`** @@ -1849,10 +1853,10 @@ For other serverless tasks, the task environment is required to be specified usi * - `spec` - Map - - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobs..environments.spec). + - The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task. In this minimal environment spec, only pip dependencies are supported. See [_](#jobsnameenvironmentsspec). -### jobs..environments.spec +### jobs.\.environments.spec **`Type: Map`** @@ -1877,7 +1881,7 @@ In this minimal environment spec, only pip dependencies are supported. - List of pip dependencies, as supported by the version of pip in this environment. Each dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be , , (WSFS or Volumes in Databricks), E.g. dependencies: ["foo==0.0.1", "-r /Workspace/test/requirements.txt"] -### jobs..git_source +### jobs.\.git_source **`Type: Map`** @@ -1910,7 +1914,7 @@ Note: dbt and SQL File tasks support only version-controlled sources. If dbt or * - `git_snapshot` - Map - - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobs..git_source.git_snapshot). + - Read-only state of the remote repository at the time the job was run. This field is only included on job runs. See [_](#jobsnamegit_sourcegit_snapshot). * - `git_tag` - String @@ -1922,10 +1926,10 @@ Note: dbt and SQL File tasks support only version-controlled sources. If dbt or * - `job_source` - Map - - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobs..git_source.job_source). + - The source of the job specification in the remote repository when the job is source controlled. See [_](#jobsnamegit_sourcejob_source). -### jobs..git_source.git_snapshot +### jobs.\.git_source.git_snapshot **`Type: Map`** @@ -1945,7 +1949,7 @@ Read-only state of the remote repository at the time the job was run. This field - Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to. -### jobs..git_source.job_source +### jobs.\.git_source.job_source **`Type: Map`** @@ -1973,7 +1977,7 @@ The source of the job specification in the remote repository when the job is sou - Path of the job YAML file that contains the job specification. -### jobs..health +### jobs.\.health **`Type: Map`** @@ -1990,10 +1994,10 @@ An optional set of health rules that can be defined for this job. * - `rules` - Sequence - - See [_](#jobs..health.rules). + - See [_](#jobsnamehealthrules). -### jobs..health.rules +### jobs.\.health.rules **`Type: Sequence`** @@ -2021,7 +2025,7 @@ An optional set of health rules that can be defined for this job. - Specifies the threshold value that the health metric should obey to satisfy the health rule. -### jobs..job_clusters +### jobs.\.job_clusters **`Type: Sequence`** @@ -2043,10 +2047,10 @@ If more than 100 job clusters are available, you can paginate through them using * - `new_cluster` - Map - - If new_cluster, a description of a cluster that is created for each task. See [_](#jobs..job_clusters.new_cluster). + - If new_cluster, a description of a cluster that is created for each task. See [_](#jobsnamejob_clustersnew_cluster). -### jobs..job_clusters.new_cluster +### jobs.\.job_clusters.new_cluster **`Type: Map`** @@ -2067,7 +2071,7 @@ If new_cluster, a description of a cluster that is created for each task. * - `autoscale` - Map - - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs..job_clusters.new_cluster.autoscale). + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobsnamejob_clustersnew_clusterautoscale). * - `autotermination_minutes` - Integer @@ -2075,15 +2079,15 @@ If new_cluster, a description of a cluster that is created for each task. * - `aws_attributes` - Map - - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.aws_attributes). + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clusteraws_attributes). * - `azure_attributes` - Map - - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.azure_attributes). + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clusterazure_attributes). * - `cluster_log_conf` - Map - - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf). + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobsnamejob_clustersnew_clustercluster_log_conf). * - `cluster_name` - String @@ -2099,7 +2103,7 @@ If new_cluster, a description of a cluster that is created for each task. * - `docker_image` - Map - - See [_](#jobs..job_clusters.new_cluster.docker_image). + - See [_](#jobsnamejob_clustersnew_clusterdocker_image). * - `driver_instance_pool_id` - String @@ -2119,11 +2123,11 @@ If new_cluster, a description of a cluster that is created for each task. * - `gcp_attributes` - Map - - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..job_clusters.new_cluster.gcp_attributes). + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnamejob_clustersnew_clustergcp_attributes). * - `init_scripts` - Sequence - - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobs..job_clusters.new_cluster.init_scripts). + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobsnamejob_clustersnew_clusterinit_scripts). * - `instance_pool_id` - String @@ -2179,10 +2183,10 @@ If new_cluster, a description of a cluster that is created for each task. * - `workload_type` - Map - - See [_](#jobs..job_clusters.new_cluster.workload_type). + - See [_](#jobsnamejob_clustersnew_clusterworkload_type). -### jobs..job_clusters.new_cluster.autoscale +### jobs.\.job_clusters.new_cluster.autoscale **`Type: Map`** @@ -2207,7 +2211,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later. - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. -### jobs..job_clusters.new_cluster.aws_attributes +### jobs.\.job_clusters.new_cluster.aws_attributes **`Type: Map`** @@ -2264,7 +2268,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. -### jobs..job_clusters.new_cluster.azure_attributes +### jobs.\.job_clusters.new_cluster.azure_attributes **`Type: Map`** @@ -2290,14 +2294,14 @@ If not specified at cluster creation, a set of default values will be used. * - `log_analytics_info` - Map - - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs..job_clusters.new_cluster.azure_attributes.log_analytics_info). + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobsnamejob_clustersnew_clusterazure_attributeslog_analytics_info). * - `spot_bid_max_price` - Any - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. -### jobs..job_clusters.new_cluster.azure_attributes.log_analytics_info +### jobs.\.job_clusters.new_cluster.azure_attributes.log_analytics_info **`Type: Map`** @@ -2321,7 +2325,7 @@ Defines values necessary to configure and run Azure Log Analytics agent - -### jobs..job_clusters.new_cluster.cluster_log_conf +### jobs.\.job_clusters.new_cluster.cluster_log_conf **`Type: Map`** @@ -2342,14 +2346,14 @@ the destination of executor logs is `$destination/$clusterId/executor`. * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnamejob_clustersnew_clustercluster_log_confdbfs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..job_clusters.new_cluster.cluster_log_conf.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnamejob_clustersnew_clustercluster_log_confs3). -### jobs..job_clusters.new_cluster.cluster_log_conf.dbfs +### jobs.\.job_clusters.new_cluster.cluster_log_conf.dbfs **`Type: Map`** @@ -2370,7 +2374,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### jobs..job_clusters.new_cluster.cluster_log_conf.s3 +### jobs.\.job_clusters.new_cluster.cluster_log_conf.s3 **`Type: Map`** @@ -2417,7 +2421,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### jobs..job_clusters.new_cluster.docker_image +### jobs.\.job_clusters.new_cluster.docker_image **`Type: Map`** @@ -2434,14 +2438,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in * - `basic_auth` - Map - - See [_](#jobs..job_clusters.new_cluster.docker_image.basic_auth). + - See [_](#jobsnamejob_clustersnew_clusterdocker_imagebasic_auth). * - `url` - String - URL of the docker image. -### jobs..job_clusters.new_cluster.docker_image.basic_auth +### jobs.\.job_clusters.new_cluster.docker_image.basic_auth **`Type: Map`** @@ -2465,7 +2469,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - Name of the user -### jobs..job_clusters.new_cluster.gcp_attributes +### jobs.\.job_clusters.new_cluster.gcp_attributes **`Type: Map`** @@ -2506,7 +2510,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. -### jobs..job_clusters.new_cluster.init_scripts +### jobs.\.job_clusters.new_cluster.init_scripts **`Type: Sequence`** @@ -2523,34 +2527,34 @@ The configuration for storing init scripts. Any number of destinations can be sp * - `abfss` - Map - - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobs..job_clusters.new_cluster.init_scripts.abfss). + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsabfss). * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsdbfs). * - `file` - Map - - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.file). + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsfile). * - `gcs` - Map - - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.gcs). + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsgcs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..job_clusters.new_cluster.init_scripts.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnamejob_clustersnew_clusterinit_scriptss3). * - `volumes` - Map - - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.volumes). + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsvolumes). * - `workspace` - Map - - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs..job_clusters.new_cluster.init_scripts.workspace). + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobsnamejob_clustersnew_clusterinit_scriptsworkspace). -### jobs..job_clusters.new_cluster.init_scripts.abfss +### jobs.\.job_clusters.new_cluster.init_scripts.abfss **`Type: Map`** @@ -2571,7 +2575,7 @@ destination needs to be provided. e.g. - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. -### jobs..job_clusters.new_cluster.init_scripts.dbfs +### jobs.\.job_clusters.new_cluster.init_scripts.dbfs **`Type: Map`** @@ -2592,7 +2596,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### jobs..job_clusters.new_cluster.init_scripts.file +### jobs.\.job_clusters.new_cluster.init_scripts.file **`Type: Map`** @@ -2613,7 +2617,7 @@ destination needs to be provided. e.g. - local file destination, e.g. `file:/my/local/file.sh` -### jobs..job_clusters.new_cluster.init_scripts.gcs +### jobs.\.job_clusters.new_cluster.init_scripts.gcs **`Type: Map`** @@ -2634,7 +2638,7 @@ destination needs to be provided. e.g. - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` -### jobs..job_clusters.new_cluster.init_scripts.s3 +### jobs.\.job_clusters.new_cluster.init_scripts.s3 **`Type: Map`** @@ -2681,7 +2685,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### jobs..job_clusters.new_cluster.init_scripts.volumes +### jobs.\.job_clusters.new_cluster.init_scripts.volumes **`Type: Map`** @@ -2702,7 +2706,7 @@ destination needs to be provided. e.g. - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` -### jobs..job_clusters.new_cluster.init_scripts.workspace +### jobs.\.job_clusters.new_cluster.init_scripts.workspace **`Type: Map`** @@ -2723,7 +2727,7 @@ destination needs to be provided. e.g. - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` -### jobs..job_clusters.new_cluster.workload_type +### jobs.\.job_clusters.new_cluster.workload_type **`Type: Map`** @@ -2740,10 +2744,10 @@ destination needs to be provided. e.g. * - `clients` - Map - - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs..job_clusters.new_cluster.workload_type.clients). + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobsnamejob_clustersnew_clusterworkload_typeclients). -### jobs..job_clusters.new_cluster.workload_type.clients +### jobs.\.job_clusters.new_cluster.workload_type.clients **`Type: Map`** @@ -2767,7 +2771,7 @@ destination needs to be provided. e.g. - With notebooks set, this cluster can be used for notebooks -### jobs..notification_settings +### jobs.\.notification_settings **`Type: Map`** @@ -2791,7 +2795,7 @@ Optional notification settings that are used when sending notifications to each - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. -### jobs..parameters +### jobs.\.parameters **`Type: Sequence`** @@ -2815,7 +2819,7 @@ Job-level parameter definitions - The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.` -### jobs..permissions +### jobs.\.permissions **`Type: Sequence`** @@ -2847,7 +2851,7 @@ Job-level parameter definitions - The name of the user that has the permission set in level. -### jobs..queue +### jobs.\.queue **`Type: Map`** @@ -2867,7 +2871,7 @@ The queue settings of the job. - If true, enable queueing for the job. This is a required field. -### jobs..run_as +### jobs.\.run_as **`Type: Map`** @@ -2893,7 +2897,7 @@ Either `user_name` or `service_principal_name` should be specified. If not, an e - The email of an active workspace user. Non-admin users can only set this field to their own email. -### jobs..schedule +### jobs.\.schedule **`Type: Map`** @@ -2921,7 +2925,7 @@ An optional periodic schedule for this job. The default behavior is that the job - A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required. -### jobs..tasks +### jobs.\.tasks **`Type: Sequence`** @@ -2939,19 +2943,19 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `clean_rooms_notebook_task` - Map - - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobs..tasks.clean_rooms_notebook_task). + - The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook when the `clean_rooms_notebook_task` field is present. See [_](#jobsnametasksclean_rooms_notebook_task). * - `condition_task` - Map - - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobs..tasks.condition_task). + - The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and does not support retries or notifications. See [_](#jobsnametaskscondition_task). * - `dbt_task` - Map - - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobs..tasks.dbt_task). + - The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse. See [_](#jobsnametasksdbt_task). * - `depends_on` - Sequence - - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobs..tasks.depends_on). + - An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true. The key is `task_key`, and the value is the name assigned to the dependent task. See [_](#jobsnametasksdepends_on). * - `description` - String @@ -2963,7 +2967,7 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `email_notifications` - Map - - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobs..tasks.email_notifications). + - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [_](#jobsnametasksemail_notifications). * - `environment_key` - String @@ -2975,11 +2979,11 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `for_each_task` - Map - - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobs..tasks.for_each_task). + - The task executes a nested task for every input provided when the `for_each_task` field is present. See [_](#jobsnametasksfor_each_task). * - `health` - Map - - An optional set of health rules that can be defined for this job. See [_](#jobs..tasks.health). + - An optional set of health rules that can be defined for this job. See [_](#jobsnametaskshealth). * - `job_cluster_key` - String @@ -2987,7 +2991,7 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `libraries` - Sequence - - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobs..tasks.libraries). + - An optional list of libraries to be installed on the cluster. The default value is an empty list. See [_](#jobsnametaskslibraries). * - `max_retries` - Integer @@ -2999,23 +3003,23 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `new_cluster` - Map - - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobs..tasks.new_cluster). + - If new_cluster, a description of a new cluster that is created for each run. See [_](#jobsnametasksnew_cluster). * - `notebook_task` - Map - - The task runs a notebook when the `notebook_task` field is present. See [_](#jobs..tasks.notebook_task). + - The task runs a notebook when the `notebook_task` field is present. See [_](#jobsnametasksnotebook_task). * - `notification_settings` - Map - - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobs..tasks.notification_settings). + - Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task. See [_](#jobsnametasksnotification_settings). * - `pipeline_task` - Map - - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobs..tasks.pipeline_task). + - The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported. See [_](#jobsnametaskspipeline_task). * - `python_wheel_task` - Map - - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobs..tasks.python_wheel_task). + - The task runs a Python wheel when the `python_wheel_task` field is present. See [_](#jobsnametaskspython_wheel_task). * - `retry_on_timeout` - Boolean @@ -3027,23 +3031,23 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `run_job_task` - Map - - The task triggers another job when the `run_job_task` field is present. See [_](#jobs..tasks.run_job_task). + - The task triggers another job when the `run_job_task` field is present. See [_](#jobsnametasksrun_job_task). * - `spark_jar_task` - Map - - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobs..tasks.spark_jar_task). + - The task runs a JAR when the `spark_jar_task` field is present. See [_](#jobsnametasksspark_jar_task). * - `spark_python_task` - Map - - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobs..tasks.spark_python_task). + - The task runs a Python file when the `spark_python_task` field is present. See [_](#jobsnametasksspark_python_task). * - `spark_submit_task` - Map - - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute. In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations. `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters. By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage. The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobs..tasks.spark_submit_task). + - (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute. In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations. `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters. By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage. The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. See [_](#jobsnametasksspark_submit_task). * - `sql_task` - Map - - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobs..tasks.sql_task). + - The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present. See [_](#jobsnametaskssql_task). * - `task_key` - String @@ -3055,10 +3059,10 @@ If more than 100 tasks are available, you can paginate through them using :metho * - `webhook_notifications` - Map - - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobs..tasks.webhook_notifications). + - A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications. See [_](#jobsnametaskswebhook_notifications). -### jobs..tasks.clean_rooms_notebook_task +### jobs.\.tasks.clean_rooms_notebook_task **`Type: Map`** @@ -3091,7 +3095,7 @@ when the `clean_rooms_notebook_task` field is present. - Name of the notebook being run. -### jobs..tasks.condition_task +### jobs.\.tasks.condition_task **`Type: Map`** @@ -3120,7 +3124,7 @@ The condition task does not require a cluster to execute and does not support re - The right operand of the condition task. Can be either a string value or a job state or parameter reference. -### jobs..tasks.dbt_task +### jobs.\.tasks.dbt_task **`Type: Map`** @@ -3164,7 +3168,7 @@ The task runs one or more dbt commands when the `dbt_task` field is present. The - ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument. -### jobs..tasks.depends_on +### jobs.\.tasks.depends_on **`Type: Sequence`** @@ -3189,7 +3193,7 @@ The key is `task_key`, and the value is the name assigned to the dependent task. - The name of the task this task depends on. -### jobs..tasks.email_notifications +### jobs.\.tasks.email_notifications **`Type: Map`** @@ -3229,7 +3233,7 @@ An optional set of email addresses that is notified when runs of this task begin - A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent. -### jobs..tasks.for_each_task +### jobs.\.tasks.for_each_task **`Type: Map`** @@ -3257,7 +3261,7 @@ The task executes a nested task for every input provided when the `for_each_task - Configuration for the task that will be run for each element in the array -### jobs..tasks.health +### jobs.\.tasks.health **`Type: Map`** @@ -3274,10 +3278,10 @@ An optional set of health rules that can be defined for this job. * - `rules` - Sequence - - See [_](#jobs..tasks.health.rules). + - See [_](#jobsnametaskshealthrules). -### jobs..tasks.health.rules +### jobs.\.tasks.health.rules **`Type: Sequence`** @@ -3305,7 +3309,7 @@ An optional set of health rules that can be defined for this job. - Specifies the threshold value that the health metric should obey to satisfy the health rule. -### jobs..tasks.libraries +### jobs.\.tasks.libraries **`Type: Sequence`** @@ -3323,7 +3327,7 @@ The default value is an empty list. * - `cran` - Map - - Specification of a CRAN library to be installed as part of the library. See [_](#jobs..tasks.libraries.cran). + - Specification of a CRAN library to be installed as part of the library. See [_](#jobsnametaskslibrariescran). * - `egg` - String @@ -3335,11 +3339,11 @@ The default value is an empty list. * - `maven` - Map - - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobs..tasks.libraries.maven). + - Specification of a maven library to be installed. For example: `{ "coordinates": "org.jsoup:jsoup:1.7.2" }`. See [_](#jobsnametaskslibrariesmaven). * - `pypi` - Map - - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobs..tasks.libraries.pypi). + - Specification of a PyPi library to be installed. For example: `{ "package": "simplejson" }`. See [_](#jobsnametaskslibrariespypi). * - `requirements` - String @@ -3350,7 +3354,7 @@ The default value is an empty list. - URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs. For example: `{ "whl": "/Workspace/path/to/library.whl" }`, `{ "whl" : "/Volumes/path/to/library.whl" }` or `{ "whl": "s3://my-bucket/library.whl" }`. If S3 is used, please make sure the cluster has read access on the library. You may need to launch the cluster with an IAM role to access the S3 URI. -### jobs..tasks.libraries.cran +### jobs.\.tasks.libraries.cran **`Type: Map`** @@ -3374,7 +3378,7 @@ Specification of a CRAN library to be installed as part of the library - The repository where the package can be found. If not specified, the default CRAN repo is used. -### jobs..tasks.libraries.maven +### jobs.\.tasks.libraries.maven **`Type: Map`** @@ -3403,7 +3407,7 @@ Specification of a maven library to be installed. For example: - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched. -### jobs..tasks.libraries.pypi +### jobs.\.tasks.libraries.pypi **`Type: Map`** @@ -3428,7 +3432,7 @@ Specification of a PyPi library to be installed. For example: - The repository where the package can be found. If not specified, the default pip index is used. -### jobs..tasks.new_cluster +### jobs.\.tasks.new_cluster **`Type: Map`** @@ -3449,7 +3453,7 @@ If new_cluster, a description of a new cluster that is created for each run. * - `autoscale` - Map - - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobs..tasks.new_cluster.autoscale). + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#jobsnametasksnew_clusterautoscale). * - `autotermination_minutes` - Integer @@ -3457,15 +3461,15 @@ If new_cluster, a description of a new cluster that is created for each run. * - `aws_attributes` - Map - - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.aws_attributes). + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clusteraws_attributes). * - `azure_attributes` - Map - - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.azure_attributes). + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clusterazure_attributes). * - `cluster_log_conf` - Map - - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobs..tasks.new_cluster.cluster_log_conf). + - The configuration for delivering spark logs to a long-term storage destination. Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. See [_](#jobsnametasksnew_clustercluster_log_conf). * - `cluster_name` - String @@ -3481,7 +3485,7 @@ If new_cluster, a description of a new cluster that is created for each run. * - `docker_image` - Map - - See [_](#jobs..tasks.new_cluster.docker_image). + - See [_](#jobsnametasksnew_clusterdocker_image). * - `driver_instance_pool_id` - String @@ -3501,11 +3505,11 @@ If new_cluster, a description of a new cluster that is created for each run. * - `gcp_attributes` - Map - - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobs..tasks.new_cluster.gcp_attributes). + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#jobsnametasksnew_clustergcp_attributes). * - `init_scripts` - Sequence - - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobs..tasks.new_cluster.init_scripts). + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#jobsnametasksnew_clusterinit_scripts). * - `instance_pool_id` - String @@ -3561,10 +3565,10 @@ If new_cluster, a description of a new cluster that is created for each run. * - `workload_type` - Map - - See [_](#jobs..tasks.new_cluster.workload_type). + - See [_](#jobsnametasksnew_clusterworkload_type). -### jobs..tasks.new_cluster.autoscale +### jobs.\.tasks.new_cluster.autoscale **`Type: Map`** @@ -3589,7 +3593,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later. - The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation. -### jobs..tasks.new_cluster.aws_attributes +### jobs.\.tasks.new_cluster.aws_attributes **`Type: Map`** @@ -3646,7 +3650,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. -### jobs..tasks.new_cluster.azure_attributes +### jobs.\.tasks.new_cluster.azure_attributes **`Type: Map`** @@ -3672,14 +3676,14 @@ If not specified at cluster creation, a set of default values will be used. * - `log_analytics_info` - Map - - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobs..tasks.new_cluster.azure_attributes.log_analytics_info). + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#jobsnametasksnew_clusterazure_attributeslog_analytics_info). * - `spot_bid_max_price` - Any - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. -### jobs..tasks.new_cluster.azure_attributes.log_analytics_info +### jobs.\.tasks.new_cluster.azure_attributes.log_analytics_info **`Type: Map`** @@ -3703,7 +3707,7 @@ Defines values necessary to configure and run Azure Log Analytics agent - -### jobs..tasks.new_cluster.cluster_log_conf +### jobs.\.tasks.new_cluster.cluster_log_conf **`Type: Map`** @@ -3724,14 +3728,14 @@ the destination of executor logs is `$destination/$clusterId/executor`. * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..tasks.new_cluster.cluster_log_conf.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnametasksnew_clustercluster_log_confdbfs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..tasks.new_cluster.cluster_log_conf.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnametasksnew_clustercluster_log_confs3). -### jobs..tasks.new_cluster.cluster_log_conf.dbfs +### jobs.\.tasks.new_cluster.cluster_log_conf.dbfs **`Type: Map`** @@ -3752,7 +3756,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### jobs..tasks.new_cluster.cluster_log_conf.s3 +### jobs.\.tasks.new_cluster.cluster_log_conf.s3 **`Type: Map`** @@ -3799,7 +3803,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### jobs..tasks.new_cluster.docker_image +### jobs.\.tasks.new_cluster.docker_image **`Type: Map`** @@ -3816,14 +3820,14 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in * - `basic_auth` - Map - - See [_](#jobs..tasks.new_cluster.docker_image.basic_auth). + - See [_](#jobsnametasksnew_clusterdocker_imagebasic_auth). * - `url` - String - URL of the docker image. -### jobs..tasks.new_cluster.docker_image.basic_auth +### jobs.\.tasks.new_cluster.docker_image.basic_auth **`Type: Map`** @@ -3847,7 +3851,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - Name of the user -### jobs..tasks.new_cluster.gcp_attributes +### jobs.\.tasks.new_cluster.gcp_attributes **`Type: Map`** @@ -3888,7 +3892,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. -### jobs..tasks.new_cluster.init_scripts +### jobs.\.tasks.new_cluster.init_scripts **`Type: Sequence`** @@ -3905,34 +3909,34 @@ The configuration for storing init scripts. Any number of destinations can be sp * - `abfss` - Map - - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobs..tasks.new_cluster.init_scripts.abfss). + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#jobsnametasksnew_clusterinit_scriptsabfss). * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsdbfs). * - `file` - Map - - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.file). + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsfile). * - `gcs` - Map - - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.gcs). + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsgcs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobs..tasks.new_cluster.init_scripts.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#jobsnametasksnew_clusterinit_scriptss3). * - `volumes` - Map - - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.volumes). + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsvolumes). * - `workspace` - Map - - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobs..tasks.new_cluster.init_scripts.workspace). + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#jobsnametasksnew_clusterinit_scriptsworkspace). -### jobs..tasks.new_cluster.init_scripts.abfss +### jobs.\.tasks.new_cluster.init_scripts.abfss **`Type: Map`** @@ -3953,7 +3957,7 @@ destination needs to be provided. e.g. - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. -### jobs..tasks.new_cluster.init_scripts.dbfs +### jobs.\.tasks.new_cluster.init_scripts.dbfs **`Type: Map`** @@ -3974,7 +3978,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### jobs..tasks.new_cluster.init_scripts.file +### jobs.\.tasks.new_cluster.init_scripts.file **`Type: Map`** @@ -3995,7 +3999,7 @@ destination needs to be provided. e.g. - local file destination, e.g. `file:/my/local/file.sh` -### jobs..tasks.new_cluster.init_scripts.gcs +### jobs.\.tasks.new_cluster.init_scripts.gcs **`Type: Map`** @@ -4016,7 +4020,7 @@ destination needs to be provided. e.g. - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` -### jobs..tasks.new_cluster.init_scripts.s3 +### jobs.\.tasks.new_cluster.init_scripts.s3 **`Type: Map`** @@ -4063,7 +4067,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### jobs..tasks.new_cluster.init_scripts.volumes +### jobs.\.tasks.new_cluster.init_scripts.volumes **`Type: Map`** @@ -4084,7 +4088,7 @@ destination needs to be provided. e.g. - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` -### jobs..tasks.new_cluster.init_scripts.workspace +### jobs.\.tasks.new_cluster.init_scripts.workspace **`Type: Map`** @@ -4105,7 +4109,7 @@ destination needs to be provided. e.g. - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` -### jobs..tasks.new_cluster.workload_type +### jobs.\.tasks.new_cluster.workload_type **`Type: Map`** @@ -4122,10 +4126,10 @@ destination needs to be provided. e.g. * - `clients` - Map - - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobs..tasks.new_cluster.workload_type.clients). + - defined what type of clients can use the cluster. E.g. Notebooks, Jobs. See [_](#jobsnametasksnew_clusterworkload_typeclients). -### jobs..tasks.new_cluster.workload_type.clients +### jobs.\.tasks.new_cluster.workload_type.clients **`Type: Map`** @@ -4149,7 +4153,7 @@ destination needs to be provided. e.g. - With notebooks set, this cluster can be used for notebooks -### jobs..tasks.notebook_task +### jobs.\.tasks.notebook_task **`Type: Map`** @@ -4181,7 +4185,7 @@ The task runs a notebook when the `notebook_task` field is present. - Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses. Note that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail. -### jobs..tasks.notification_settings +### jobs.\.tasks.notification_settings **`Type: Map`** @@ -4209,7 +4213,7 @@ Optional notification settings that are used when sending notifications to each - If true, do not send notifications to recipients specified in `on_failure` if the run is skipped. -### jobs..tasks.pipeline_task +### jobs.\.tasks.pipeline_task **`Type: Map`** @@ -4233,7 +4237,7 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - The full name of the pipeline task to execute. -### jobs..tasks.python_wheel_task +### jobs.\.tasks.python_wheel_task **`Type: Map`** @@ -4265,7 +4269,7 @@ The task runs a Python wheel when the `python_wheel_task` field is present. - Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null. -### jobs..tasks.run_job_task +### jobs.\.tasks.run_job_task **`Type: Map`** @@ -4302,7 +4306,7 @@ The task triggers another job when the `run_job_task` field is present. * - `pipeline_params` - Map - - Controls whether the pipeline should perform a full refresh. See [_](#jobs..tasks.run_job_task.pipeline_params). + - Controls whether the pipeline should perform a full refresh. See [_](#jobsnametasksrun_job_taskpipeline_params). * - `python_named_params` - Map @@ -4321,7 +4325,7 @@ The task triggers another job when the `run_job_task` field is present. - A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. -### jobs..tasks.run_job_task.pipeline_params +### jobs.\.tasks.run_job_task.pipeline_params **`Type: Map`** @@ -4341,7 +4345,7 @@ Controls whether the pipeline should perform a full refresh - If true, triggers a full refresh on the delta live table. -### jobs..tasks.spark_jar_task +### jobs.\.tasks.spark_jar_task **`Type: Map`** @@ -4373,7 +4377,7 @@ The task runs a JAR when the `spark_jar_task` field is present. - Deprecated. A value of `false` is no longer supported. -### jobs..tasks.spark_python_task +### jobs.\.tasks.spark_python_task **`Type: Map`** @@ -4401,7 +4405,7 @@ The task runs a Python file when the `spark_python_task` field is present. - Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local Databricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`, the Python file will be retrieved from a Git repository defined in `git_source`. * `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI. * `GIT`: The Python file is located in a remote Git repository. -### jobs..tasks.spark_submit_task +### jobs.\.tasks.spark_submit_task **`Type: Map`** @@ -4429,7 +4433,7 @@ The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths. - Command-line parameters passed to spark submit. Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs. -### jobs..tasks.sql_task +### jobs.\.tasks.sql_task **`Type: Map`** @@ -4446,15 +4450,15 @@ The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL d * - `alert` - Map - - If alert, indicates that this job must refresh a SQL alert. See [_](#jobs..tasks.sql_task.alert). + - If alert, indicates that this job must refresh a SQL alert. See [_](#jobsnametaskssql_taskalert). * - `dashboard` - Map - - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobs..tasks.sql_task.dashboard). + - If dashboard, indicates that this job must refresh a SQL dashboard. See [_](#jobsnametaskssql_taskdashboard). * - `file` - Map - - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobs..tasks.sql_task.file). + - If file, indicates that this job runs a SQL file in a remote Git repository. See [_](#jobsnametaskssql_taskfile). * - `parameters` - Map @@ -4462,14 +4466,14 @@ The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL d * - `query` - Map - - If query, indicates that this job must execute a SQL query. See [_](#jobs..tasks.sql_task.query). + - If query, indicates that this job must execute a SQL query. See [_](#jobsnametaskssql_taskquery). * - `warehouse_id` - String - The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs. -### jobs..tasks.sql_task.alert +### jobs.\.tasks.sql_task.alert **`Type: Map`** @@ -4494,10 +4498,10 @@ If alert, indicates that this job must refresh a SQL alert. * - `subscriptions` - Sequence - - If specified, alert notifications are sent to subscribers. See [_](#jobs..tasks.sql_task.alert.subscriptions). + - If specified, alert notifications are sent to subscribers. See [_](#jobsnametaskssql_taskalertsubscriptions). -### jobs..tasks.sql_task.alert.subscriptions +### jobs.\.tasks.sql_task.alert.subscriptions **`Type: Sequence`** @@ -4521,7 +4525,7 @@ If specified, alert notifications are sent to subscribers. - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications. -### jobs..tasks.sql_task.dashboard +### jobs.\.tasks.sql_task.dashboard **`Type: Map`** @@ -4550,10 +4554,10 @@ If dashboard, indicates that this job must refresh a SQL dashboard. * - `subscriptions` - Sequence - - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobs..tasks.sql_task.dashboard.subscriptions). + - If specified, dashboard snapshots are sent to subscriptions. See [_](#jobsnametaskssql_taskdashboardsubscriptions). -### jobs..tasks.sql_task.dashboard.subscriptions +### jobs.\.tasks.sql_task.dashboard.subscriptions **`Type: Sequence`** @@ -4577,7 +4581,7 @@ If specified, dashboard snapshots are sent to subscriptions. - The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications. -### jobs..tasks.sql_task.file +### jobs.\.tasks.sql_task.file **`Type: Map`** @@ -4601,7 +4605,7 @@ If file, indicates that this job runs a SQL file in a remote Git repository. - Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved from the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. * `WORKSPACE`: SQL file is located in Databricks workspace. * `GIT`: SQL file is located in cloud Git provider. -### jobs..tasks.sql_task.query +### jobs.\.tasks.sql_task.query **`Type: Map`** @@ -4621,7 +4625,7 @@ If query, indicates that this job must execute a SQL query. - The canonical identifier of the SQL query. -### jobs..tasks.webhook_notifications +### jobs.\.tasks.webhook_notifications **`Type: Map`** @@ -4638,26 +4642,26 @@ A collection of system notification IDs to notify when runs of this task begin o * - `on_duration_warning_threshold_exceeded` - Sequence - - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs..tasks.webhook_notifications.on_duration_warning_threshold_exceeded). + - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobsnametaskswebhook_notificationson_duration_warning_threshold_exceeded). * - `on_failure` - Sequence - - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs..tasks.webhook_notifications.on_failure). + - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobsnametaskswebhook_notificationson_failure). * - `on_start` - Sequence - - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs..tasks.webhook_notifications.on_start). + - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobsnametaskswebhook_notificationson_start). * - `on_streaming_backlog_exceeded` - Sequence - - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs..tasks.webhook_notifications.on_streaming_backlog_exceeded). + - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobsnametaskswebhook_notificationson_streaming_backlog_exceeded). * - `on_success` - Sequence - - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs..tasks.webhook_notifications.on_success). + - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobsnametaskswebhook_notificationson_success). -### jobs..tasks.webhook_notifications.on_duration_warning_threshold_exceeded +### jobs.\.tasks.webhook_notifications.on_duration_warning_threshold_exceeded **`Type: Sequence`** @@ -4677,7 +4681,7 @@ An optional list of system notification IDs to call when the duration of a run e - -### jobs..tasks.webhook_notifications.on_failure +### jobs.\.tasks.webhook_notifications.on_failure **`Type: Sequence`** @@ -4697,7 +4701,7 @@ An optional list of system notification IDs to call when the run fails. A maximu - -### jobs..tasks.webhook_notifications.on_start +### jobs.\.tasks.webhook_notifications.on_start **`Type: Sequence`** @@ -4717,7 +4721,7 @@ An optional list of system notification IDs to call when the run starts. A maxim - -### jobs..tasks.webhook_notifications.on_streaming_backlog_exceeded +### jobs.\.tasks.webhook_notifications.on_streaming_backlog_exceeded **`Type: Sequence`** @@ -4740,7 +4744,7 @@ A maximum of 3 destinations can be specified for the `on_streaming_backlog_excee - -### jobs..tasks.webhook_notifications.on_success +### jobs.\.tasks.webhook_notifications.on_success **`Type: Sequence`** @@ -4760,7 +4764,7 @@ An optional list of system notification IDs to call when the run completes succe - -### jobs..trigger +### jobs.\.trigger **`Type: Map`** @@ -4777,7 +4781,7 @@ A configuration to trigger a run when certain conditions are met. The default be * - `file_arrival` - Map - - File arrival trigger settings. See [_](#jobs..trigger.file_arrival). + - File arrival trigger settings. See [_](#jobsnametriggerfile_arrival). * - `pause_status` - String @@ -4785,18 +4789,18 @@ A configuration to trigger a run when certain conditions are met. The default be * - `periodic` - Map - - Periodic trigger settings. See [_](#jobs..trigger.periodic). + - Periodic trigger settings. See [_](#jobsnametriggerperiodic). * - `table` - Map - - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobs..trigger.table). + - Old table trigger settings name. Deprecated in favor of `table_update`. See [_](#jobsnametriggertable). * - `table_update` - Map - - See [_](#jobs..trigger.table_update). + - See [_](#jobsnametriggertable_update). -### jobs..trigger.file_arrival +### jobs.\.trigger.file_arrival **`Type: Map`** @@ -4824,7 +4828,7 @@ File arrival trigger settings. - If set, the trigger starts a run only after no file activity has occurred for the specified amount of time. This makes it possible to wait for a batch of incoming files to arrive before triggering a run. The minimum allowed value is 60 seconds. -### jobs..trigger.periodic +### jobs.\.trigger.periodic **`Type: Map`** @@ -4848,7 +4852,7 @@ Periodic trigger settings. - The unit of time for the interval. -### jobs..trigger.table +### jobs.\.trigger.table **`Type: Map`** @@ -4880,7 +4884,7 @@ Old table trigger settings name. Deprecated in favor of `table_update`. - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. -### jobs..trigger.table_update +### jobs.\.trigger.table_update **`Type: Map`** @@ -4912,7 +4916,7 @@ Old table trigger settings name. Deprecated in favor of `table_update`. - If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum allowed value is 60 seconds. -### jobs..webhook_notifications +### jobs.\.webhook_notifications **`Type: Map`** @@ -4929,26 +4933,26 @@ A collection of system notification IDs to notify when runs of this job begin or * - `on_duration_warning_threshold_exceeded` - Sequence - - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobs..webhook_notifications.on_duration_warning_threshold_exceeded). + - An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property. See [_](#jobsnamewebhook_notificationson_duration_warning_threshold_exceeded). * - `on_failure` - Sequence - - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobs..webhook_notifications.on_failure). + - An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property. See [_](#jobsnamewebhook_notificationson_failure). * - `on_start` - Sequence - - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobs..webhook_notifications.on_start). + - An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property. See [_](#jobsnamewebhook_notificationson_start). * - `on_streaming_backlog_exceeded` - Sequence - - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobs..webhook_notifications.on_streaming_backlog_exceeded). + - An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes. A maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property. See [_](#jobsnamewebhook_notificationson_streaming_backlog_exceeded). * - `on_success` - Sequence - - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobs..webhook_notifications.on_success). + - An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. See [_](#jobsnamewebhook_notificationson_success). -### jobs..webhook_notifications.on_duration_warning_threshold_exceeded +### jobs.\.webhook_notifications.on_duration_warning_threshold_exceeded **`Type: Sequence`** @@ -4968,7 +4972,7 @@ An optional list of system notification IDs to call when the duration of a run e - -### jobs..webhook_notifications.on_failure +### jobs.\.webhook_notifications.on_failure **`Type: Sequence`** @@ -4988,7 +4992,7 @@ An optional list of system notification IDs to call when the run fails. A maximu - -### jobs..webhook_notifications.on_start +### jobs.\.webhook_notifications.on_start **`Type: Sequence`** @@ -5008,7 +5012,7 @@ An optional list of system notification IDs to call when the run starts. A maxim - -### jobs..webhook_notifications.on_streaming_backlog_exceeded +### jobs.\.webhook_notifications.on_streaming_backlog_exceeded **`Type: Sequence`** @@ -5031,7 +5035,7 @@ A maximum of 3 destinations can be specified for the `on_streaming_backlog_excee - -### jobs..webhook_notifications.on_success +### jobs.\.webhook_notifications.on_success **`Type: Sequence`** @@ -5073,11 +5077,11 @@ model_serving_endpoints: * - `ai_gateway` - Map - - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpoints..ai_gateway). + - The AI Gateway configuration for the serving endpoint. NOTE: Only external model and provisioned throughput endpoints are currently supported. See [_](#model_serving_endpointsnameai_gateway). * - `config` - Map - - The core config of the serving endpoint. See [_](#model_serving_endpoints..config). + - The core config of the serving endpoint. See [_](#model_serving_endpointsnameconfig). * - `name` - String @@ -5085,11 +5089,11 @@ model_serving_endpoints: * - `permissions` - Sequence - - See [_](#model_serving_endpoints..permissions). + - See [_](#model_serving_endpointsnamepermissions). * - `rate_limits` - Sequence - - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpoints..rate_limits). + - Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits. See [_](#model_serving_endpointsnamerate_limits). * - `route_optimized` - Boolean @@ -5097,7 +5101,7 @@ model_serving_endpoints: * - `tags` - Sequence - - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpoints..tags). + - Tags to be attached to the serving endpoint and automatically propagated to billing logs. See [_](#model_serving_endpointsnametags). **Example** @@ -5124,7 +5128,7 @@ resources: value: "data science" ``` -### model_serving_endpoints..ai_gateway +### model_serving_endpoints.\.ai_gateway **`Type: Map`** @@ -5141,22 +5145,22 @@ The AI Gateway configuration for the serving endpoint. NOTE: Only external model * - `guardrails` - Map - - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpoints..ai_gateway.guardrails). + - Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses. See [_](#model_serving_endpointsnameai_gatewayguardrails). * - `inference_table_config` - Map - - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpoints..ai_gateway.inference_table_config). + - Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality. See [_](#model_serving_endpointsnameai_gatewayinference_table_config). * - `rate_limits` - Sequence - - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpoints..ai_gateway.rate_limits). + - Configuration for rate limits which can be set to limit endpoint traffic. See [_](#model_serving_endpointsnameai_gatewayrate_limits). * - `usage_tracking_config` - Map - - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpoints..ai_gateway.usage_tracking_config). + - Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs. See [_](#model_serving_endpointsnameai_gatewayusage_tracking_config). -### model_serving_endpoints..ai_gateway.guardrails +### model_serving_endpoints.\.ai_gateway.guardrails **`Type: Map`** @@ -5173,14 +5177,14 @@ Configuration for AI Guardrails to prevent unwanted data and unsafe data in requ * - `input` - Map - - Configuration for input guardrail filters. See [_](#model_serving_endpoints..ai_gateway.guardrails.input). + - Configuration for input guardrail filters. See [_](#model_serving_endpointsnameai_gatewayguardrailsinput). * - `output` - Map - - Configuration for output guardrail filters. See [_](#model_serving_endpoints..ai_gateway.guardrails.output). + - Configuration for output guardrail filters. See [_](#model_serving_endpointsnameai_gatewayguardrailsoutput). -### model_serving_endpoints..ai_gateway.guardrails.input +### model_serving_endpoints.\.ai_gateway.guardrails.input **`Type: Map`** @@ -5201,7 +5205,7 @@ Configuration for input guardrail filters. * - `pii` - Map - - Configuration for guardrail PII filter. See [_](#model_serving_endpoints..ai_gateway.guardrails.input.pii). + - Configuration for guardrail PII filter. See [_](#model_serving_endpointsnameai_gatewayguardrailsinputpii). * - `safety` - Boolean @@ -5212,7 +5216,7 @@ Configuration for input guardrail filters. - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. -### model_serving_endpoints..ai_gateway.guardrails.input.pii +### model_serving_endpoints.\.ai_gateway.guardrails.input.pii **`Type: Map`** @@ -5232,7 +5236,7 @@ Configuration for guardrail PII filter. - Configuration for input guardrail filters. -### model_serving_endpoints..ai_gateway.guardrails.output +### model_serving_endpoints.\.ai_gateway.guardrails.output **`Type: Map`** @@ -5253,7 +5257,7 @@ Configuration for output guardrail filters. * - `pii` - Map - - Configuration for guardrail PII filter. See [_](#model_serving_endpoints..ai_gateway.guardrails.output.pii). + - Configuration for guardrail PII filter. See [_](#model_serving_endpointsnameai_gatewayguardrailsoutputpii). * - `safety` - Boolean @@ -5264,7 +5268,7 @@ Configuration for output guardrail filters. - The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics. -### model_serving_endpoints..ai_gateway.guardrails.output.pii +### model_serving_endpoints.\.ai_gateway.guardrails.output.pii **`Type: Map`** @@ -5284,7 +5288,7 @@ Configuration for guardrail PII filter. - Configuration for input guardrail filters. -### model_serving_endpoints..ai_gateway.inference_table_config +### model_serving_endpoints.\.ai_gateway.inference_table_config **`Type: Map`** @@ -5317,7 +5321,7 @@ Use these tables to monitor and audit data being sent to and received from model - The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name. -### model_serving_endpoints..ai_gateway.rate_limits +### model_serving_endpoints.\.ai_gateway.rate_limits **`Type: Sequence`** @@ -5345,7 +5349,7 @@ Configuration for rate limits which can be set to limit endpoint traffic. - Renewal period field for a rate limit. Currently, only 'minute' is supported. -### model_serving_endpoints..ai_gateway.usage_tracking_config +### model_serving_endpoints.\.ai_gateway.usage_tracking_config **`Type: Map`** @@ -5366,7 +5370,7 @@ These tables allow you to monitor operational usage on endpoints and their assoc - Whether to enable usage tracking. -### model_serving_endpoints..config +### model_serving_endpoints.\.config **`Type: Map`** @@ -5383,22 +5387,22 @@ The core config of the serving endpoint. * - `auto_capture_config` - Map - - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpoints..config.auto_capture_config). + - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [_](#model_serving_endpointsnameconfigauto_capture_config). * - `served_entities` - Sequence - - The list of served entities under the serving endpoint config. See [_](#model_serving_endpoints..config.served_entities). + - The list of served entities under the serving endpoint config. See [_](#model_serving_endpointsnameconfigserved_entities). * - `served_models` - Sequence - - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpoints..config.served_models). + - (Deprecated, use served_entities instead) The list of served models under the serving endpoint config. See [_](#model_serving_endpointsnameconfigserved_models). * - `traffic_config` - Map - - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpoints..config.traffic_config). + - The traffic configuration associated with the serving endpoint config. See [_](#model_serving_endpointsnameconfigtraffic_config). -### model_serving_endpoints..config.auto_capture_config +### model_serving_endpoints.\.config.auto_capture_config **`Type: Map`** @@ -5433,7 +5437,7 @@ in these cases please use AI Gateway to manage inference tables. - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. -### model_serving_endpoints..config.served_entities +### model_serving_endpoints.\.config.served_entities **`Type: Sequence`** @@ -5462,7 +5466,7 @@ The list of served entities under the serving endpoint config. * - `external_model` - Map - - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpoints..config.served_entities.external_model). + - The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_model). * - `instance_profile_arn` - String @@ -5493,7 +5497,7 @@ The list of served entities under the serving endpoint config. - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). -### model_serving_endpoints..config.served_entities.external_model +### model_serving_endpoints.\.config.served_entities.external_model **`Type: Map`** @@ -5510,27 +5514,27 @@ The external model to be served. NOTE: Only one of external_model and (entity_na * - `ai21labs_config` - Map - - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpoints..config.served_entities.external_model.ai21labs_config). + - AI21Labs Config. Only required if the provider is 'ai21labs'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelai21labs_config). * - `amazon_bedrock_config` - Map - - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpoints..config.served_entities.external_model.amazon_bedrock_config). + - Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelamazon_bedrock_config). * - `anthropic_config` - Map - - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpoints..config.served_entities.external_model.anthropic_config). + - Anthropic Config. Only required if the provider is 'anthropic'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelanthropic_config). * - `cohere_config` - Map - - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpoints..config.served_entities.external_model.cohere_config). + - Cohere Config. Only required if the provider is 'cohere'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelcohere_config). * - `databricks_model_serving_config` - Map - - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpoints..config.served_entities.external_model.databricks_model_serving_config). + - Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modeldatabricks_model_serving_config). * - `google_cloud_vertex_ai_config` - Map - - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpoints..config.served_entities.external_model.google_cloud_vertex_ai_config). + - Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelgoogle_cloud_vertex_ai_config). * - `name` - String @@ -5538,11 +5542,11 @@ The external model to be served. NOTE: Only one of external_model and (entity_na * - `openai_config` - Map - - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpoints..config.served_entities.external_model.openai_config). + - OpenAI Config. Only required if the provider is 'openai'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelopenai_config). * - `palm_config` - Map - - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpoints..config.served_entities.external_model.palm_config). + - PaLM Config. Only required if the provider is 'palm'. See [_](#model_serving_endpointsnameconfigserved_entitiesexternal_modelpalm_config). * - `provider` - String @@ -5553,7 +5557,7 @@ The external model to be served. NOTE: Only one of external_model and (entity_na - The task type of the external model. -### model_serving_endpoints..config.served_entities.external_model.ai21labs_config +### model_serving_endpoints.\.config.served_entities.external_model.ai21labs_config **`Type: Map`** @@ -5577,7 +5581,7 @@ AI21Labs Config. Only required if the provider is 'ai21labs'. - An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`. -### model_serving_endpoints..config.served_entities.external_model.amazon_bedrock_config +### model_serving_endpoints.\.config.served_entities.external_model.amazon_bedrock_config **`Type: Map`** @@ -5617,7 +5621,7 @@ Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'. - The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. -### model_serving_endpoints..config.served_entities.external_model.anthropic_config +### model_serving_endpoints.\.config.served_entities.external_model.anthropic_config **`Type: Map`** @@ -5641,7 +5645,7 @@ Anthropic Config. Only required if the provider is 'anthropic'. - The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`. -### model_serving_endpoints..config.served_entities.external_model.cohere_config +### model_serving_endpoints.\.config.served_entities.external_model.cohere_config **`Type: Map`** @@ -5669,7 +5673,7 @@ Cohere Config. Only required if the provider is 'cohere'. - The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`. -### model_serving_endpoints..config.served_entities.external_model.databricks_model_serving_config +### model_serving_endpoints.\.config.served_entities.external_model.databricks_model_serving_config **`Type: Map`** @@ -5697,7 +5701,7 @@ Databricks Model Serving Config. Only required if the provider is 'databricks-mo - The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model. -### model_serving_endpoints..config.served_entities.external_model.google_cloud_vertex_ai_config +### model_serving_endpoints.\.config.served_entities.external_model.google_cloud_vertex_ai_config **`Type: Map`** @@ -5729,7 +5733,7 @@ Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-ve - This is the region for the Google Cloud Vertex AI Service. See [supported regions] for more details. Some models are only available in specific regions. [supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations -### model_serving_endpoints..config.served_entities.external_model.openai_config +### model_serving_endpoints.\.config.served_entities.external_model.openai_config **`Type: Map`** @@ -5789,7 +5793,7 @@ OpenAI Config. Only required if the provider is 'openai'. - This is an optional field to specify the organization in OpenAI or Azure OpenAI. -### model_serving_endpoints..config.served_entities.external_model.palm_config +### model_serving_endpoints.\.config.served_entities.external_model.palm_config **`Type: Map`** @@ -5813,7 +5817,7 @@ PaLM Config. Only required if the provider is 'palm'. - The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`. -### model_serving_endpoints..config.served_models +### model_serving_endpoints.\.config.served_models **`Type: Sequence`** @@ -5869,7 +5873,7 @@ PaLM Config. Only required if the provider is 'palm'. - The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is "CPU". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types). -### model_serving_endpoints..config.traffic_config +### model_serving_endpoints.\.config.traffic_config **`Type: Map`** @@ -5886,10 +5890,10 @@ The traffic configuration associated with the serving endpoint config. * - `routes` - Sequence - - The list of routes that define traffic to each served entity. See [_](#model_serving_endpoints..config.traffic_config.routes). + - The list of routes that define traffic to each served entity. See [_](#model_serving_endpointsnameconfigtraffic_configroutes). -### model_serving_endpoints..config.traffic_config.routes +### model_serving_endpoints.\.config.traffic_config.routes **`Type: Sequence`** @@ -5913,7 +5917,7 @@ The list of routes that define traffic to each served entity. - The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive. -### model_serving_endpoints..permissions +### model_serving_endpoints.\.permissions **`Type: Sequence`** @@ -5945,7 +5949,7 @@ The list of routes that define traffic to each served entity. - The name of the user that has the permission set in level. -### model_serving_endpoints..rate_limits +### model_serving_endpoints.\.rate_limits **`Type: Sequence`** @@ -5973,7 +5977,7 @@ Rate limits to be applied to the serving endpoint. NOTE: this field is deprecate - Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported. -### model_serving_endpoints..tags +### model_serving_endpoints.\.tags **`Type: Sequence`** @@ -6031,7 +6035,7 @@ models: * - `latest_versions` - Sequence - - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#models..latest_versions). + - Collection of latest model versions for each stage. Only contains models with current `READY` status. See [_](#modelsnamelatest_versions). * - `name` - String @@ -6039,18 +6043,18 @@ models: * - `permissions` - Sequence - - See [_](#models..permissions). + - See [_](#modelsnamepermissions). * - `tags` - Sequence - - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#models..tags). + - Tags: Additional metadata key-value pairs for this `registered_model`. See [_](#modelsnametags). * - `user_id` - String - User that created this `registered_model` -### models..latest_versions +### models.\.latest_versions **`Type: Sequence`** @@ -6108,7 +6112,7 @@ Only contains models with current `READY` status. * - `tags` - Sequence - - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#models..latest_versions.tags). + - Tags: Additional metadata key-value pairs for this `model_version`. See [_](#modelsnamelatest_versionstags). * - `user_id` - String @@ -6119,7 +6123,7 @@ Only contains models with current `READY` status. - Model's version number. -### models..latest_versions.tags +### models.\.latest_versions.tags **`Type: Sequence`** @@ -6143,7 +6147,7 @@ Tags: Additional metadata key-value pairs for this `model_version`. - The tag value. -### models..permissions +### models.\.permissions **`Type: Sequence`** @@ -6175,7 +6179,7 @@ Tags: Additional metadata key-value pairs for this `model_version`. - The name of the user that has the permission set in level. -### models..tags +### models.\.tags **`Type: Sequence`** @@ -6219,6 +6223,10 @@ pipelines: - Type - Description + * - `allow_duplicate_names` + - Boolean + - If false, deployment will fail if name conflicts with that of another pipeline. + * - `budget_policy_id` - String - Budget policy of this pipeline. @@ -6233,7 +6241,7 @@ pipelines: * - `clusters` - Sequence - - Cluster settings for this pipeline deployment. See [_](#pipelines..clusters). + - Cluster settings for this pipeline deployment. See [_](#pipelinesnameclusters). * - `configuration` - Map @@ -6245,23 +6253,27 @@ pipelines: * - `deployment` - Map - - Deployment type of this pipeline. See [_](#pipelines..deployment). + - Deployment type of this pipeline. See [_](#pipelinesnamedeployment). * - `development` - Boolean - Whether the pipeline is in Development mode. Defaults to false. + * - `dry_run` + - Boolean + - + * - `edition` - String - Pipeline product edition. * - `filters` - Map - - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelines..filters). + - Filters on which Pipeline packages to include in the deployed graph. See [_](#pipelinesnamefilters). * - `gateway_definition` - Map - - The definition of a gateway pipeline to support change data capture. See [_](#pipelines..gateway_definition). + - The definition of a gateway pipeline to support change data capture. See [_](#pipelinesnamegateway_definition). * - `id` - String @@ -6269,11 +6281,11 @@ pipelines: * - `ingestion_definition` - Map - - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelines..ingestion_definition). + - The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings. See [_](#pipelinesnameingestion_definition). * - `libraries` - Sequence - - Libraries or code needed by this deployment. See [_](#pipelines..libraries). + - Libraries or code needed by this deployment. See [_](#pipelinesnamelibraries). * - `name` - String @@ -6281,11 +6293,11 @@ pipelines: * - `notifications` - Sequence - - List of notification settings for this pipeline. See [_](#pipelines..notifications). + - List of notification settings for this pipeline. See [_](#pipelinesnamenotifications). * - `permissions` - Sequence - - See [_](#pipelines..permissions). + - See [_](#pipelinesnamepermissions). * - `photon` - Boolean @@ -6293,7 +6305,11 @@ pipelines: * - `restart_window` - Map - - Restart window of this pipeline. See [_](#pipelines..restart_window). + - Restart window of this pipeline. See [_](#pipelinesnamerestart_window). + + * - `run_as` + - Map + - Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline. Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. See [_](#pipelinesnamerun_as). * - `schema` - String @@ -6313,7 +6329,7 @@ pipelines: * - `trigger` - Map - - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelines..trigger). + - Which pipeline trigger to use. Deprecated: Use `continuous` instead. See [_](#pipelinesnametrigger). **Example** @@ -6338,7 +6354,7 @@ resources: path: ./pipeline.py ``` -### pipelines..clusters +### pipelines.\.clusters **`Type: Sequence`** @@ -6359,19 +6375,19 @@ Cluster settings for this pipeline deployment. * - `autoscale` - Map - - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelines..clusters.autoscale). + - Parameters needed in order to automatically scale clusters up and down based on load. Note: autoscaling works best with DB runtime versions 3.0 or later. See [_](#pipelinesnameclustersautoscale). * - `aws_attributes` - Map - - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.aws_attributes). + - Attributes related to clusters running on Amazon Web Services. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersaws_attributes). * - `azure_attributes` - Map - - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.azure_attributes). + - Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersazure_attributes). * - `cluster_log_conf` - Map - - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelines..clusters.cluster_log_conf). + - The configuration for delivering spark logs to a long-term storage destination. Only dbfs destinations are supported. Only one destination can be specified for one cluster. If the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is `$destination/$clusterId/executor`. . See [_](#pipelinesnameclusterscluster_log_conf). * - `custom_tags` - Map @@ -6391,11 +6407,11 @@ Cluster settings for this pipeline deployment. * - `gcp_attributes` - Map - - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelines..clusters.gcp_attributes). + - Attributes related to clusters running on Google Cloud Platform. If not specified at cluster creation, a set of default values will be used. See [_](#pipelinesnameclustersgcp_attributes). * - `init_scripts` - Sequence - - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#pipelines..clusters.init_scripts). + - The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`. See [_](#pipelinesnameclustersinit_scripts). * - `instance_pool_id` - String @@ -6430,7 +6446,7 @@ Cluster settings for this pipeline deployment. - SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. -### pipelines..clusters.autoscale +### pipelines.\.clusters.autoscale **`Type: Map`** @@ -6459,7 +6475,7 @@ Note: autoscaling works best with DB runtime versions 3.0 or later. - Databricks Enhanced Autoscaling optimizes cluster utilization by automatically allocating cluster resources based on workload volume, with minimal impact to the data processing latency of your pipelines. Enhanced Autoscaling is available for `updates` clusters only. The legacy autoscaling feature is used for `maintenance` clusters. -### pipelines..clusters.aws_attributes +### pipelines.\.clusters.aws_attributes **`Type: Map`** @@ -6516,7 +6532,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone/datacenter in which the cluster resides. This string will be of a form like "us-west-2a". The provided availability zone must be in the same region as the Databricks deployment. For example, "us-west-2a" is not a valid zone id if the Databricks deployment resides in the "us-east-1" region. This is an optional field at cluster creation, and if not specified, a default zone will be used. If the zone specified is "auto", will try to place cluster in a zone with high availability, and will retry placement in a different AZ if there is not enough capacity. The list of available zones as well as the default value can be found by using the `List Zones` method. -### pipelines..clusters.azure_attributes +### pipelines.\.clusters.azure_attributes **`Type: Map`** @@ -6542,14 +6558,14 @@ If not specified at cluster creation, a set of default values will be used. * - `log_analytics_info` - Map - - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelines..clusters.azure_attributes.log_analytics_info). + - Defines values necessary to configure and run Azure Log Analytics agent. See [_](#pipelinesnameclustersazure_attributeslog_analytics_info). * - `spot_bid_max_price` - Any - The max bid price to be used for Azure spot instances. The Max price for the bid cannot be higher than the on-demand price of the instance. If not specified, the default value is -1, which specifies that the instance cannot be evicted on the basis of price, and only on the basis of availability. Further, the value should > 0 or -1. -### pipelines..clusters.azure_attributes.log_analytics_info +### pipelines.\.clusters.azure_attributes.log_analytics_info **`Type: Map`** @@ -6573,7 +6589,7 @@ Defines values necessary to configure and run Azure Log Analytics agent - -### pipelines..clusters.cluster_log_conf +### pipelines.\.clusters.cluster_log_conf **`Type: Map`** @@ -6595,14 +6611,14 @@ the destination of executor logs is `$destination/$clusterId/executor`. * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines..clusters.cluster_log_conf.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelinesnameclusterscluster_log_confdbfs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines..clusters.cluster_log_conf.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelinesnameclusterscluster_log_confs3). -### pipelines..clusters.cluster_log_conf.dbfs +### pipelines.\.clusters.cluster_log_conf.dbfs **`Type: Map`** @@ -6623,7 +6639,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### pipelines..clusters.cluster_log_conf.s3 +### pipelines.\.clusters.cluster_log_conf.s3 **`Type: Map`** @@ -6670,7 +6686,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### pipelines..clusters.gcp_attributes +### pipelines.\.clusters.gcp_attributes **`Type: Map`** @@ -6711,7 +6727,7 @@ If not specified at cluster creation, a set of default values will be used. - Identifier for the availability zone in which the cluster resides. This can be one of the following: - "HA" => High availability, spread nodes across availability zones for a Databricks deployment region [default] - "AUTO" => Databricks picks an availability zone to schedule the cluster on. - A GCP availability zone => Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones. -### pipelines..clusters.init_scripts +### pipelines.\.clusters.init_scripts **`Type: Sequence`** @@ -6728,34 +6744,34 @@ The configuration for storing init scripts. Any number of destinations can be sp * - `abfss` - Map - - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#pipelines..clusters.init_scripts.abfss). + - destination needs to be provided. e.g. `{ "abfss" : { "destination" : "abfss://@.dfs.core.windows.net/" } }. See [_](#pipelinesnameclustersinit_scriptsabfss). * - `dbfs` - Map - - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelines..clusters.init_scripts.dbfs). + - destination needs to be provided. e.g. `{ "dbfs" : { "destination" : "dbfs:/home/cluster_log" } }`. See [_](#pipelinesnameclustersinit_scriptsdbfs). * - `file` - Map - - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelines..clusters.init_scripts.file). + - destination needs to be provided. e.g. `{ "file" : { "destination" : "file:/my/local/file.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsfile). * - `gcs` - Map - - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelines..clusters.init_scripts.gcs). + - destination needs to be provided. e.g. `{ "gcs": { "destination": "gs://my-bucket/file.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsgcs). * - `s3` - Map - - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelines..clusters.init_scripts.s3). + - destination and either the region or endpoint need to be provided. e.g. `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }` Cluster iam role is used to access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to write data to the s3 destination. See [_](#pipelinesnameclustersinit_scriptss3). * - `volumes` - Map - - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelines..clusters.init_scripts.volumes). + - destination needs to be provided. e.g. `{ "volumes" : { "destination" : "/Volumes/my-init.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsvolumes). * - `workspace` - Map - - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelines..clusters.init_scripts.workspace). + - destination needs to be provided. e.g. `{ "workspace" : { "destination" : "/Users/user1@databricks.com/my-init.sh" } }`. See [_](#pipelinesnameclustersinit_scriptsworkspace). -### pipelines..clusters.init_scripts.abfss +### pipelines.\.clusters.init_scripts.abfss **`Type: Map`** @@ -6776,7 +6792,7 @@ destination needs to be provided. e.g. - abfss destination, e.g. `abfss://@.dfs.core.windows.net/`. -### pipelines..clusters.init_scripts.dbfs +### pipelines.\.clusters.init_scripts.dbfs **`Type: Map`** @@ -6797,7 +6813,7 @@ destination needs to be provided. e.g. - dbfs destination, e.g. `dbfs:/my/path` -### pipelines..clusters.init_scripts.file +### pipelines.\.clusters.init_scripts.file **`Type: Map`** @@ -6818,7 +6834,7 @@ destination needs to be provided. e.g. - local file destination, e.g. `file:/my/local/file.sh` -### pipelines..clusters.init_scripts.gcs +### pipelines.\.clusters.init_scripts.gcs **`Type: Map`** @@ -6839,7 +6855,7 @@ destination needs to be provided. e.g. - GCS destination/URI, e.g. `gs://my-bucket/some-prefix` -### pipelines..clusters.init_scripts.s3 +### pipelines.\.clusters.init_scripts.s3 **`Type: Map`** @@ -6886,7 +6902,7 @@ Cluster iam role is used to access s3, please make sure the cluster iam role in - S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set, endpoint will be used. -### pipelines..clusters.init_scripts.volumes +### pipelines.\.clusters.init_scripts.volumes **`Type: Map`** @@ -6907,7 +6923,7 @@ destination needs to be provided. e.g. - Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh` -### pipelines..clusters.init_scripts.workspace +### pipelines.\.clusters.init_scripts.workspace **`Type: Map`** @@ -6928,7 +6944,7 @@ destination needs to be provided. e.g. - workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh` -### pipelines..deployment +### pipelines.\.deployment **`Type: Map`** @@ -6952,7 +6968,7 @@ Deployment type of this pipeline. - The path to the file containing metadata about the deployment. -### pipelines..filters +### pipelines.\.filters **`Type: Map`** @@ -6976,7 +6992,7 @@ Filters on which Pipeline packages to include in the deployed graph. - Paths to include. -### pipelines..gateway_definition +### pipelines.\.gateway_definition **`Type: Map`** @@ -7012,7 +7028,7 @@ The definition of a gateway pipeline to support change data capture. - Required, Immutable. The name of the schema for the gateway pipelines's storage location. -### pipelines..ingestion_definition +### pipelines.\.ingestion_definition **`Type: Map`** @@ -7037,14 +7053,14 @@ The configuration for a managed ingestion pipeline. These settings cannot be use * - `objects` - Sequence - - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelines..ingestion_definition.objects). + - Required. Settings specifying tables to replicate and the destination for the replicated tables. See [_](#pipelinesnameingestion_definitionobjects). * - `table_configuration` - Map - - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelines..ingestion_definition.table_configuration). + - Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline. See [_](#pipelinesnameingestion_definitiontable_configuration). -### pipelines..ingestion_definition.objects +### pipelines.\.ingestion_definition.objects **`Type: Sequence`** @@ -7061,18 +7077,18 @@ Required. Settings specifying tables to replicate and the destination for the re * - `report` - Map - - Select a specific source report. See [_](#pipelines..ingestion_definition.objects.report). + - Select a specific source report. See [_](#pipelinesnameingestion_definitionobjectsreport). * - `schema` - Map - - Select all tables from a specific source schema. See [_](#pipelines..ingestion_definition.objects.schema). + - Select all tables from a specific source schema. See [_](#pipelinesnameingestion_definitionobjectsschema). * - `table` - Map - - Select a specific source table. See [_](#pipelines..ingestion_definition.objects.table). + - Select a specific source table. See [_](#pipelinesnameingestion_definitionobjectstable). -### pipelines..ingestion_definition.objects.report +### pipelines.\.ingestion_definition.objects.report **`Type: Map`** @@ -7105,10 +7121,10 @@ Select a specific source report. * - `table_configuration` - Map - - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines..ingestion_definition.objects.report.table_configuration). + - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelinesnameingestion_definitionobjectsreporttable_configuration). -### pipelines..ingestion_definition.objects.report.table_configuration +### pipelines.\.ingestion_definition.objects.report.table_configuration **`Type: Map`** @@ -7140,7 +7156,7 @@ Configuration settings to control the ingestion of tables. These settings overri - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. -### pipelines..ingestion_definition.objects.schema +### pipelines.\.ingestion_definition.objects.schema **`Type: Map`** @@ -7173,10 +7189,10 @@ Select all tables from a specific source schema. * - `table_configuration` - Map - - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelines..ingestion_definition.objects.schema.table_configuration). + - Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. See [_](#pipelinesnameingestion_definitionobjectsschematable_configuration). -### pipelines..ingestion_definition.objects.schema.table_configuration +### pipelines.\.ingestion_definition.objects.schema.table_configuration **`Type: Map`** @@ -7208,7 +7224,7 @@ Configuration settings to control the ingestion of tables. These settings are ap - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. -### pipelines..ingestion_definition.objects.table +### pipelines.\.ingestion_definition.objects.table **`Type: Map`** @@ -7249,10 +7265,10 @@ Select a specific source table. * - `table_configuration` - Map - - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelines..ingestion_definition.objects.table.table_configuration). + - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [_](#pipelinesnameingestion_definitionobjectstabletable_configuration). -### pipelines..ingestion_definition.objects.table.table_configuration +### pipelines.\.ingestion_definition.objects.table.table_configuration **`Type: Map`** @@ -7284,7 +7300,7 @@ Configuration settings to control the ingestion of tables. These settings overri - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. -### pipelines..ingestion_definition.table_configuration +### pipelines.\.ingestion_definition.table_configuration **`Type: Map`** @@ -7316,7 +7332,7 @@ Configuration settings to control the ingestion of tables. These settings are ap - The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order. -### pipelines..libraries +### pipelines.\.libraries **`Type: Sequence`** @@ -7333,7 +7349,7 @@ Libraries or code needed by this deployment. * - `file` - Map - - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelines..libraries.file). + - The path to a file that defines a pipeline and is stored in the Databricks Repos. . See [_](#pipelinesnamelibrariesfile). * - `jar` - String @@ -7341,18 +7357,18 @@ Libraries or code needed by this deployment. * - `maven` - Map - - Specification of a maven library to be installed. . See [_](#pipelines..libraries.maven). + - Specification of a maven library to be installed. . See [_](#pipelinesnamelibrariesmaven). * - `notebook` - Map - - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelines..libraries.notebook). + - The path to a notebook that defines a pipeline and is stored in the Databricks workspace. . See [_](#pipelinesnamelibrariesnotebook). * - `whl` - String - URI of the whl to be installed. -### pipelines..libraries.file +### pipelines.\.libraries.file **`Type: Map`** @@ -7373,7 +7389,7 @@ The path to a file that defines a pipeline and is stored in the Databricks Repos - The absolute path of the file. -### pipelines..libraries.maven +### pipelines.\.libraries.maven **`Type: Map`** @@ -7402,7 +7418,7 @@ Specification of a maven library to be installed. - Maven repo to install the Maven package from. If omitted, both Maven Central Repository and Spark Packages are searched. -### pipelines..libraries.notebook +### pipelines.\.libraries.notebook **`Type: Map`** @@ -7423,7 +7439,7 @@ The path to a notebook that defines a pipeline and is stored in the Databricks w - The absolute path of the notebook. -### pipelines..notifications +### pipelines.\.notifications **`Type: Sequence`** @@ -7447,7 +7463,7 @@ List of notification settings for this pipeline. - A list of email addresses notified when a configured alert is triggered. -### pipelines..permissions +### pipelines.\.permissions **`Type: Sequence`** @@ -7479,7 +7495,7 @@ List of notification settings for this pipeline. - The name of the user that has the permission set in level. -### pipelines..restart_window +### pipelines.\.restart_window **`Type: Map`** @@ -7507,7 +7523,7 @@ Restart window of this pipeline. - Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. -### pipelines..restart_window.days_of_week +### pipelines.\.restart_window.days_of_week **`Type: Sequence`** @@ -7515,7 +7531,33 @@ Days of week in which the restart is allowed to happen (within a five-hour windo If not specified all days of the week will be used. -### pipelines..trigger +### pipelines.\.run_as + +**`Type: Map`** + +Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline. + +Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `service_principal_name` + - String + - Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + + * - `user_name` + - String + - The email of an active workspace user. Users can only set this field to their own email. + + +### pipelines.\.trigger **`Type: Map`** @@ -7532,14 +7574,14 @@ Which pipeline trigger to use. Deprecated: Use `continuous` instead. * - `cron` - Map - - See [_](#pipelines..trigger.cron). + - See [_](#pipelinesnametriggercron). * - `manual` - Map - -### pipelines..trigger.cron +### pipelines.\.trigger.cron **`Type: Map`** @@ -7563,7 +7605,7 @@ Which pipeline trigger to use. Deprecated: Use `continuous` instead. - -### pipelines..trigger.manual +### pipelines.\.trigger.manual **`Type: Map`** @@ -7600,19 +7642,19 @@ quality_monitors: * - `custom_metrics` - Sequence - - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitors..custom_metrics). + - Custom metrics to compute on the monitored table. These can be aggregate metrics, derived metrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time windows). . See [_](#quality_monitorsnamecustom_metrics). * - `data_classification_config` - Map - - The data classification config for the monitor. See [_](#quality_monitors..data_classification_config). + - The data classification config for the monitor. See [_](#quality_monitorsnamedata_classification_config). * - `inference_log` - Map - - Configuration for monitoring inference logs. See [_](#quality_monitors..inference_log). + - Configuration for monitoring inference logs. See [_](#quality_monitorsnameinference_log). * - `notifications` - Map - - The notification settings for the monitor. See [_](#quality_monitors..notifications). + - The notification settings for the monitor. See [_](#quality_monitorsnamenotifications). * - `output_schema_name` - String @@ -7620,7 +7662,7 @@ quality_monitors: * - `schedule` - Map - - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitors..schedule). + - The schedule for automatically updating and refreshing metric tables. See [_](#quality_monitorsnameschedule). * - `skip_builtin_dashboard` - Boolean @@ -7640,7 +7682,7 @@ quality_monitors: * - `time_series` - Map - - Configuration for monitoring time series tables. See [_](#quality_monitors..time_series). + - Configuration for monitoring time series tables. See [_](#quality_monitorsnametime_series). * - `warehouse_id` - String @@ -7670,7 +7712,7 @@ resources: timezone_id: UTC ``` -### quality_monitors..custom_metrics +### quality_monitors.\.custom_metrics **`Type: Sequence`** @@ -7709,7 +7751,7 @@ windows). - Can only be one of ``"CUSTOM_METRIC_TYPE_AGGREGATE"``, ``"CUSTOM_METRIC_TYPE_DERIVED"``, or ``"CUSTOM_METRIC_TYPE_DRIFT"``. The ``"CUSTOM_METRIC_TYPE_AGGREGATE"`` and ``"CUSTOM_METRIC_TYPE_DERIVED"`` metrics are computed on a single table, whereas the ``"CUSTOM_METRIC_TYPE_DRIFT"`` compare metrics across baseline and input table, or across the two consecutive time windows. - CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table - CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics - CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics -### quality_monitors..data_classification_config +### quality_monitors.\.data_classification_config **`Type: Map`** @@ -7729,7 +7771,7 @@ The data classification config for the monitor. - Whether data classification is enabled. -### quality_monitors..inference_log +### quality_monitors.\.inference_log **`Type: Map`** @@ -7773,7 +7815,7 @@ Configuration for monitoring inference logs. - Column that contains the timestamps of requests. The column must be one of the following: - A ``TimestampType`` column - A column whose values can be converted to timestamps through the pyspark ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html). -### quality_monitors..notifications +### quality_monitors.\.notifications **`Type: Map`** @@ -7790,14 +7832,14 @@ The notification settings for the monitor. * - `on_failure` - Map - - Who to send notifications to on monitor failure. See [_](#quality_monitors..notifications.on_failure). + - Who to send notifications to on monitor failure. See [_](#quality_monitorsnamenotificationson_failure). * - `on_new_classification_tag_detected` - Map - - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitors..notifications.on_new_classification_tag_detected). + - Who to send notifications to when new data classification tags are detected. See [_](#quality_monitorsnamenotificationson_new_classification_tag_detected). -### quality_monitors..notifications.on_failure +### quality_monitors.\.notifications.on_failure **`Type: Map`** @@ -7817,7 +7859,7 @@ Who to send notifications to on monitor failure. - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. -### quality_monitors..notifications.on_new_classification_tag_detected +### quality_monitors.\.notifications.on_new_classification_tag_detected **`Type: Map`** @@ -7837,7 +7879,7 @@ Who to send notifications to when new data classification tags are detected. - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported. -### quality_monitors..schedule +### quality_monitors.\.schedule **`Type: Map`** @@ -7865,14 +7907,14 @@ The schedule for automatically updating and refreshing metric tables. - The timezone id (e.g., ``"PST"``) in which to evaluate the quartz expression. -### quality_monitors..snapshot +### quality_monitors.\.snapshot **`Type: Map`** Configuration for monitoring snapshot tables. -### quality_monitors..time_series +### quality_monitors.\.time_series **`Type: Map`** @@ -7926,7 +7968,7 @@ registered_models: * - `grants` - Sequence - - See [_](#registered_models..grants). + - See [_](#registered_modelsnamegrants). * - `name` - String @@ -7959,7 +8001,7 @@ resources: principal: account users ``` -### registered_models..grants +### registered_models.\.grants **`Type: Sequence`** @@ -8016,7 +8058,7 @@ schemas: * - `grants` - Sequence - - See [_](#schemas..grants). + - See [_](#schemasnamegrants). * - `name` - String @@ -8073,7 +8115,7 @@ resources: catalog_name: main ``` -### schemas..grants +### schemas.\.grants **`Type: Sequence`** @@ -8131,7 +8173,7 @@ volumes: * - `grants` - Sequence - - See [_](#volumes..grants). + - See [_](#volumesnamegrants). * - `name` - String @@ -8165,7 +8207,7 @@ resources: For an example bundle that runs a job that writes to a file in volume, see the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/write_from_job_to_volume). -### volumes..grants +### volumes.\.grants **`Type: Sequence`** diff --git a/bundle/docsgen/testdata/anchors.md b/bundle/docsgen/testdata/anchors.md new file mode 100644 index 000000000..0145d8cc9 --- /dev/null +++ b/bundle/docsgen/testdata/anchors.md @@ -0,0 +1,28 @@ +Header + +## some_field + +**`Type: Map`** + +This is a description + + + +.. list-table:: + :header-rows: 1 + + * - Key + - Type + - Description + + * - `my_attribute` + - Map + - Desc with link. See [_](#some_fieldnamemy_attribute). + + +### some_field.\.my_attribute + +**`Type: Boolean`** + +Another description + \ No newline at end of file diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 2d1a6a3d8..c10f43b04 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -414,16 +414,6 @@ github.com/databricks/cli/bundle/config/resources.Permission: "user_name": "description": |- The name of the user that has the permission set in level. -github.com/databricks/cli/bundle/config/resources.Pipeline: - "allow_duplicate_names": - "description": |- - PLACEHOLDER - "dry_run": - "description": |- - PLACEHOLDER - "run_as": - "description": |- - PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 051243c4d..620f1cb70 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -124,3 +124,23 @@ func getAnnotations(path string) (annotation.File, error) { err = yaml.Unmarshal(b, &data) return data, err } + +func TestNoDuplicatedAnnotations(t *testing.T) { + // Check for duplicated annotations in annotation files + files := []string{ + "annotations_openapi_overrides.yml", + "annotations.yml", + } + + annotations := map[string]bool{} + for _, file := range files { + annotationsFile, err := getAnnotations(file) + assert.NoError(t, err) + for k := range annotationsFile { + if _, ok := annotations[k]; ok { + t.Errorf("Annotation `%s` is duplicated in %s", k, file) + } + annotations[k] = true + } + } +} From ddcfac5069aba78256e39f6ffa61f4c86feee124 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 21 Feb 2025 19:19:47 +0530 Subject: [PATCH 105/105] Add library for spawning a daemon (#2354) ## Changes This PR adds a library for spawning a daemon process. Our needs are different from those of a typical daemon process in that we want to handle being orphaned gracefully. This is because, in the vast majority of telemetry use cases, the main CLI process (i.e., the parent process) will exit before the telemetry process has a chance to finish uploading the logs. To achieve this we "detach" the child process from the parent process, which requires different flags for UNIX vs. non-unix systems. Here are the properties that we want to ensure for our telemetry child/daemon processes: 1. They do not block the parent process. Reason: The main CLI process should not be blocked on the telemetry child process. 2. The child can read from stdin. The parent can write to stdin. Reason: Telemetry logs will be passed to the child process via stdin. 3. Output logs do not leak from the child process. Reason: Telemetry logs should not be visible to users of the CLI. ## Tests Unit test --- go.mod | 3 +- libs/daemon/daemon.go | 120 ++++++++++++++++++ libs/daemon/daemon_test.go | 51 ++++++++ libs/daemon/daemon_unix.go | 17 +++ libs/daemon/daemon_windows.go | 16 +++ libs/daemon/internal/parent_process/main.go | 30 +++++ libs/daemon/internal/parent_process/server.py | 40 ++++++ 7 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 libs/daemon/daemon.go create mode 100644 libs/daemon/daemon_test.go create mode 100644 libs/daemon/daemon_unix.go create mode 100644 libs/daemon/daemon_windows.go create mode 100644 libs/daemon/internal/parent_process/main.go create mode 100644 libs/daemon/internal/parent_process/server.py diff --git a/go.mod b/go.mod index 2e2505361..b4157c61b 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require golang.org/x/sys v0.30.0 + require ( cloud.google.com/go/auth v0.4.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect @@ -72,7 +74,6 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.30.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/libs/daemon/daemon.go b/libs/daemon/daemon.go new file mode 100644 index 000000000..91914477b --- /dev/null +++ b/libs/daemon/daemon.go @@ -0,0 +1,120 @@ +package daemon + +import ( + "fmt" + "io" + "os" + "os/exec" + "strconv" +) + +const DatabricksCliParentPid = "DATABRICKS_CLI_PARENT_PID" + +type Daemon struct { + // If provided, the child process's pid will be written in the file at this + // path. + PidFilePath string + + // Environment variables to set in the child process. + Env []string + + // Path to executable to run. If empty, the current executable is used. + Executable string + + // Arguments to pass to the child process. + Args []string + + // Log file to write the child process's output to. + LogFile string + + logFile *os.File + cmd *exec.Cmd + stdin io.WriteCloser +} + +func (d *Daemon) Start() error { + cli, err := os.Executable() + if err != nil { + return err + } + + executable := d.Executable + if executable == "" { + executable = cli + } + + d.cmd = exec.Command(executable, d.Args...) + + // Set environment variable so that the child process knows its parent's PID. + // In unix systems orphaned processes are automatically re-parented to init (pid 1) + // so we cannot rely on os.Getppid() to get the original parent's pid. + d.Env = append(d.Env, fmt.Sprintf("%s=%d", DatabricksCliParentPid, os.Getpid())) + d.cmd.Env = d.Env + + d.cmd.SysProcAttr = sysProcAttr() + + // By default redirect stdout and stderr to /dev/null. + d.cmd.Stdout = nil + d.cmd.Stderr = nil + + // If a log file is provided, redirect stdout and stderr to the log file. + if d.LogFile != "" { + d.logFile, err = os.OpenFile(d.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return fmt.Errorf("failed to open log file: %w", err) + } + + d.cmd.Stdout = d.logFile + d.cmd.Stderr = d.logFile + } + + d.stdin, err = d.cmd.StdinPipe() + if err != nil { + return fmt.Errorf("failed to get stdin pipe: %w", err) + } + + err = d.cmd.Start() + if err != nil { + return err + } + + if d.PidFilePath != "" { + err = os.WriteFile(d.PidFilePath, []byte(strconv.Itoa(d.cmd.Process.Pid)), 0o644) + if err != nil { + return fmt.Errorf("failed to write pid file: %w", err) + } + } + + return nil +} + +func (d *Daemon) WriteInput(b []byte) error { + _, err := d.stdin.Write(b) + return err +} + +func (d *Daemon) Release() error { + if d.stdin != nil { + err := d.stdin.Close() + if err != nil { + return fmt.Errorf("failed to close stdin: %w", err) + } + } + + // Note that the child process will stream it's output directly to the log file. + // So it's safe to close this file handle even if the child process is still running. + if d.logFile != nil { + err := d.logFile.Close() + if err != nil { + return fmt.Errorf("failed to close log file: %w", err) + } + } + + if d.cmd == nil { + return nil + } + + // The docs for [os.Process.Release] recommend calling Release if Wait is not called. + // It's probably not necessary but we call it just to be safe. + return d.cmd.Process.Release() +} diff --git a/libs/daemon/daemon_test.go b/libs/daemon/daemon_test.go new file mode 100644 index 000000000..ee9d92baa --- /dev/null +++ b/libs/daemon/daemon_test.go @@ -0,0 +1,51 @@ +package daemon + +import ( + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/databricks/cli/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDaemon(t *testing.T) { + tmpDir := t.TempDir() + cmd := exec.Command("go", "run", "internal/parent_process/main.go", tmpDir) + + // cmd.Run() will block until the parent process exits. + err := cmd.Run() + require.NoError(t, err) + + // Assert that a PID file was created for the child process. + assert.FileExists(t, filepath.Join(tmpDir, "child.pid")) + + // Wait 10 seconds for the server to start and to write the port number to + // a file. + portFilePath := filepath.Join(tmpDir, "port.txt") + assert.Eventually(t, func() bool { + _, err := os.Stat(portFilePath) + return err == nil + }, 10*time.Second, 100*time.Millisecond) + + port, err := strconv.Atoi(testutil.ReadFile(t, portFilePath)) + require.NoError(t, err) + + // Query the local server, which should be alive even after the parent process + // has exited. + r, err := http.Get("http://localhost:" + strconv.Itoa(port)) + require.NoError(t, err) + defer r.Body.Close() + + // The server should respond with "child says hi". + assert.Equal(t, http.StatusOK, r.StatusCode) + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + assert.Equal(t, "child says hi", string(b)) +} diff --git a/libs/daemon/daemon_unix.go b/libs/daemon/daemon_unix.go new file mode 100644 index 000000000..b9a7023a7 --- /dev/null +++ b/libs/daemon/daemon_unix.go @@ -0,0 +1,17 @@ +//go:build linux || darwin + +package daemon + +import "syscall" + +// References: +// 1. linux: https://go.dev/src/syscall/exec_linux.go +// 2. macos (arm): https://go.dev/src/syscall/exec_libc2.go +func sysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + // Create a new session for the child process. This ensures that the daemon + // is not terminated when the parent session is closed. This can happen + // for example when a ssh session is terminated. + Setsid: true, + } +} diff --git a/libs/daemon/daemon_windows.go b/libs/daemon/daemon_windows.go new file mode 100644 index 000000000..bccf22e4b --- /dev/null +++ b/libs/daemon/daemon_windows.go @@ -0,0 +1,16 @@ +//go:build windows + +package daemon + +import ( + "syscall" + + "golang.org/x/sys/windows" +) + +func sysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + HideWindow: true, + CreationFlags: windows.CREATE_NEW_PROCESS_GROUP | windows.DETACHED_PROCESS, + } +} diff --git a/libs/daemon/internal/parent_process/main.go b/libs/daemon/internal/parent_process/main.go new file mode 100644 index 000000000..87c1bdda2 --- /dev/null +++ b/libs/daemon/internal/parent_process/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/databricks/cli/libs/daemon" +) + +func main() { + tmpDir := os.Args[1] + + d := daemon.Daemon{ + PidFilePath: filepath.Join(tmpDir, "child.pid"), + Executable: "python3", + // The server script writes the port number the server is listening on + // to the specified file. + Args: []string{"./internal/parent_process/server.py", filepath.Join(tmpDir, "port.txt")}, + } + + err := d.Start() + if err != nil { + panic(err) + } + + err = d.Release() + if err != nil { + panic(err) + } +} diff --git a/libs/daemon/internal/parent_process/server.py b/libs/daemon/internal/parent_process/server.py new file mode 100644 index 000000000..ad341f992 --- /dev/null +++ b/libs/daemon/internal/parent_process/server.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +if len(sys.argv) < 2: + print("Usage: python script.py ") + sys.exit(1) + +port_file_path = sys.argv[1] + + +class SimpleHandler(BaseHTTPRequestHandler): + def do_GET(self): + # Send HTTP 200 response with plain text content + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + self.wfile.write(b"child says hi") + + +# Bind to localhost on port 0 to let the OS pick an available port. +server_address = ("localhost", 0) +httpd = HTTPServer(server_address, SimpleHandler) + +# Retrieve the assigned port. +assigned_port = httpd.server_address[1] + +# Write the port number to the provided file path. +with open(port_file_path, "w") as f: + f.write(str(assigned_port)) + +try: + # Automatically shut down the server after 2 minutes. This is a precaution to + # prevent the server from running indefinitely incase the GET API is never called. + httpd.timeout = 120 + + # This server will exit after one request. + httpd.handle_request() +except KeyboardInterrupt: + print("\nServer is shutting down.")