steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge

This commit is contained in:
Shreyas Goenka 2025-01-16 12:50:28 +01:00
parent 69d477ff0b 8f34fc7961
commit a5243d628a
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
403 changed files with 7748 additions and 2740 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.codegen/_openapi_sha
View File

@ -1 +1 @@
a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d
779817ed8d63031f5ea761fbd25ee84f38feec0d

4
.codegen/service.go.tmpl
View File

@ -140,9 +140,9 @@ func new{{.PascalName}}() *cobra.Command {
{{- end}}
{{$method := .}}
{{ if not .IsJsonOnly }}
{{range $request.Fields -}}
{{range .AllFields -}}
{{- if not .Required -}}
{{if .Entity.IsObject }}// TODO: complex arg: {{.Name}}
{{if .Entity.IsObject}}{{if not (eq . $method.RequestBodyField) }}// TODO: complex arg: {{.Name}}{{end}}
{{else if .Entity.IsAny }}// TODO: any: {{.Name}}
{{else if .Entity.ArrayValue }}// TODO: array: {{.Name}}
{{else if .Entity.MapValue }}// TODO: map via StringToStringVar: {{.Name}}

4
.github/dependabot.yml vendored
View File

@ -4,3 +4,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"

8
.github/workflows/close-stale-issues.yml vendored
View File

@ -18,7 +18,7 @@ jobs:
pull-requests: write
steps:
- uses: actions/stale@v9
- uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
with:
stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
@ -31,10 +31,8 @@ jobs:
exempt-pr-labels: No Autoclose
# Issue timing
days-before-stale: 30
days-before-close: 7
days-before-stale: 60
days-before-close: 30
repo-token: ${{ secrets.GITHUB_TOKEN }}
loglevel: DEBUG
# TODO: Remove dry-run after merge when confirmed it works correctly
dry-run: true

2
.github/workflows/external-message.yml vendored
View File

@ -25,7 +25,7 @@ jobs:
if: "${{ github.event.pull_request.head.repo.fork }}"
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Delete old comments
env:

2
.github/workflows/integration-main.yml vendored
View File

@ -20,7 +20,7 @@ jobs:
steps:
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@v1
uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
with:
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}

2
.github/workflows/integration-pr.yml vendored
View File

@ -23,7 +23,7 @@ jobs:
steps:
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@v1
uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
with:
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}

16
.github/workflows/publish-winget.yml vendored
View File

@ -2,15 +2,27 @@ name: publish-winget
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to publish'
default: ''
jobs:
publish-to-winget-pkgs:
runs-on: windows-latest
runs-on:
group: databricks-protected-runner-group
labels: windows-server-latest
environment: release
steps:
- uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
- uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # v2
with:
identifier: Databricks.DatabricksCLI
installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
fork-user: eng-dev-ecosystem-bot
# Use the tag from the input, or the ref name if the input is not provided.
# The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
release-tag: ${{ inputs.tag || github.ref_name }}

44
.github/workflows/push.yml vendored
View File

@ -13,12 +13,26 @@ on:
# seed the build cache.
branches:
- main
schedule:
- cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily
env:
GOTESTSUM_FORMAT: github-actions
jobs:
cleanups:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
steps:
- name: Clean up cache if running on schedule
if: ${{ github.event_name == 'schedule' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh cache delete --all --repo databricks/cli || true
tests:
needs: cleanups
runs-on: ${{ matrix.os }}
strategy:
@ -31,20 +45,20 @@ jobs:
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version: 1.23.4
- name: Setup Python
uses: actions/setup-python@v5
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
python-version: '3.9'
- name: Install uv
uses: astral-sh/setup-uv@v4
uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
- name: Set go env
run: |
@ -61,13 +75,18 @@ jobs:
run: make test
golangci:
needs: cleanups
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version: 1.23.4
# Use different schema from regular job, to avoid overwriting the same key
cache-dependency-path: |
go.sum
.golangci.yaml
- name: Run go mod tidy
run: |
go mod tidy
@ -76,22 +95,27 @@ jobs:
# Exit with status code 1 if there are differences (i.e. unformatted files)
git diff --exit-code
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
with:
version: v1.63.1
version: v1.63.4
args: --timeout=15m
validate-bundle-schema:
needs: cleanups
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version: 1.23.4
# Use different schema from regular job, to avoid overwriting the same key
cache-dependency-path: |
go.sum
bundle/internal/schema/*.*
- name: Verify that the schema is up to date
run: |

14
.github/workflows/release-snapshot.yml vendored
View File

@ -26,13 +26,13 @@ jobs:
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
fetch-tags: true
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version: 1.23.4
@ -48,27 +48,27 @@ jobs:
- name: Run GoReleaser
id: releaser
uses: goreleaser/goreleaser-action@v6
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
with:
version: ~> v2
args: release --snapshot --skip docker
- name: Upload macOS binaries
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
name: cli_darwin_snapshot
path: |
dist/*_darwin_*/
- name: Upload Linux binaries
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
name: cli_linux_snapshot
path: |
dist/*_linux_*/
- name: Upload Windows binaries
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
with:
name: cli_windows_snapshot
path: |
@ -88,7 +88,7 @@ jobs:
# Snapshot release may only be updated for commits to the main branch.
if: github.ref == 'refs/heads/main'
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
with:
name: Snapshot
prerelease: true

34
.github/workflows/release.yml vendored
View File

@ -18,13 +18,13 @@ jobs:
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
fetch-tags: true
- name: Setup Go
uses: actions/setup-go@v5
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version: 1.23.4
@ -37,7 +37,7 @@ jobs:
# Log into the GitHub Container Registry. The goreleaser action will create
# the docker images and push them to the GitHub Container Registry.
- uses: "docker/login-action@v3"
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: "ghcr.io"
username: "${{ github.actor }}"
@ -46,11 +46,11 @@ jobs:
# QEMU is required to build cross platform docker images using buildx.
# It allows virtualization of the CPU architecture at the application level.
- name: Set up QEMU dependency
uses: docker/setup-qemu-action@v3
uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
- name: Run GoReleaser
id: releaser
uses: goreleaser/goreleaser-action@v6
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
with:
version: ~> v2
args: release
@ -58,8 +58,12 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
create-setup-cli-release-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |
@ -67,7 +71,7 @@ jobs:
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
- name: Update setup-cli
uses: actions/github-script@v7
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
script: |
@ -82,8 +86,12 @@ jobs:
});
create-homebrew-tap-release-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |
@ -91,7 +99,7 @@ jobs:
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
- name: Update homebrew-tap
uses: actions/github-script@v7
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
script: |
@ -119,8 +127,12 @@ jobs:
});
create-vscode-extension-update-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |
@ -128,7 +140,7 @@ jobs:
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
- name: Update CLI version in the VSCode extension
uses: actions/github-script@v7
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
script: |

1
.gitignore vendored
View File

@ -20,6 +20,7 @@ dist/
*.log
coverage.txt
coverage-acceptance.txt
__pycache__
*.pyc

4
.golangci.yaml
View File

@ -14,6 +14,8 @@ linters:
- testifylint
- intrange
- mirror
- perfsprint
- unconvert
linters-settings:
govet:
enable-all: true
@ -40,6 +42,8 @@ linters-settings:
disable:
# good check, but we have too many assert.(No)?Errorf? so excluding for now
- require-error
copyloopvar:
check-alias: true
issues:
exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/
max-issues-per-linter: 1000

9
CHANGELOG.md
View File

@ -1,5 +1,14 @@
# Version changelog
## [Release] Release v0.238.0
Bundles:
* Fix finding Python within virtualenv on Windows ([#2034](https://github.com/databricks/cli/pull/2034)).
* Include missing field descriptions in JSON schema ([#2045](https://github.com/databricks/cli/pull/2045)).
* Add validation for volume referenced from `artifact_path` ([#2050](https://github.com/databricks/cli/pull/2050)).
* Handle `${workspace.file_path}` references in source-linked deployments ([#2046](https://github.com/databricks/cli/pull/2046)).
* Set the write bit for files written during template initialization ([#2068](https://github.com/databricks/cli/pull/2068)).
## [Release] Release v0.237.0
Bundles:

25
Makefile
View File

@ -1,15 +1,21 @@
default: build
default: vendor fmt lint
PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... .
PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
GOTESTSUM_FORMAT ?= pkgname-and-test-fails
lint:
./lint.sh ./...
golangci-lint run --fix
lintcheck:
golangci-lint run ./...
# Note 'make lint' will do formatting as well. However, if there are compilation errors,
# formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
# If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
fmt:
golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
test:
gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
@ -19,6 +25,17 @@ cover:
showcover:
go tool cover -html=coverage.txt
acc-cover:
rm -fr ./acceptance/build/cover/
CLI_GOCOVERDIR=build/cover go test ./acceptance
rm -fr ./acceptance/build/cover-merged/
mkdir -p acceptance/build/cover-merged/
go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
acc-showcover:
go tool cover -html=coverage-acceptance.txt
build: vendor
go build -mod vendor
@ -39,4 +56,4 @@ integration:
integration-short:
$(INTEGRATION) -short
.PHONY: lint lintcheck test cover showcover build snapshot vendor schema integration integration-short
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover

19
acceptance/README.md Normal file
View File

@ -0,0 +1,19 @@
Acceptance tests are blackbox tests that are run against compiled binary.
Currently these tests are run against "fake" HTTP server pretending to be Databricks API. However, they will be extended to run against real environment as regular integration tests.
To author a test,
- Add a new directory under `acceptance`. Any level of nesting is supported.
- Add `databricks.yml` there.
- Add `script` with commands to run, e.g. `$CLI bundle validate`. The test case is recognized by presence of `script`.
The test runner will run script and capture output and compare it with `output.txt` file in the same directory.
In order to write `output.txt` for the first time or overwrite it with the current output pass -update flag to go test.
The scripts are run with `bash -e` so any errors will be propagated. They are captured in `output.txt` by appending `Exit code: N` line at the end.
For more complex tests one can also use:
- `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
- `trace` helper: prints the arguments before executing the command.
- custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.

375
acceptance/acceptance_test.go Normal file
View File

@ -0,0 +1,375 @@
package acceptance_test
import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"sort"
"strings"
"testing"
"time"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/testdiff"
"github.com/databricks/databricks-sdk-go"
"github.com/stretchr/testify/require"
)
var KeepTmp = os.Getenv("KEEP_TMP") != ""
const (
EntryPointScript = "script"
CleanupScript = "script.cleanup"
PrepareScript = "script.prepare"
)
var Scripts = map[string]bool{
EntryPointScript: true,
CleanupScript: true,
PrepareScript: true,
}
func TestAccept(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
coverDir := os.Getenv("CLI_GOCOVERDIR")
if coverDir != "" {
require.NoError(t, os.MkdirAll(coverDir, os.ModePerm))
coverDir, err = filepath.Abs(coverDir)
require.NoError(t, err)
t.Logf("Writing coverage to %s", coverDir)
}
execPath := BuildCLI(t, cwd, coverDir)
// $CLI is what test scripts are using
t.Setenv("CLI", execPath)
// Make helper scripts available
t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
repls := testdiff.ReplacementsContext{}
repls.Set(execPath, "$CLI")
tempHomeDir := t.TempDir()
repls.Set(tempHomeDir, "$TMPHOME")
t.Logf("$TMPHOME=%v", tempHomeDir)
// Prevent CLI from downloading terraform in each test:
t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
ctx := context.Background()
cloudEnv := os.Getenv("CLOUD_ENV")
if cloudEnv == "" {
server := StartServer(t)
AddHandlers(server)
// Redirect API access to local server:
t.Setenv("DATABRICKS_HOST", server.URL)
t.Setenv("DATABRICKS_TOKEN", "dapi1234")
homeDir := t.TempDir()
// Do not read user's ~/.databrickscfg
t.Setenv(env.HomeEnvVar(), homeDir)
}
workspaceClient, err := databricks.NewWorkspaceClient()
require.NoError(t, err)
user, err := workspaceClient.CurrentUser.Me(ctx)
require.NoError(t, err)
require.NotNil(t, user)
testdiff.PrepareReplacementsUser(t, &repls, *user)
testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
testDirs := getTests(t)
require.NotEmpty(t, testDirs)
for _, dir := range testDirs {
testName := strings.ReplaceAll(dir, "\\", "/")
t.Run(testName, func(t *testing.T) {
t.Parallel()
runTest(t, dir, coverDir, repls)
})
}
}
func getTests(t *testing.T) []string {
testDirs := make([]string, 0, 128)
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
name := filepath.Base(path)
if name == EntryPointScript {
// Presence of 'script' marks a test case in this directory
testDirs = append(testDirs, filepath.Dir(path))
}
return nil
})
require.NoError(t, err)
sort.Strings(testDirs)
return testDirs
}
func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
var tmpDir string
var err error
if KeepTmp {
tempDirBase := filepath.Join(os.TempDir(), "acceptance")
_ = os.Mkdir(tempDirBase, 0o755)
tmpDir, err = os.MkdirTemp(tempDirBase, "")
require.NoError(t, err)
t.Logf("Created directory: %s", tmpDir)
} else {
tmpDir = t.TempDir()
}
scriptContents := readMergedScriptContents(t, dir)
testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
inputs := make(map[string]bool, 2)
outputs := make(map[string]bool, 2)
err = CopyDir(dir, tmpDir, inputs, outputs)
require.NoError(t, err)
args := []string{"bash", "-euo", "pipefail", EntryPointScript}
cmd := exec.Command(args[0], args[1:]...)
if coverDir != "" {
// Creating individual coverage directory for each test, because writing to the same one
// results in sporadic failures like this one (only if tests are running in parallel):
// +error: coverage meta-data emit failed: writing ... rename .../tmp.covmeta.b3f... .../covmeta.b3f2c...: no such file or directory
coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
err := os.MkdirAll(coverDir, os.ModePerm)
require.NoError(t, err)
cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
}
cmd.Dir = tmpDir
outB, err := cmd.CombinedOutput()
out := formatOutput(string(outB), err)
out = repls.Replace(out)
doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
for key := range outputs {
if key == "output.txt" {
// handled above
continue
}
pathNew := filepath.Join(tmpDir, key)
newValBytes, err := os.ReadFile(pathNew)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
} else {
t.Errorf("%s: could not read: %s", key, err)
}
continue
}
pathExpected := filepath.Join(dir, key)
newVal := repls.Replace(string(newValBytes))
doComparison(t, pathExpected, pathNew, newVal)
}
// Make sure there are not unaccounted for new files
files, err := os.ReadDir(tmpDir)
require.NoError(t, err)
for _, f := range files {
name := f.Name()
if _, ok := inputs[name]; ok {
continue
}
if _, ok := outputs[name]; ok {
continue
}
t.Errorf("Unexpected output: %s", f)
if strings.HasPrefix(name, "out") {
// We have a new file starting with "out"
// Show the contents & support overwrite mode for it:
pathNew := filepath.Join(tmpDir, name)
newVal := testutil.ReadFile(t, pathNew)
newVal = repls.Replace(newVal)
doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
}
}
}
func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
valueNew = testdiff.NormalizeNewlines(valueNew)
valueExpected := string(readIfExists(t, pathExpected))
valueExpected = testdiff.NormalizeNewlines(valueExpected)
testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
if testdiff.OverwriteMode {
if valueNew != "" {
t.Logf("Overwriting: %s", pathExpected)
testutil.WriteFile(t, pathExpected, valueNew)
} else {
t.Logf("Removing: %s", pathExpected)
_ = os.Remove(pathExpected)
}
}
}
// Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ...
// Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
func readMergedScriptContents(t *testing.T, dir string) string {
scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
// Wrap script contents in a subshell such that changing the working
// directory only affects the main script and not cleanup.
scriptContents = "(\n" + scriptContents + ")\n"
prepares := []string{}
cleanups := []string{}
for {
x := readIfExists(t, filepath.Join(dir, CleanupScript))
if len(x) > 0 {
cleanups = append(cleanups, string(x))
}
x = readIfExists(t, filepath.Join(dir, PrepareScript))
if len(x) > 0 {
prepares = append(prepares, string(x))
}
if dir == "" || dir == "." {
break
}
dir = filepath.Dir(dir)
require.True(t, filepath.IsLocal(dir))
}
slices.Reverse(prepares)
prepares = append(prepares, scriptContents)
prepares = append(prepares, cleanups...)
return strings.Join(prepares, "\n")
}
func BuildCLI(t *testing.T, cwd, coverDir string) string {
execPath := filepath.Join(cwd, "build", "databricks")
if runtime.GOOS == "windows" {
execPath += ".exe"
}
start := time.Now()
args := []string{
"go", "build",
"-mod", "vendor",
"-o", execPath,
}
if coverDir != "" {
args = append(args, "-cover")
}
if runtime.GOOS == "windows" {
// Get this error on my local Windows:
// error obtaining VCS status: exit status 128
// Use -buildvcs=false to disable VCS stamping.
args = append(args, "-buildvcs=false")
}
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = ".."
out, err := cmd.CombinedOutput()
elapsed := time.Since(start)
t.Logf("%s took %s", args, elapsed)
require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
if len(out) > 0 {
t.Logf("go build output: %s: %s", args, out)
}
// Quick check + warm up cache:
cmd = exec.Command(execPath, "--version")
out, err = cmd.CombinedOutput()
require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
return execPath
}
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
return err
}
func formatOutput(out string, err error) string {
if err == nil {
return out
}
if exiterr, ok := err.(*exec.ExitError); ok {
exitCode := exiterr.ExitCode()
out += fmt.Sprintf("\nExit code: %d\n", exitCode)
} else {
out += fmt.Sprintf("\nError: %s\n", err)
}
return out
}
func readIfExists(t *testing.T, path string) []byte {
data, err := os.ReadFile(path)
if err == nil {
return data
}
if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%s: %s", path, err)
}
return []byte{}
}
func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
name := info.Name()
relPath, err := filepath.Rel(src, path)
if err != nil {
return err
}
if strings.HasPrefix(name, "out") {
outputs[relPath] = true
return nil
} else {
inputs[relPath] = true
}
if _, ok := Scripts[name]; ok {
return nil
}
destPath := filepath.Join(dst, relPath)
if info.IsDir() {
return os.MkdirAll(destPath, info.Mode())
}
return copyFile(path, destPath)
})
}

21
acceptance/bin/sort_blocks.py Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""
Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
This is to workaround non-determinism in the output.
"""
import sys
blocks = []
for line in sys.stdin:
if not line.strip():
if blocks and blocks[-1]:
blocks.append('')
continue
if not blocks:
blocks.append('')
blocks[-1] += line
blocks.sort()
print("\n".join(blocks))

1
acceptance/build/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
databricks

6
acceptance/bundle/init/dbt-sql/input.json Normal file
View File

@ -0,0 +1,6 @@
{
"project_name": "my_dbt_sql",
"http_path": "/sql/2.0/warehouses/f00dcafe",
"default_catalog": "main",
"personal_schemas": "yes, use a schema based on the current user name during development"
}

32
acceptance/bundle/init/dbt-sql/output.txt Normal file
View File

@ -0,0 +1,32 @@
>>> $CLI bundle init dbt-sql --config-file ./input.json
Welcome to the dbt template for Databricks Asset Bundles!
A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
workspace_host: $DATABRICKS_URL
📊 Your new project has been created in the 'my_dbt_sql' directory!
If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started.
Refer to the README.md file for full "getting started" guide and production setup instructions.
>>> $CLI bundle validate -t dev
Name: my_dbt_sql
Target: dev
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev
Validation OK!
>>> $CLI bundle validate -t prod
Name: my_dbt_sql
Target: prod
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod
Validation OK!

5
acceptance/bundle/init/dbt-sql/script Normal file
View File

@ -0,0 +1,5 @@
trace $CLI bundle init dbt-sql --config-file ./input.json
cd my_dbt_sql
trace $CLI bundle validate -t dev
trace $CLI bundle validate -t prod

1
acceptance/bundle/init/dbt-sql/script.cleanup Normal file
View File

@ -0,0 +1 @@
rm -fr my_dbt_sql

6
acceptance/bundle/init/default-python/input.json Normal file
View File

@ -0,0 +1,6 @@
{
"project_name": "my_default_python",
"include_notebook": "yes",
"include_dlt": "yes",
"include_python": "yes"
}

30
acceptance/bundle/init/default-python/output.txt Normal file
View File

@ -0,0 +1,30 @@
>>> $CLI bundle init default-python --config-file ./input.json
Welcome to the default Python template for Databricks Asset Bundles!
Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): $DATABRICKS_URL
✨ Your new project has been created in the 'my_default_python' directory!
Please refer to the README.md file for "getting started" instructions.
See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
>>> $CLI bundle validate -t dev
Name: my_default_python
Target: dev
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev
Validation OK!
>>> $CLI bundle validate -t prod
Name: my_default_python
Target: prod
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod
Validation OK!

5
acceptance/bundle/init/default-python/script Normal file
View File

@ -0,0 +1,5 @@
trace $CLI bundle init default-python --config-file ./input.json
cd my_default_python
trace $CLI bundle validate -t dev
trace $CLI bundle validate -t prod

1
acceptance/bundle/init/default-python/script.cleanup Normal file
View File

@ -0,0 +1 @@
rm -fr my_default_python

6
acceptance/bundle/init/default-sql/input.json Normal file
View File

@ -0,0 +1,6 @@
{
"project_name": "my_default_sql",
"http_path": "/sql/2.0/warehouses/f00dcafe",
"default_catalog": "main",
"personal_schemas": "yes, automatically use a schema based on the current user name during development"
}

32
acceptance/bundle/init/default-sql/output.txt Normal file
View File

@ -0,0 +1,32 @@
>>> $CLI bundle init default-sql --config-file ./input.json
Welcome to the default SQL template for Databricks Asset Bundles!
A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html.
workspace_host: $DATABRICKS_URL
✨ Your new project has been created in the 'my_default_sql' directory!
Please refer to the README.md file for "getting started" instructions.
See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
>>> $CLI bundle validate -t dev
Name: my_default_sql
Target: dev
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev
Validation OK!
>>> $CLI bundle validate -t prod
Name: my_default_sql
Target: prod
Workspace:
Host: $DATABRICKS_URL
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod
Validation OK!

5
acceptance/bundle/init/default-sql/script Normal file
View File

@ -0,0 +1,5 @@
trace $CLI bundle init default-sql --config-file ./input.json
cd my_default_sql
trace $CLI bundle validate -t dev
trace $CLI bundle validate -t prod

1
acceptance/bundle/init/default-sql/script.cleanup Normal file
View File

@ -0,0 +1 @@
rm -fr my_default_sql

3
bundle/tests/clusters/databricks.yml → acceptance/bundle/override/clusters/databricks.yml
View File

@ -1,9 +1,6 @@
bundle:
name: clusters
workspace:
host: https://acme.cloud.databricks.com/
resources:
clusters:
foo:

33
acceptance/bundle/override/clusters/output.txt Normal file
View File

@ -0,0 +1,33 @@
>>> $CLI bundle validate -o json -t default
{
"autoscale": {
"max_workers": 7,
"min_workers": 2
},
"cluster_name": "foo",
"custom_tags": {},
"node_type_id": "i3.xlarge",
"num_workers": 2,
"spark_conf": {
"spark.executor.memory": "2g"
},
"spark_version": "13.3.x-scala2.12"
}
>>> $CLI bundle validate -o json -t development
{
"autoscale": {
"max_workers": 3,
"min_workers": 1
},
"cluster_name": "foo-override",
"custom_tags": {},
"node_type_id": "m5.xlarge",
"num_workers": 3,
"spark_conf": {
"spark.executor.memory": "4g",
"spark.executor.memory2": "4g"
},
"spark_version": "15.2.x-scala2.12"
}

2
acceptance/bundle/override/clusters/script Normal file
View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t default | jq .resources.clusters.foo
trace $CLI bundle validate -o json -t development | jq .resources.clusters.foo

3
bundle/tests/override_job_cluster/databricks.yml → acceptance/bundle/override/job_cluster/databricks.yml
View File

@ -1,9 +1,6 @@
bundle:
name: override_job_cluster
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
foo:

56
acceptance/bundle/override/job_cluster/output.txt Normal file
View File

@ -0,0 +1,56 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}

2
acceptance/bundle/override/job_cluster/script Normal file
View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'

36
acceptance/bundle/override/job_cluster_var/databricks.yml Normal file
View File

@ -0,0 +1,36 @@
bundle:
name: override_job_cluster
variables:
mykey:
default: key
resources:
jobs:
foo:
name: job
job_clusters:
- job_cluster_key: key
new_cluster:
spark_version: 13.3.x-scala2.12
targets:
development:
resources:
jobs:
foo:
job_clusters:
- job_cluster_key: "${var.mykey}"
new_cluster:
node_type_id: i3.xlarge
num_workers: 1
staging:
resources:
jobs:
foo:
job_clusters:
- job_cluster_key: "${var.mykey}"
new_cluster:
node_type_id: i3.2xlarge
num_workers: 4

74
acceptance/bundle/override/job_cluster_var/output.txt Normal file
View File

@ -0,0 +1,74 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -t development
Name: override_job_cluster
Target: development
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/development
Validation OK!
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -t staging
Name: override_job_cluster
Target: staging
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging
Validation OK!

4
acceptance/bundle/override/job_cluster_var/script Normal file
View File

@ -0,0 +1,4 @@
trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
trace $CLI bundle validate -t development
trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
trace $CLI bundle validate -t staging

3
bundle/tests/override_job_tasks/databricks.yml → acceptance/bundle/override/job_tasks/databricks.yml
View File

@ -1,9 +1,6 @@
bundle:
name: override_job_tasks
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
foo:

6
acceptance/bundle/override/job_tasks/out.development.stderr.txt Normal file
View File

@ -0,0 +1,6 @@
>>> errcode $CLI bundle validate -o json -t development
Error: file ./test1.py not found
Exit code: 1

77
acceptance/bundle/override/job_tasks/output.txt Normal file
View File

@ -0,0 +1,77 @@
{
"name": "job",
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test1.py"
},
"task_key": "key1"
},
{
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test2.py"
},
"task_key": "key2"
}
]
}
>>> errcode $CLI bundle validate -o json -t staging
Error: file ./test1.py not found
Exit code: 1
{
"name": "job",
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test1.py"
},
"task_key": "key1"
},
{
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test3.py"
},
"task_key": "key2"
}
]
}
>>> errcode $CLI bundle validate -t staging
Error: file ./test1.py not found
Name: override_job_tasks
Target: staging
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/override_job_tasks/staging
Found 1 error
Exit code: 1

3
acceptance/bundle/override/job_tasks/script Normal file
View File

@ -0,0 +1,3 @@
trace errcode $CLI bundle validate -o json -t development 2> out.development.stderr.txt | jq .resources.jobs.foo
trace errcode $CLI bundle validate -o json -t staging | jq .resources.jobs.foo
trace errcode $CLI bundle validate -t staging

13
acceptance/bundle/override/merge-string-map/databricks.yml Normal file
View File

@ -0,0 +1,13 @@
bundle:
name: merge-string-map
resources:
clusters:
my_cluster: "hello"
targets:
dev:
resources:
clusters:
my_cluster:
spark_version: "25"

27
acceptance/bundle/override/merge-string-map/output.txt Normal file
View File

@ -0,0 +1,27 @@
>>> $CLI bundle validate -o json -t dev
Warning: expected map, found string
at resources.clusters.my_cluster
in databricks.yml:6:17
{
"clusters": {
"my_cluster": {
"custom_tags": {},
"spark_version": "25"
}
}
}
>>> $CLI bundle validate -t dev
Warning: expected map, found string
at resources.clusters.my_cluster
in databricks.yml:6:17
Name: merge-string-map
Target: dev
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/merge-string-map/dev
Found 1 warning

2
acceptance/bundle/override/merge-string-map/script Normal file
View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t dev | jq .resources
trace $CLI bundle validate -t dev

3
bundle/tests/override_pipeline_cluster/databricks.yml → acceptance/bundle/override/pipeline_cluster/databricks.yml
View File

@ -1,9 +1,6 @@
bundle:
name: override_pipeline_cluster
workspace:
host: https://acme.cloud.databricks.com/
resources:
pipelines:
foo:

44
acceptance/bundle/override/pipeline_cluster/output.txt Normal file
View File

@ -0,0 +1,44 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"clusters": [
{
"label": "default",
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_conf": {
"foo": "bar"
}
}
],
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/development/state/metadata.json"
},
"name": "job",
"permissions": []
}
}
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"clusters": [
{
"label": "default",
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_conf": {
"foo": "bar"
}
}
],
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/staging/state/metadata.json"
},
"name": "job",
"permissions": []
}
}

2
acceptance/bundle/override/pipeline_cluster/script Normal file
View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t development | jq .resources.pipelines
trace $CLI bundle validate -o json -t staging | jq .resources.pipelines

0
bundle/tests/undefined_resources/databricks.yml → acceptance/bundle/undefined_resources/databricks.yml
View File

19
acceptance/bundle/undefined_resources/output.txt Normal file
View File

@ -0,0 +1,19 @@
Error: experiment undefined-experiment is not defined
at resources.experiments.undefined-experiment
in databricks.yml:11:26
Error: job undefined-job is not defined
at resources.jobs.undefined-job
in databricks.yml:6:19
Error: pipeline undefined-pipeline is not defined
at resources.pipelines.undefined-pipeline
in databricks.yml:14:24
Found 3 errors
Name: undefined-job
Target: default
Exit code: 1

2
acceptance/bundle/undefined_resources/script Normal file
View File

@ -0,0 +1,2 @@
# We need sort_blocks.py because the order of diagnostics is currently randomized
$CLI bundle validate 2>&1 | sort_blocks.py

27
acceptance/bundle/variables/complex-simple/databricks.yml Normal file
View File

@ -0,0 +1,27 @@
# This example works and properly merges resources.jobs.job1.job_clusters.new_cluster and ${var.cluster}.
# retaining num_workers, spark_version and overriding node_type_id.
bundle:
name: TestResolveComplexVariable
variables:
cluster:
type: "complex"
value:
node_type_id: "Standard_DS3_v2"
num_workers: 2
resources:
jobs:
job1:
job_clusters:
- new_cluster:
node_type_id: "random"
spark_version: 13.3.x-scala2.12
targets:
dev:
resources:
jobs:
job1:
job_clusters:
- new_cluster: ${var.cluster}

10
acceptance/bundle/variables/complex-simple/output.txt Normal file
View File

@ -0,0 +1,10 @@
[
{
"job_cluster_key": "",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"spark_version": "13.3.x-scala2.12"
}
}
]

1
acceptance/bundle/variables/complex-simple/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters

22
acceptance/bundle/variables/complex-transitive-deeper/databricks.yml Normal file
View File

@ -0,0 +1,22 @@
bundle:
name: complex-transitive-deeper
variables:
catalog_1:
default:
name: hive_metastore
catalog:
default: ${var.catalog_1}
spark_conf:
default:
"spark.databricks.sql.initial.catalog.name": ${var.catalog.name}
etl_cluster_config:
type: complex
default:
spark_version: 14.3.x-scala2.12
runtime_engine: PHOTON
spark_conf: ${var.spark_conf}
resources:
clusters:
my_cluster: ${var.etl_cluster_config}

7
acceptance/bundle/variables/complex-transitive-deeper/output.txt Normal file
View File

@ -0,0 +1,7 @@
Error: expected a map to index "variables.catalog.value.name", found string
{
"my_cluster": "${var.etl_cluster_config}"
}
Exit code: 1

2
acceptance/bundle/variables/complex-transitive-deeper/script Normal file
View File

@ -0,0 +1,2 @@
# Currently, this errors instead of interpolating variables
$CLI bundle validate -o json | jq '.resources.clusters'

19
acceptance/bundle/variables/complex-transitive/databricks.yml Normal file
View File

@ -0,0 +1,19 @@
bundle:
name: complex-transitive
variables:
catalog:
default: hive_metastore
spark_conf:
default:
"spark.databricks.sql.initial.catalog.name": ${var.catalog}
etl_cluster_config:
type: complex
default:
spark_version: 14.3.x-scala2.12
runtime_engine: PHOTON
spark_conf: ${var.spark_conf}
resources:
clusters:
my_cluster: ${var.etl_cluster_config}

3
acceptance/bundle/variables/complex-transitive/output.txt Normal file
View File

@ -0,0 +1,3 @@
{
"spark.databricks.sql.initial.catalog.name": "hive_metastore"
}

2
acceptance/bundle/variables/complex-transitive/script Normal file
View File

@ -0,0 +1,2 @@
# Currently, this incorrectly outputs variable reference instead of resolved value
$CLI bundle validate -o json | jq '.resources.clusters.my_cluster.spark_conf'

17
acceptance/bundle/variables/complex-with-var-reference/databricks.yml Normal file
View File

@ -0,0 +1,17 @@
bundle:
name: TestResolveComplexVariableWithVarReference
variables:
package_version:
default: "1.0.0"
cluster_libraries:
type: "complex"
default:
- pypi:
package: "cicd_template==${var.package_version}"
resources:
jobs:
job1:
tasks:
- libraries: ${var.cluster_libraries}

12
acceptance/bundle/variables/complex-with-var-reference/output.txt Normal file
View File

@ -0,0 +1,12 @@
[
{
"libraries": [
{
"pypi": {
"package": "cicd_template==1.0.0"
}
}
],
"task_key": ""
}
]

1
acceptance/bundle/variables/complex-with-var-reference/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.tasks

34
acceptance/bundle/variables/complex-within-complex/databricks.yml Normal file
View File

@ -0,0 +1,34 @@
# Does not work currently, explicitly disabled, even though it works if you remove 'type: "complex"' lines
# Also fails to merge clusters.
bundle:
name: TestResolveComplexVariableReferencesWithComplexVariablesError
variables:
cluster:
type: "complex"
value:
node_type_id: "Standard_DS3_v2"
num_workers: 2
spark_conf: "${var.spark_conf}"
spark_conf:
type: "complex"
value:
spark.executor.memory: "4g"
spark.executor.cores: "2"
resources:
jobs:
job1:
job_clusters:
- job_cluster_key: my_cluster
new_cluster:
node_type_id: "random"
targets:
dev:
resources:
jobs:
job1:
job_clusters:
- job_cluster_key: my_cluster
new_cluster: ${var.cluster}

17
acceptance/bundle/variables/complex-within-complex/output.txt Normal file
View File

@ -0,0 +1,17 @@
Warning: unknown field: node_type_id
at resources.jobs.job1.job_clusters[0]
in databricks.yml:25:11
[
{
"job_cluster_key": "my_cluster",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"spark_conf": {
"spark.executor.cores": "2",
"spark.executor.memory": "4g"
}
}
}
]

1
acceptance/bundle/variables/complex-within-complex/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters

20
bundle/tests/variables/complex/databricks.yml → acceptance/bundle/variables/complex/databricks.yml
View File

@ -11,6 +11,7 @@ resources:
- task_key: test
job_cluster_key: key
libraries: ${variables.libraries.value}
# specific fields of complex variable are referenced:
task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"
variables:
@ -35,30 +36,21 @@ variables:
- jar: "/path/to/jar"
- egg: "/path/to/egg"
- whl: "/path/to/whl"
complexvar:
type: complex
description: "A complex variable"
default:
key1: "value1"
key2: "value2"
key3: "value3"
targets:
default:
default: true
dev:
variables:
node_type: "Standard_DS3_v3"
cluster:
# complex variables are not merged, so missing variables (policy_id) are not inherited
spark_version: "14.2.x-scala2.11"
node_type_id: ${var.node_type}
num_workers: 4
spark_conf:
spark.speculation: false
spark.databricks.delta.retentionDurationCheck.enabled: false
complexvar:
type: complex
default:
key1: "1"
key2: "2"
key3: "3"
libraries:
- jar: "/newpath/to/jar"
- whl: "/newpath/to/whl"

110
acceptance/bundle/variables/complex/out.default.json Normal file
View File

@ -0,0 +1,110 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.random": "true",
"spark.speculation": "true"
},
"spark_version": "13.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"job_cluster_key": "key",
"libraries": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
],
"task_key": "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
}
]
}
}
},
"variables": {
"cluster": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.random": true,
"spark.speculation": true
},
"spark_version": "13.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.random": true,
"spark.speculation": true
},
"spark_version": "13.2.x-scala2.11"
}
},
"libraries": {
"default": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
],
"description": "A libraries definition",
"type": "complex",
"value": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
]
},
"node_type": {
"default": "Standard_DS3_v2",
"value": "Standard_DS3_v2"
}
}
}

95
acceptance/bundle/variables/complex/out.dev.json Normal file
View File

@ -0,0 +1,95 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"job_cluster_key": "key",
"libraries": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
],
"task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
}
]
}
}
},
"variables": {
"cluster": {
"default": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"libraries": {
"default": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
],
"description": "A libraries definition",
"type": "complex",
"value": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
]
},
"node_type": {
"default": "Standard_DS3_v3",
"value": "Standard_DS3_v3"
}
}
}

14
acceptance/bundle/variables/complex/output.txt Normal file
View File

@ -0,0 +1,14 @@
>>> $CLI bundle validate -o json
>>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json
"task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
>>> $CLI bundle validate -o json -t dev
>>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
policy_id and spark_conf.spark_random fields do not exist in dev target:
>>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json
null

8
acceptance/bundle/variables/complex/script Normal file
View File

@ -0,0 +1,8 @@
trace $CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
trace jq .resources.jobs.my_job.tasks[0].task_key out.default.json | grep "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
trace $CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
trace jq .resources.jobs.my_job.tasks[0].task_key out.dev.json | grep "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
echo policy_id and spark_conf.spark_random fields do not exist in dev target:
trace jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json | grep null

0
bundle/tests/variables/complex_multiple_files/databricks.yml → acceptance/bundle/variables/complex_multiple_files/databricks.yml
View File

159
acceptance/bundle/variables/complex_multiple_files/output.txt Normal file
View File

@ -0,0 +1,159 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key1",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key2",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key3",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key4",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
},
"variables": {
"cluster1": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster2": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster3": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster4": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
}
}
}

1
acceptance/bundle/variables/complex_multiple_files/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -t dev -o json | jq '{resources, variables}'

0
bundle/tests/variables/complex_multiple_files/variables/clusters.yml → acceptance/bundle/variables/complex_multiple_files/variables/clusters.yml
View File

0
bundle/tests/variables/empty/databricks.yml → acceptance/bundle/variables/empty/databricks.yml
View File

11
acceptance/bundle/variables/empty/output.txt Normal file
View File

@ -0,0 +1,11 @@
Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
Name: empty${var.a}
Target: default
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/empty${var.a}/default
Found 1 error
Exit code: 1

1
acceptance/bundle/variables/empty/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate

0
bundle/tests/variables/env_overrides/databricks.yml → acceptance/bundle/variables/env_overrides/databricks.yml
View File

40
acceptance/bundle/variables/env_overrides/output.txt Normal file
View File

@ -0,0 +1,40 @@
>>> $CLI bundle validate -t env-with-single-variable-override -o json
"default-a dev-b"
>>> $CLI bundle validate -t env-with-two-variable-overrides -o json
"prod-a prod-b"
>>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json
"prod-a env-var-b"
>>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
Name: test bundle
Target: env-missing-a-required-variable-assignment
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/test bundle/env-missing-a-required-variable-assignment
Found 1 error
Exit code: 1
>>> errcode $CLI bundle validate -t env-using-an-undefined-variable
Error: variable c is not defined but is assigned a value
Name: test bundle
Found 1 error
Exit code: 1
>>> $CLI bundle validate -t env-overrides-lookup -o json
{
"a": "default-a",
"b": "prod-b",
"d": "4321",
"e": "1234",
"f": "9876"
}

6
acceptance/bundle/variables/env_overrides/script Normal file
View File

@ -0,0 +1,6 @@
trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile
trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
trace errcode $CLI bundle validate -t env-using-an-undefined-variable
trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)'

19
acceptance/bundle/variables/git-branch/databricks.yml Normal file
View File

@ -0,0 +1,19 @@
bundle:
name: git
git:
# This is currently not supported
branch: ${var.deployment_branch}
variables:
deployment_branch:
# By setting deployment_branch to "" we set bundle.git.branch to "" which is the same unsetting it.
# This this should make CLI read branch from git and update bundle.git.branch accordingly. It should
# Also set bundle.git.inferred to true.
default: ""
targets:
prod:
default: true
dev:
variables:
deployment_branch: dev-branch

98
acceptance/bundle/variables/git-branch/output.txt Normal file
View File

@ -0,0 +1,98 @@
>>> $CLI bundle validate -o json
{
"bundle": {
"environment": "prod",
"git": {
"actual_branch": "main",
"branch": "",
"bundle_root_path": ".",
},
"name": "git",
"target": "prod",
"terraform": {
"exec_path": "$TMPHOME"
}
},
"sync": {
"paths": [
"."
]
},
"targets": null,
"variables": {
"deployment_branch": {
"default": "",
"value": ""
}
},
"workspace": {
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
"current_user": {
"short_name": "$USERNAME",
"userName": "$USERNAME"
},
"file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files",
"resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources",
"root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod",
"state_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/state"
}
}
>>> $CLI bundle validate
Name: git
Target: prod
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/git/prod
Validation OK!
>>> $CLI bundle validate -o json -t dev
{
"bundle": {
"environment": "dev",
"git": {
"actual_branch": "main",
"branch": "dev-branch",
"bundle_root_path": ".",
},
"name": "git",
"target": "dev",
"terraform": {
"exec_path": "$TMPHOME"
}
},
"sync": {
"paths": [
"."
]
},
"targets": null,
"variables": {
"deployment_branch": {
"default": "dev-branch",
"value": "dev-branch"
}
},
"workspace": {
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
"current_user": {
"short_name": "$USERNAME",
"userName": "$USERNAME"
},
"file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files",
"resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources",
"root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev",
"state_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/state"
}
}
>>> $CLI bundle validate -t dev
Name: git
Target: dev
Workspace:
User: $USERNAME
Path: /Workspace/Users/$USERNAME/.bundle/git/dev
Validation OK!

6
acceptance/bundle/variables/git-branch/script Normal file
View File

@ -0,0 +1,6 @@
git-repo-init
trace $CLI bundle validate -o json | grep -v '"commit"'
trace $CLI bundle validate
trace $CLI bundle validate -o json -t dev | grep -v '"commit"'
trace $CLI bundle validate -t dev | grep -v '"commit"'
rm -fr .git

10
acceptance/bundle/variables/host/databricks.yml Normal file
View File

@ -0,0 +1,10 @@
bundle:
name: host
variables:
host:
default: https://nonexistent123.staging.cloud.databricks.com
workspace:
# This is currently not supported
host: ${var.host}

38
acceptance/bundle/variables/host/output.txt Normal file
View File

@ -0,0 +1,38 @@
>>> errcode $CLI bundle validate -o json
Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
{
"bundle": {
"environment": "default",
"name": "host",
"target": "default"
},
"sync": {
"paths": [
"."
]
},
"targets": null,
"variables": {
"host": {
"default": "https://nonexistent123.staging.cloud.databricks.com"
}
},
"workspace": {
"host": "${var.host}"
}
}
Exit code: 1
>>> errcode $CLI bundle validate
Error: failed during request visitor: parse "https://${var.host}": invalid character "{" in host name
Name: host
Target: default
Workspace:
Host: ${var.host}
Found 1 error
Exit code: 1

2
acceptance/bundle/variables/host/script Normal file
View File

@ -0,0 +1,2 @@
trace errcode $CLI bundle validate -o json
trace errcode $CLI bundle validate

6
acceptance/bundle/variables/resolve-builtin/databricks.yml Normal file
View File

@ -0,0 +1,6 @@
bundle:
name: TestResolveVariableReferences
workspace:
root_path: "${bundle.name}/bar"
file_path: "${workspace.root_path}/baz"

11
acceptance/bundle/variables/resolve-builtin/output.txt Normal file
View File

@ -0,0 +1,11 @@
{
"artifact_path": "TestResolveVariableReferences/bar/artifacts",
"current_user": {
"short_name": "$USERNAME",
"userName": "$USERNAME"
},
"file_path": "TestResolveVariableReferences/bar/baz",
"resource_path": "TestResolveVariableReferences/bar/resources",
"root_path": "TestResolveVariableReferences/bar",
"state_path": "TestResolveVariableReferences/bar/state"
}

1
acceptance/bundle/variables/resolve-builtin/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .workspace

10
acceptance/bundle/variables/resolve-empty/databricks.yml Normal file
View File

@ -0,0 +1,10 @@
bundle:
name: TestResolveVariableReferencesToEmptyFields
git:
branch: ""
resources:
jobs:
job1:
tags:
git_branch: "${bundle.git.branch}"

3
acceptance/bundle/variables/resolve-empty/output.txt Normal file
View File

@ -0,0 +1,3 @@
{
"git_branch": ""
}

1
acceptance/bundle/variables/resolve-empty/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.tags

16
acceptance/bundle/variables/resolve-field-within-complex/databricks.yml Normal file
View File

@ -0,0 +1,16 @@
bundle:
name: TestResolveComplexVariableReferencesToFields
variables:
cluster:
type: "complex"
default:
node_type_id: "Standard_DS3_v2"
num_workers: 2
resources:
jobs:
job1:
job_clusters:
- new_cluster:
node_type_id: "${var.cluster.node_type_id}"

3
acceptance/bundle/variables/resolve-field-within-complex/output.txt Normal file
View File

@ -0,0 +1,3 @@
{
"node_type_id": "Standard_DS3_v2"
}

1
acceptance/bundle/variables/resolve-field-within-complex/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters[0].new_cluster

23
acceptance/bundle/variables/resolve-nonstrings/databricks.yml Normal file
View File

@ -0,0 +1,23 @@
bundle:
name: TestResolveVariableReferencesForPrimitiveNonStringFields
variables:
no_alert_for_canceled_runs: {}
no_alert_for_skipped_runs: {}
min_workers: {}
max_workers: {}
spot_bid_max_price: {}
resources:
jobs:
job1:
notification_settings:
no_alert_for_canceled_runs: ${var.no_alert_for_canceled_runs}
no_alert_for_skipped_runs: ${var.no_alert_for_skipped_runs}
tasks:
- new_cluster:
autoscale:
min_workers: ${var.min_workers}
max_workers: ${var.max_workers}
azure_attributes:
spot_bid_max_price: ${var.spot_bid_max_price}

52
acceptance/bundle/variables/resolve-nonstrings/output.txt Normal file
View File

@ -0,0 +1,52 @@
{
"variables": {
"max_workers": {
"value": "2"
},
"min_workers": {
"value": "1"
},
"no_alert_for_canceled_runs": {
"value": "true"
},
"no_alert_for_skipped_runs": {
"value": "false"
},
"spot_bid_max_price": {
"value": "0.5"
}
},
"jobs": {
"job1": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/TestResolveVariableReferencesForPrimitiveNonStringFields/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"notification_settings": {
"no_alert_for_canceled_runs": true,
"no_alert_for_skipped_runs": false
},
"permissions": [],
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"new_cluster": {
"autoscale": {
"max_workers": 2,
"min_workers": 1
},
"azure_attributes": {
"spot_bid_max_price": 0.5
}
},
"task_key": ""
}
]
}
}
}

Some files were not shown because too many files have changed in this diff Show More