Merge remote-tracking branch 'origin/main' into cp-encourage-root-path

This commit is contained in:
Lennart Kats 2025-01-13 10:56:53 +01:00
commit 3b63fb20d4
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
693 changed files with 14882 additions and 5406 deletions

View File

@ -11,7 +11,7 @@
"required": ["go"],
"post_generate": [
"go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
"go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
"make schema",
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
"echo 'go.sum linguist-generated=true' >> ./.gitattributes",
"echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"

View File

@ -1 +1 @@
f2385add116e3716c8a90a0b68e204deb40f996c
a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d

View File

@ -411,5 +411,5 @@ func new{{.PascalName}}() *cobra.Command {
{{- define "request-body-obj" -}}
{{- $method := .Method -}}
{{- $field := .Field -}}
{{$method.CamelName}}Req{{ if (and $method.RequestBodyField (not $field.IsPath)) }}.{{$method.RequestBodyField.PascalName}}{{end}}.{{$field.PascalName}}
{{$method.CamelName}}Req{{ if (and $method.RequestBodyField (and (not $field.IsPath) (not $field.IsQuery))) }}.{{$method.RequestBodyField.PascalName}}{{end}}.{{$field.PascalName}}
{{- end -}}

8
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,8 @@
# Enable gofumpt and goimports in golangci-lint (#1999)
2e018cfaec200a02ee2bd5b389e7da3c6f15f460
# Enable errcheck everywhere and fix or silent remaining issues (#1987)
8d5351c1c3d7befda4baae5d6adb99367aa50b3c
# Add error checking in tests and enable errcheck there (#1980)
1b2be1b2cb4b7909df2a8ad4cb6a0f43e8fcf0c6

5
.gitattributes vendored
View File

@ -8,6 +8,7 @@ cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=
cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
cmd/account/federation-policy/federation-policy.go linguist-generated=true
cmd/account/groups/groups.go linguist-generated=true
cmd/account/ip-access-lists/ip-access-lists.go linguist-generated=true
cmd/account/log-delivery/log-delivery.go linguist-generated=true
@ -19,6 +20,7 @@ cmd/account/o-auth-published-apps/o-auth-published-apps.go linguist-generated=tr
cmd/account/personal-compute/personal-compute.go linguist-generated=true
cmd/account/private-access/private-access.go linguist-generated=true
cmd/account/published-app-integration/published-app-integration.go linguist-generated=true
cmd/account/service-principal-federation-policy/service-principal-federation-policy.go linguist-generated=true
cmd/account/service-principal-secrets/service-principal-secrets.go linguist-generated=true
cmd/account/service-principals/service-principals.go linguist-generated=true
cmd/account/settings/settings.go linguist-generated=true
@ -37,6 +39,9 @@ cmd/workspace/apps/apps.go linguist-generated=true
cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true
cmd/workspace/automatic-cluster-update/automatic-cluster-update.go linguist-generated=true
cmd/workspace/catalogs/catalogs.go linguist-generated=true
cmd/workspace/clean-room-assets/clean-room-assets.go linguist-generated=true
cmd/workspace/clean-room-task-runs/clean-room-task-runs.go linguist-generated=true
cmd/workspace/clean-rooms/clean-rooms.go linguist-generated=true
cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true
cmd/workspace/clusters/clusters.go linguist-generated=true
cmd/workspace/cmd.go linguist-generated=true

1
.github/CODEOWNERS vendored Normal file
View File

@ -0,0 +1 @@
* @pietern @andrewnester @shreyas-goenka @denik

View File

@ -4,3 +4,7 @@ updates:
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"

View File

@ -0,0 +1,38 @@
name: "Close Stale Issues"
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *" # Run at midnight every day
jobs:
cleanup:
name: Stale issue job
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
permissions:
issues: write
contents: read
pull-requests: write
steps:
- uses: actions/stale@v9
with:
stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
# These labels are required
stale-issue-label: Stale
stale-pr-label: Stale
exempt-issue-labels: No Autoclose
exempt-pr-labels: No Autoclose
# Issue timing
days-before-stale: 60
days-before-close: 30
repo-token: ${{ secrets.GITHUB_TOKEN }}
loglevel: DEBUG

View File

@ -13,10 +13,17 @@ on:
jobs:
comment-on-pr:
runs-on: ubuntu-latest
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
permissions:
pull-requests: write
# Only run this job for PRs from forks.
# Integration tests are not run automatically for PRs from forks.
if: "${{ github.event.pull_request.head.repo.fork }}"
steps:
- uses: actions/checkout@v4
@ -43,7 +50,7 @@ jobs:
run: |
gh pr comment ${{ github.event.pull_request.number }} --body \
"<!-- INTEGRATION_TESTS_MANUAL -->
If integration tests don't run automatically, an authorized user can run them manually by following the instructions below:
An authorized user can trigger integration tests manually by following the instructions below:
Trigger:
[go/deco-tests-run/cli](https://go/deco-tests-run/cli)

View File

@ -0,0 +1,34 @@
name: integration-approve
on:
merge_group:
jobs:
# Trigger for merge groups.
#
# Statuses and checks apply to specific commits (by hash).
# Enforcement of required checks is done both at the PR level and the merge queue level.
# In case of multiple commits in a single PR, the hash of the squashed commit
# will not match the one for the latest (approved) commit in the PR.
#
# We auto approve the check for the merge queue for two reasons:
#
# * Queue times out due to duration of tests.
# * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
#
trigger:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
steps:
- name: Auto-approve squashed commit
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
run: |
gh api -X POST -H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/statuses/${{ github.sha }} \
-f 'state=success' \
-f 'context=Integration Tests Check'

36
.github/workflows/integration-main.yml vendored Normal file
View File

@ -0,0 +1,36 @@
name: integration-main
on:
push:
branches:
- main
jobs:
# Trigger for pushes to the main branch.
#
# This workflow triggers the integration test workflow in a different repository.
# It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
trigger:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
environment: "test-trigger-is"
steps:
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
owner: ${{ secrets.ORG_NAME }}
repositories: ${{secrets.REPO_NAME}}
- name: Trigger Workflow in Another Repo
env:
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
run: |
gh workflow run cli-isolated-nightly.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
--ref main \
-f commit_sha=${{ github.event.after }}

40
.github/workflows/integration-pr.yml vendored Normal file
View File

@ -0,0 +1,40 @@
name: integration-pr
on:
pull_request:
types: [opened, synchronize]
jobs:
# Trigger for pull requests.
#
# This workflow triggers the integration test workflow in a different repository.
# It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
trigger:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
environment: "test-trigger-is"
# Only run this job for PRs from branches on the main repository and not from forks.
# Workflows triggered by PRs from forks don't have access to the "test-trigger-is" environment.
if: "${{ !github.event.pull_request.head.repo.fork }}"
steps:
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
owner: ${{ secrets.ORG_NAME }}
repositories: ${{secrets.REPO_NAME}}
- name: Trigger Workflow in Another Repo
env:
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
run: |
gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
--ref main \
-f pull_request_number=${{ github.event.pull_request.number }} \
-f commit_sha=${{ github.event.pull_request.head.sha }}

View File

@ -1,78 +0,0 @@
name: integration
on:
pull_request:
types: [opened, synchronize]
merge_group:
jobs:
check-token:
runs-on: ubuntu-latest
environment: "test-trigger-is"
outputs:
has_token: ${{ steps.set-token-status.outputs.has_token }}
steps:
- name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set
id: set-token-status
run: |
if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then
echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets."
echo "::set-output name=has_token::false"
else
echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
echo "::set-output name=has_token::true"
fi
trigger-tests:
runs-on: ubuntu-latest
needs: check-token
if: github.event_name == 'pull_request' && needs.check-token.outputs.has_token == 'true'
environment: "test-trigger-is"
steps:
- uses: actions/checkout@v4
- name: Generate GitHub App Token
id: generate-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
owner: ${{ secrets.ORG_NAME }}
repositories: ${{secrets.REPO_NAME}}
- name: Trigger Workflow in Another Repo
env:
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
run: |
gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
--ref main \
-f pull_request_number=${{ github.event.pull_request.number }} \
-f commit_sha=${{ github.event.pull_request.head.sha }}
# Statuses and checks apply to specific commits (by hash).
# Enforcement of required checks is done both at the PR level and the merge queue level.
# In case of multiple commits in a single PR, the hash of the squashed commit
# will not match the one for the latest (approved) commit in the PR.
# We auto approve the check for the merge queue for two reasons:
# * Queue times out due to duration of tests.
# * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
auto-approve:
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- name: Mark Check
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
run: |
gh api -X POST -H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/statuses/${{ github.sha }} \
-f 'state=success' \
-f 'context=Integration Tests Check'

View File

@ -2,11 +2,19 @@ name: publish-winget
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to publish'
default: ''
jobs:
publish-to-winget-pkgs:
runs-on: windows-latest
runs-on:
group: databricks-protected-runner-group
labels: windows-server-latest
environment: release
steps:
- uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
with:
@ -14,3 +22,7 @@ jobs:
installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
fork-user: eng-dev-ecosystem-bot
# Use the tag from the input, or the ref name if the input is not provided.
# The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
release-tag: ${{ inputs.tag || github.ref_name }}

View File

@ -13,9 +13,26 @@ on:
# seed the build cache.
branches:
- main
schedule:
- cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily
env:
GOTESTSUM_FORMAT: github-actions
jobs:
cleanups:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
steps:
- name: Clean up cache if running on schedule
if: ${{ github.event_name == 'schedule' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh cache delete --all --repo databricks/cli || true
tests:
needs: cleanups
runs-on: ${{ matrix.os }}
strategy:
@ -33,18 +50,21 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.23.2
go-version: 1.23.4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Install uv
uses: astral-sh/setup-uv@v5
- name: Set go env
run: |
echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
go install gotest.tools/gotestsum@latest
go install gotest.tools/gotestsum@v1.12.0
- name: Pull external libraries
run: |
@ -52,58 +72,36 @@ jobs:
pip3 install wheel
- name: Run tests
run: make testonly
- name: Publish test coverage
uses: codecov/codecov-action@v4
fmt:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.23.2
# No need to download cached dependencies when running gofmt.
cache: false
- name: Install goimports
run: |
go install golang.org/x/tools/cmd/goimports@latest
- name: Run make fmt
run: |
make fmt
- name: Run go mod tidy
run: |
go mod tidy
- name: Fail on differences
run: |
# Exit with status code 1 if there are differences (i.e. unformatted files)
git diff --exit-code
run: make test
golangci:
needs: cleanups
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: 1.23.2
go-version: 1.23.4
# Use different schema from regular job, to avoid overwriting the same key
cache-dependency-path: |
go.sum
.golangci.yaml
- name: Run go mod tidy
run: |
go mod tidy
- name: Fail on differences
run: |
# Exit with status code 1 if there are differences (i.e. unformatted files)
git diff --exit-code
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
version: v1.62.2
version: v1.63.4
args: --timeout=15m
validate-bundle-schema:
needs: cleanups
runs-on: ubuntu-latest
steps:
@ -113,7 +111,18 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.23.2
go-version: 1.23.4
# Use different schema from regular job, to avoid overwriting the same key
cache-dependency-path: |
go.sum
bundle/internal/schema/*.*
- name: Verify that the schema is up to date
run: |
if ! ( make schema && git diff --exit-code ); then
echo "The schema is not up to date. Please run 'make schema' and commit the changes."
exit 1
fi
# Github repo: https://github.com/ajv-validator/ajv-cli
- name: Install ajv-cli
@ -124,14 +133,19 @@ jobs:
# By default the ajv-cli runs in strict mode which will fail if the schema
# itself is not valid. Strict mode is more strict than the JSON schema
# specification. See for details: https://ajv.js.org/options.html#strict-mode-options
# The ajv-cli is configured to use the markdownDescription keyword which is not part of the JSON schema specification,
# but is used in editors like VSCode to render markdown in the description field
- name: Validate bundle schema
run: |
go run main.go bundle schema > schema.json
# Add markdownDescription keyword to ajv
echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js
for file in ./bundle/internal/schema/testdata/pass/*.yml; do
ajv test -s schema.json -d $file --valid
ajv test -s schema.json -d $file --valid -c=./keywords.js
done
for file in ./bundle/internal/schema/testdata/fail/*.yml; do
ajv test -s schema.json -d $file --invalid
ajv test -s schema.json -d $file --invalid -c=./keywords.js
done

View File

@ -20,7 +20,10 @@ on:
jobs:
goreleaser:
runs-on: ubuntu-latest
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
@ -31,7 +34,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.23.2
go-version: 1.23.4
# The default cache key for this action considers only the `go.sum` file.
# We include .goreleaser.yaml here to differentiate from the cache used by the push action

View File

@ -9,9 +9,13 @@ on:
jobs:
goreleaser:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
outputs:
artifacts: ${{ steps.releaser.outputs.artifacts }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository and submodules
uses: actions/checkout@v4
@ -22,7 +26,7 @@ jobs:
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.23.2
go-version: 1.23.4
# The default cache key for this action considers only the `go.sum` file.
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
@ -54,8 +58,12 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
create-setup-cli-release-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |
@ -78,8 +86,12 @@ jobs:
});
create-homebrew-tap-release-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |
@ -115,8 +127,12 @@ jobs:
});
create-vscode-extension-update-pr:
runs-on:
group: databricks-deco-testing-runner-group
labels: ubuntu-latest-deco
needs: goreleaser
runs-on: ubuntu-latest
steps:
- name: Set VERSION variable from tag
run: |

View File

@ -2,21 +2,46 @@ linters:
disable-all: true
enable:
- bodyclose
# errcheck and govet are part of default setup and should be included but give too many errors now
# once errors are fixed, they should be enabled here:
#- errcheck
- errcheck
- gosimple
#- govet
- govet
- ineffassign
- staticcheck
- unused
- gofmt
- gofumpt
- goimports
- testifylint
- intrange
- mirror
- perfsprint
linters-settings:
govet:
enable-all: true
disable:
- fieldalignment
- shadow
gofmt:
rewrite-rules:
- pattern: 'a[b:len(a)]'
replacement: 'a[b:]'
- pattern: 'interface{}'
replacement: 'any'
errcheck:
exclude-functions:
- (*github.com/spf13/cobra.Command).RegisterFlagCompletionFunc
- (*github.com/spf13/cobra.Command).MarkFlagRequired
- (*github.com/spf13/pflag.FlagSet).MarkDeprecated
- (*github.com/spf13/pflag.FlagSet).MarkHidden
gofumpt:
module-path: github.com/databricks/cli
extra-rules: true
testifylint:
enable-all: true
disable:
# good check, but we have too many assert.(No)?Errorf? so excluding for now
- require-error
issues:
exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/
max-issues-per-linter: 1000
max-same-issues: 1000

View File

@ -7,11 +7,14 @@
"go.lintFlags": [
"--fast"
],
"go.useLanguageServer": true,
"gopls": {
"formatting.gofumpt": true
},
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true,
"files.trimFinalNewlines": true,
"python.envFile": "${workspaceRoot}/.env",
"databricks.python.envFile": "${workspaceFolder}/.env",
"python.analysis.stubPath": ".vscode",
"jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
"jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------"

View File

@ -1,5 +1,37 @@
# Version changelog
## [Release] Release v0.238.0
Bundles:
* Fix finding Python within virtualenv on Windows ([#2034](https://github.com/databricks/cli/pull/2034)).
* Include missing field descriptions in JSON schema ([#2045](https://github.com/databricks/cli/pull/2045)).
* Add validation for volume referenced from `artifact_path` ([#2050](https://github.com/databricks/cli/pull/2050)).
* Handle `${workspace.file_path}` references in source-linked deployments ([#2046](https://github.com/databricks/cli/pull/2046)).
* Set the write bit for files written during template initialization ([#2068](https://github.com/databricks/cli/pull/2068)).
## [Release] Release v0.237.0
Bundles:
* Allow overriding compute for non-development mode targets ([#1899](https://github.com/databricks/cli/pull/1899)).
* Show an error when using a cluster override with 'mode: production' ([#1994](https://github.com/databricks/cli/pull/1994)).
API Changes:
* Added `databricks account federation-policy` command group.
* Added `databricks account service-principal-federation-policy` command group.
* Added `databricks aibi-dashboard-embedding-access-policy delete` command.
* Added `databricks aibi-dashboard-embedding-approved-domains delete` command.
OpenAPI commit a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d (2024-12-16)
Dependency updates:
* Upgrade TF provider to 1.62.0 ([#2030](https://github.com/databricks/cli/pull/2030)).
* Upgrade Go SDK to 0.54.0 ([#2029](https://github.com/databricks/cli/pull/2029)).
* Bump TF codegen dependencies to latest ([#1961](https://github.com/databricks/cli/pull/1961)).
* Bump golang.org/x/term from 0.26.0 to 0.27.0 ([#1983](https://github.com/databricks/cli/pull/1983)).
* Bump golang.org/x/sync from 0.9.0 to 0.10.0 ([#1984](https://github.com/databricks/cli/pull/1984)).
* Bump github.com/databricks/databricks-sdk-go from 0.52.0 to 0.53.0 ([#1985](https://github.com/databricks/cli/pull/1985)).
* Bump golang.org/x/crypto from 0.24.0 to 0.31.0 ([#2006](https://github.com/databricks/cli/pull/2006)).
* Bump golang.org/x/crypto from 0.30.0 to 0.31.0 in /bundle/internal/tf/codegen ([#2005](https://github.com/databricks/cli/pull/2005)).
## [Release] Release v0.236.0
**New features for Databricks Asset Bundles:**

View File

@ -1,40 +1,48 @@
default: build
PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
GOTESTSUM_FORMAT ?= pkgname-and-test-fails
lint:
golangci-lint run --fix
lintcheck:
golangci-lint run ./...
# Note 'make lint' will do formatting as well. However, if there are compilation errors,
# formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
# If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
fmt:
@echo "✓ Formatting source code with goimports ..."
@goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
@echo "✓ Formatting source code with gofmt ..."
@gofmt -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
lint: vendor
@echo "✓ Linting source code with https://golangci-lint.run/ ..."
@golangci-lint run ./...
test:
gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
lintfix: vendor
@echo "✓ Linting source code with 'golangci-lint run --fix' ..."
@golangci-lint run --fix ./...
cover:
gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
test: lint testonly
testonly:
@echo "✓ Running tests ..."
@gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./...
coverage: test
@echo "✓ Opening coverage for unit tests ..."
@go tool cover -html=coverage.txt
showcover:
go tool cover -html=coverage.txt
build: vendor
@echo "✓ Building source code with go build ..."
@go build -mod vendor
go build -mod vendor
snapshot:
@echo "✓ Building dev snapshot"
@go build -o .databricks/databricks
go build -o .databricks/databricks
vendor:
@echo "✓ Filling vendor folder with library code ..."
@go mod vendor
go mod vendor
schema:
go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
.PHONY: build vendor coverage test lint fmt
INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
integration:
$(INTEGRATION)
integration-short:
$(INTEGRATION) -short
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short

12
NOTICE
View File

@ -73,10 +73,6 @@ fatih/color - https://github.com/fatih/color
Copyright (c) 2013 Fatih Arslan
License - https://github.com/fatih/color/blob/main/LICENSE.md
ghodss/yaml - https://github.com/ghodss/yaml
Copyright (c) 2014 Sam Ghods
License - https://github.com/ghodss/yaml/blob/master/LICENSE
Masterminds/semver - https://github.com/Masterminds/semver
Copyright (C) 2014-2019, Matt Butcher and Matt Farina
License - https://github.com/Masterminds/semver/blob/master/LICENSE.txt
@ -101,3 +97,11 @@ License - https://github.com/stretchr/testify/blob/master/LICENSE
whilp/git-urls - https://github.com/whilp/git-urls
Copyright (c) 2020 Will Maier
License - https://github.com/whilp/git-urls/blob/master/LICENSE
github.com/wI2L/jsondiff v0.6.1
Copyright (c) 2020-2024 William Poussier <william.poussier@gmail.com>
License - https://github.com/wI2L/jsondiff/blob/master/LICENSE
https://github.com/hexops/gotextdiff
Copyright (c) 2009 The Go Authors. All rights reserved.
License - https://github.com/hexops/gotextdiff/blob/main/LICENSE

19
acceptance/README.md Normal file
View File

@ -0,0 +1,19 @@
Acceptance tests are blackbox tests that are run against compiled binary.
Currently these tests are run against "fake" HTTP server pretending to be Databricks API. However, they will be extended to run against real environment as regular integration tests.
To author a test,
- Add a new directory under `acceptance`. Any level of nesting is supported.
- Add `databricks.yml` there.
- Add `script` with commands to run, e.g. `$CLI bundle validate`. The test case is recognized by presence of `script`.
The test runner will run script and capture output and compare it with `output.txt` file in the same directory.
In order to write `output.txt` for the first time or overwrite it with the current output pass -update flag to go test.
The scripts are run with `bash -e` so any errors will be propagated. They are captured in `output.txt` by appending `Exit code: N` line at the end.
For more complex tests one can also use:
- `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
- `trace` helper: prints the arguments before executing the command.
- custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.

View File

@ -0,0 +1,311 @@
package acceptance_test
import (
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"sort"
"strings"
"testing"
"time"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/testdiff"
"github.com/stretchr/testify/require"
)
var KeepTmp = os.Getenv("KEEP_TMP") != ""
const (
EntryPointScript = "script"
CleanupScript = "script.cleanup"
PrepareScript = "script.prepare"
)
var Scripts = map[string]bool{
EntryPointScript: true,
CleanupScript: true,
PrepareScript: true,
}
func TestAccept(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
execPath := BuildCLI(t, cwd)
// $CLI is what test scripts are using
t.Setenv("CLI", execPath)
// Make helper scripts available
t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
server := StartServer(t)
AddHandlers(server)
// Redirect API access to local server:
t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
t.Setenv("DATABRICKS_TOKEN", "dapi1234")
homeDir := t.TempDir()
// Do not read user's ~/.databrickscfg
t.Setenv(env.HomeEnvVar(), homeDir)
repls := testdiff.ReplacementsContext{}
repls.Set(execPath, "$CLI")
testDirs := getTests(t)
require.NotEmpty(t, testDirs)
for _, dir := range testDirs {
t.Run(dir, func(t *testing.T) {
t.Parallel()
runTest(t, dir, repls)
})
}
}
func getTests(t *testing.T) []string {
testDirs := make([]string, 0, 128)
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
name := filepath.Base(path)
if name == EntryPointScript {
// Presence of 'script' marks a test case in this directory
testDirs = append(testDirs, filepath.Dir(path))
}
return nil
})
require.NoError(t, err)
sort.Strings(testDirs)
return testDirs
}
func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
var tmpDir string
var err error
if KeepTmp {
tempDirBase := filepath.Join(os.TempDir(), "acceptance")
_ = os.Mkdir(tempDirBase, 0o755)
tmpDir, err = os.MkdirTemp(tempDirBase, "")
require.NoError(t, err)
t.Logf("Created directory: %s", tmpDir)
} else {
tmpDir = t.TempDir()
}
scriptContents := readMergedScriptContents(t, dir)
testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
inputs := make(map[string]bool, 2)
outputs := make(map[string]bool, 2)
err = CopyDir(dir, tmpDir, inputs, outputs)
require.NoError(t, err)
args := []string{"bash", "-euo", "pipefail", EntryPointScript}
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = tmpDir
outB, err := cmd.CombinedOutput()
out := formatOutput(string(outB), err)
out = repls.Replace(out)
doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
for key := range outputs {
if key == "output.txt" {
// handled above
continue
}
pathNew := filepath.Join(tmpDir, key)
newValBytes, err := os.ReadFile(pathNew)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
} else {
t.Errorf("%s: could not read: %s", key, err)
}
continue
}
pathExpected := filepath.Join(dir, key)
newVal := repls.Replace(string(newValBytes))
doComparison(t, pathExpected, pathNew, newVal)
}
// Make sure there are not unaccounted for new files
files, err := os.ReadDir(tmpDir)
require.NoError(t, err)
for _, f := range files {
name := f.Name()
if _, ok := inputs[name]; ok {
continue
}
if _, ok := outputs[name]; ok {
continue
}
t.Errorf("Unexpected output: %s", f)
if strings.HasPrefix(name, "out") {
// We have a new file starting with "out"
// Show the contents & support overwrite mode for it:
pathNew := filepath.Join(tmpDir, name)
newVal := testutil.ReadFile(t, pathNew)
newVal = repls.Replace(newVal)
doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
}
}
}
func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
valueNew = testdiff.NormalizeNewlines(valueNew)
valueExpected := string(readIfExists(t, pathExpected))
valueExpected = testdiff.NormalizeNewlines(valueExpected)
testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
if testdiff.OverwriteMode {
if valueNew != "" {
t.Logf("Overwriting: %s", pathExpected)
testutil.WriteFile(t, pathExpected, valueNew)
} else {
t.Logf("Removing: %s", pathExpected)
_ = os.Remove(pathExpected)
}
}
}
// Returns combined script.prepare (root) + script.prepare (parent) + ... + script + ... + script.cleanup (parent) + ...
// Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
func readMergedScriptContents(t *testing.T, dir string) string {
scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
prepares := []string{}
cleanups := []string{}
for {
x := readIfExists(t, filepath.Join(dir, CleanupScript))
if len(x) > 0 {
cleanups = append(cleanups, string(x))
}
x = readIfExists(t, filepath.Join(dir, PrepareScript))
if len(x) > 0 {
prepares = append(prepares, string(x))
}
if dir == "" || dir == "." {
break
}
dir = filepath.Dir(dir)
require.True(t, filepath.IsLocal(dir))
}
slices.Reverse(prepares)
prepares = append(prepares, scriptContents)
prepares = append(prepares, cleanups...)
return strings.Join(prepares, "\n")
}
func BuildCLI(t *testing.T, cwd string) string {
execPath := filepath.Join(cwd, "build", "databricks")
if runtime.GOOS == "windows" {
execPath += ".exe"
}
start := time.Now()
args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = ".."
out, err := cmd.CombinedOutput()
elapsed := time.Since(start)
t.Logf("%s took %s", args, elapsed)
require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
if len(out) > 0 {
t.Logf("go build output: %s: %s", args, out)
}
// Quick check + warm up cache:
cmd = exec.Command(execPath, "--version")
out, err = cmd.CombinedOutput()
require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
return execPath
}
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
return err
}
func formatOutput(out string, err error) string {
if err == nil {
return out
}
if exiterr, ok := err.(*exec.ExitError); ok {
exitCode := exiterr.ExitCode()
out += fmt.Sprintf("\nExit code: %d\n", exitCode)
} else {
out += fmt.Sprintf("\nError: %s\n", err)
}
return out
}
func readIfExists(t *testing.T, path string) []byte {
data, err := os.ReadFile(path)
if err == nil {
return data
}
if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%s: %s", path, err)
}
return []byte{}
}
func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
name := info.Name()
relPath, err := filepath.Rel(src, path)
if err != nil {
return err
}
if strings.HasPrefix(name, "out") {
outputs[relPath] = true
return nil
} else {
inputs[relPath] = true
}
if _, ok := Scripts[name]; ok {
return nil
}
destPath := filepath.Join(dst, relPath)
if info.IsDir() {
return os.MkdirAll(destPath, info.Mode())
}
return copyFile(path, destPath)
})
}

21
acceptance/bin/sort_blocks.py Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python3
"""
Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
This is to workaround non-determinism in the output.
"""
import sys
blocks = []
for line in sys.stdin:
if not line.strip():
if blocks and blocks[-1]:
blocks.append('')
continue
if not blocks:
blocks.append('')
blocks[-1] += line
blocks.sort()
print("\n".join(blocks))

1
acceptance/build/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
databricks

View File

@ -1,9 +1,6 @@
bundle:
name: clusters
workspace:
host: https://acme.cloud.databricks.com/
resources:
clusters:
foo:

View File

@ -0,0 +1,33 @@
>>> $CLI bundle validate -o json -t default
{
"autoscale": {
"max_workers": 7,
"min_workers": 2
},
"cluster_name": "foo",
"custom_tags": {},
"node_type_id": "i3.xlarge",
"num_workers": 2,
"spark_conf": {
"spark.executor.memory": "2g"
},
"spark_version": "13.3.x-scala2.12"
}
>>> $CLI bundle validate -o json -t development
{
"autoscale": {
"max_workers": 3,
"min_workers": 1
},
"cluster_name": "foo-override",
"custom_tags": {},
"node_type_id": "m5.xlarge",
"num_workers": 3,
"spark_conf": {
"spark.executor.memory": "4g",
"spark.executor.memory2": "4g"
},
"spark_version": "15.2.x-scala2.12"
}

View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t default | jq .resources.clusters.foo
trace $CLI bundle validate -o json -t development | jq .resources.clusters.foo

View File

@ -1,9 +1,6 @@
bundle:
name: override_job_cluster
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
foo:

View File

@ -0,0 +1,56 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_version": "13.3.x-scala2.12"
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}

View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'

View File

@ -0,0 +1,37 @@
bundle:
name: override_job_cluster
variables:
mykey:
default: key
resources:
jobs:
foo:
name: job
job_clusters:
- job_cluster_key: key
new_cluster:
spark_version: 13.3.x-scala2.12
targets:
development:
resources:
jobs:
foo:
job_clusters:
# This does not work because merging is done before resolution
- job_cluster_key: "${var.mykey}"
new_cluster:
node_type_id: i3.xlarge
num_workers: 1
staging:
resources:
jobs:
foo:
job_clusters:
- job_cluster_key: "${var.mykey}"
new_cluster:
node_type_id: i3.2xlarge
num_workers: 4

View File

@ -0,0 +1,84 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
}
},
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -t development
Name: override_job_cluster
Target: development
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
Validation OK!
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
}
},
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4
}
}
],
"name": "job",
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
>>> $CLI bundle validate -t staging
Name: override_job_cluster
Target: staging
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
Validation OK!

View File

@ -0,0 +1,4 @@
trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
trace $CLI bundle validate -t development
trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
trace $CLI bundle validate -t staging

View File

@ -1,9 +1,6 @@
bundle:
name: override_job_tasks
workspace:
host: https://acme.cloud.databricks.com/
resources:
jobs:
foo:

View File

@ -0,0 +1,6 @@
>>> errcode $CLI bundle validate -o json -t development
Error: file ./test1.py not found
Exit code: 1

View File

@ -0,0 +1,77 @@
{
"name": "job",
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"new_cluster": {
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test1.py"
},
"task_key": "key1"
},
{
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test2.py"
},
"task_key": "key2"
}
]
}
>>> errcode $CLI bundle validate -o json -t staging
Error: file ./test1.py not found
Exit code: 1
{
"name": "job",
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"new_cluster": {
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test1.py"
},
"task_key": "key1"
},
{
"new_cluster": {
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_version": "13.3.x-scala2.12"
},
"spark_python_task": {
"python_file": "./test3.py"
},
"task_key": "key2"
}
]
}
>>> errcode $CLI bundle validate -t staging
Error: file ./test1.py not found
Name: override_job_tasks
Target: staging
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_tasks/staging
Found 1 error
Exit code: 1

View File

@ -0,0 +1,3 @@
trace errcode $CLI bundle validate -o json -t development 2> out.development.stderr.txt | jq .resources.jobs.foo
trace errcode $CLI bundle validate -o json -t staging | jq .resources.jobs.foo
trace errcode $CLI bundle validate -t staging

View File

@ -0,0 +1,13 @@
bundle:
name: merge-string-map
resources:
clusters:
my_cluster: "hello"
targets:
dev:
resources:
clusters:
my_cluster:
spark_version: "25"

View File

@ -0,0 +1,27 @@
>>> $CLI bundle validate -o json -t dev
Warning: expected map, found string
at resources.clusters.my_cluster
in databricks.yml:6:17
{
"clusters": {
"my_cluster": {
"custom_tags": {},
"spark_version": "25"
}
}
}
>>> $CLI bundle validate -t dev
Warning: expected map, found string
at resources.clusters.my_cluster
in databricks.yml:6:17
Name: merge-string-map
Target: dev
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/merge-string-map/dev
Found 1 warning

View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t dev | jq .resources
trace $CLI bundle validate -t dev

View File

@ -1,9 +1,6 @@
bundle:
name: override_pipeline_cluster
workspace:
host: https://acme.cloud.databricks.com/
resources:
pipelines:
foo:

View File

@ -0,0 +1,44 @@
>>> $CLI bundle validate -o json -t development
{
"foo": {
"clusters": [
{
"label": "default",
"node_type_id": "i3.xlarge",
"num_workers": 1,
"spark_conf": {
"foo": "bar"
}
}
],
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/development/state/metadata.json"
},
"name": "job",
"permissions": []
}
}
>>> $CLI bundle validate -o json -t staging
{
"foo": {
"clusters": [
{
"label": "default",
"node_type_id": "i3.2xlarge",
"num_workers": 4,
"spark_conf": {
"foo": "bar"
}
}
],
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/staging/state/metadata.json"
},
"name": "job",
"permissions": []
}
}

View File

@ -0,0 +1,2 @@
trace $CLI bundle validate -o json -t development | jq .resources.pipelines
trace $CLI bundle validate -o json -t staging | jq .resources.pipelines

View File

@ -0,0 +1,19 @@
Error: experiment undefined-experiment is not defined
at resources.experiments.undefined-experiment
in databricks.yml:11:26
Error: job undefined-job is not defined
at resources.jobs.undefined-job
in databricks.yml:6:19
Error: pipeline undefined-pipeline is not defined
at resources.pipelines.undefined-pipeline
in databricks.yml:14:24
Found 3 errors
Name: undefined-job
Target: default
Exit code: 1

View File

@ -0,0 +1,2 @@
# We need sort_blocks.py because the order of diagnostics is currently randomized
$CLI bundle validate 2>&1 | sort_blocks.py

View File

@ -0,0 +1,19 @@
bundle:
name: complex-transitive
variables:
catalog:
default: hive_metastore
spark_conf:
default:
"spark.databricks.sql.initial.catalog.name": ${var.catalog}
etl_cluster_config:
type: complex
default:
spark_version: 14.3.x-scala2.12
runtime_engine: PHOTON
spark_conf: ${var.spark_conf}
resources:
clusters:
my_cluster: ${var.etl_cluster_config}

View File

@ -0,0 +1,3 @@
{
"spark.databricks.sql.initial.catalog.name": "${var.catalog}"
}

View File

@ -0,0 +1,2 @@
# Currently, this incorrectly outputs variable reference instead of resolved value
$CLI bundle validate -o json | jq '.resources.clusters.my_cluster.spark_conf'

View File

@ -11,6 +11,7 @@ resources:
- task_key: test
job_cluster_key: key
libraries: ${variables.libraries.value}
# specific fields of complex variable are referenced:
task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"
variables:
@ -35,30 +36,21 @@ variables:
- jar: "/path/to/jar"
- egg: "/path/to/egg"
- whl: "/path/to/whl"
complexvar:
type: complex
description: "A complex variable"
default:
key1: "value1"
key2: "value2"
key3: "value3"
targets:
default:
default: true
dev:
variables:
node_type: "Standard_DS3_v3"
cluster:
# complex variables are not merged, so missing variables (policy_id) are not inherited
spark_version: "14.2.x-scala2.11"
node_type_id: ${var.node_type}
num_workers: 4
spark_conf:
spark.speculation: false
spark.databricks.delta.retentionDurationCheck.enabled: false
complexvar:
type: complex
default:
key1: "1"
key2: "2"
key3: "3"
libraries:
- jar: "/newpath/to/jar"
- whl: "/newpath/to/whl"

View File

@ -0,0 +1,110 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.random": "true",
"spark.speculation": "true"
},
"spark_version": "13.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"job_cluster_key": "key",
"libraries": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
],
"task_key": "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
}
]
}
}
},
"variables": {
"cluster": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.random": true,
"spark.speculation": true
},
"spark_version": "13.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 2,
"policy_id": "some-policy-id",
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.random": true,
"spark.speculation": true
},
"spark_version": "13.2.x-scala2.11"
}
},
"libraries": {
"default": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
],
"description": "A libraries definition",
"type": "complex",
"value": [
{
"jar": "/path/to/jar"
},
{
"egg": "/path/to/egg"
},
{
"whl": "/path/to/whl"
}
]
},
"node_type": {
"default": "Standard_DS3_v2",
"value": "Standard_DS3_v2"
}
}
}

View File

@ -0,0 +1,95 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key",
"new_cluster": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {},
"tasks": [
{
"job_cluster_key": "key",
"libraries": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
],
"task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
}
]
}
}
},
"variables": {
"cluster": {
"default": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v3",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"libraries": {
"default": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
],
"description": "A libraries definition",
"type": "complex",
"value": [
{
"jar": "/newpath/to/jar"
},
{
"whl": "/newpath/to/whl"
}
]
},
"node_type": {
"default": "Standard_DS3_v3",
"value": "Standard_DS3_v3"
}
}
}

View File

@ -0,0 +1,14 @@
>>> $CLI bundle validate -o json
>>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json
"task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
>>> $CLI bundle validate -o json -t dev
>>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
policy_id and spark_conf.spark_random fields do not exist in dev target:
>>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json
null

View File

@ -0,0 +1,8 @@
trace $CLI bundle validate -o json | jq '{resources,variables}' > out.default.json
trace jq .resources.jobs.my_job.tasks[0].task_key out.default.json | grep "task with spark version 13.2.x-scala2.11 and jar /path/to/jar"
trace $CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json
trace jq .resources.jobs.my_job.tasks[0].task_key out.dev.json | grep "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
echo policy_id and spark_conf.spark_random fields do not exist in dev target:
trace jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json | grep null

View File

@ -0,0 +1,159 @@
{
"resources": {
"jobs": {
"my_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/complex-variables-multiple-files/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"job_clusters": [
{
"job_cluster_key": "key1",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key2",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key3",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
},
{
"job_cluster_key": "key4",
"new_cluster": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": "false",
"spark.speculation": "false"
},
"spark_version": "14.2.x-scala2.11"
}
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {}
}
}
},
"variables": {
"cluster1": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster2": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster3": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
},
"cluster4": {
"default": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
},
"description": "A cluster definition",
"type": "complex",
"value": {
"node_type_id": "Standard_DS3_v2",
"num_workers": 4,
"spark_conf": {
"spark.databricks.delta.retentionDurationCheck.enabled": false,
"spark.speculation": false
},
"spark_version": "14.2.x-scala2.11"
}
}
}
}

View File

@ -0,0 +1 @@
$CLI bundle validate -t dev -o json | jq '{resources, variables}'

View File

@ -0,0 +1,11 @@
Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable
Name: empty${var.a}
Target: default
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/empty${var.a}/default
Found 1 error
Exit code: 1

View File

@ -0,0 +1 @@
$CLI bundle validate

View File

@ -0,0 +1,40 @@
>>> $CLI bundle validate -t env-with-single-variable-override -o json
"default-a dev-b"
>>> $CLI bundle validate -t env-with-two-variable-overrides -o json
"prod-a prod-b"
>>> BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json
"prod-a env-var-b"
>>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
Name: test bundle
Target: env-missing-a-required-variable-assignment
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/test bundle/env-missing-a-required-variable-assignment
Found 1 error
Exit code: 1
>>> errcode $CLI bundle validate -t env-using-an-undefined-variable
Error: variable c is not defined but is assigned a value
Name: test bundle
Found 1 error
Exit code: 1
>>> $CLI bundle validate -t env-overrides-lookup -o json
{
"a": "default-a",
"b": "prod-b",
"d": "4321",
"e": "1234",
"f": "9876"
}

View File

@ -0,0 +1,6 @@
trace $CLI bundle validate -t env-with-single-variable-override -o json | jq .workspace.profile
trace $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
trace BUNDLE_VAR_b=env-var-b $CLI bundle validate -t env-with-two-variable-overrides -o json | jq .workspace.profile
trace errcode $CLI bundle validate -t env-missing-a-required-variable-assignment
trace errcode $CLI bundle validate -t env-using-an-undefined-variable
trace $CLI bundle validate -t env-overrides-lookup -o json | jq '.variables | map_values(.value)'

View File

@ -0,0 +1,6 @@
bundle:
name: TestResolveVariableReferences
workspace:
root_path: "${bundle.name}/bar"
file_path: "${workspace.root_path}/baz"

View File

@ -0,0 +1,11 @@
{
"artifact_path": "TestResolveVariableReferences/bar/artifacts",
"current_user": {
"short_name": "tester",
"userName": "tester@databricks.com"
},
"file_path": "TestResolveVariableReferences/bar/baz",
"resource_path": "TestResolveVariableReferences/bar/resources",
"root_path": "TestResolveVariableReferences/bar",
"state_path": "TestResolveVariableReferences/bar/state"
}

View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .workspace

View File

@ -0,0 +1,10 @@
bundle:
name: TestResolveVariableReferencesToEmptyFields
git:
branch: ""
resources:
jobs:
job1:
tags:
git_branch: "${bundle.git.branch}"

View File

@ -0,0 +1,3 @@
{
"git_branch": ""
}

View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.tags

View File

@ -0,0 +1,16 @@
bundle:
name: TestResolveComplexVariableReferencesToFields
variables:
cluster:
type: "complex"
default:
node_type_id: "Standard_DS3_v2"
num_workers: 2
resources:
jobs:
job1:
job_clusters:
- new_cluster:
node_type_id: "${var.cluster.node_type_id}"

View File

@ -0,0 +1,3 @@
{
"node_type_id": "Standard_DS3_v2"
}

View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.jobs.job1.job_clusters[0].new_cluster

View File

@ -0,0 +1,16 @@
>>> BUNDLE_VAR_b=def $CLI bundle validate -o json
"abc def"
>>> errcode $CLI bundle validate
Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable
Name: ${var.a} ${var.b}
Target: default
Workspace:
User: tester@databricks.com
Path: /Workspace/Users/tester@databricks.com/.bundle/${var.a} ${var.b}/default
Found 1 error
Exit code: 1

View File

@ -0,0 +1,2 @@
trace BUNDLE_VAR_b=def $CLI bundle validate -o json | jq .bundle.name
trace errcode $CLI bundle validate

View File

@ -0,0 +1,84 @@
>>> $CLI bundle validate -o json -t use-default-variable-values
{
"pipelines": {
"my_pipeline": {
"clusters": [
{
"label": "default",
"num_workers": 42
}
],
"continuous": true,
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/use-default-variable-values/state/metadata.json"
},
"name": "a_string",
"permissions": []
}
}
}
>>> $CLI bundle validate -o json -t override-string-variable
{
"pipelines": {
"my_pipeline": {
"clusters": [
{
"label": "default",
"num_workers": 42
}
],
"continuous": true,
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-string-variable/state/metadata.json"
},
"name": "overridden_string",
"permissions": []
}
}
}
>>> $CLI bundle validate -o json -t override-int-variable
{
"pipelines": {
"my_pipeline": {
"clusters": [
{
"label": "default",
"num_workers": 43
}
],
"continuous": true,
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-int-variable/state/metadata.json"
},
"name": "a_string",
"permissions": []
}
}
}
>>> $CLI bundle validate -o json -t override-both-bool-and-string-variables
{
"pipelines": {
"my_pipeline": {
"clusters": [
{
"label": "default",
"num_workers": 42
}
],
"continuous": false,
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
},
"name": "overridden_string",
"permissions": []
}
}
}

View File

@ -0,0 +1,4 @@
trace $CLI bundle validate -o json -t use-default-variable-values | jq .resources
trace $CLI bundle validate -o json -t override-string-variable | jq .resources
trace $CLI bundle validate -o json -t override-int-variable | jq .resources
trace $CLI bundle validate -o json -t override-both-bool-and-string-variables | jq .resources

View File

@ -0,0 +1,4 @@
{
"a": "foo",
"b": "bar"
}

View File

@ -0,0 +1 @@
BUNDLE_VAR_a=foo BUNDLE_VAR_b=bar $CLI bundle validate -o json | jq '.variables | map_values(.value)'

143
acceptance/help/output.txt Normal file
View File

@ -0,0 +1,143 @@
Databricks CLI
Usage:
databricks [command]
Databricks Workspace
fs Filesystem related commands
git-credentials Registers personal access token for Databricks to do operations on behalf of the user.
repos The Repos API allows users to manage their git repos.
secrets The Secrets API allows you to manage secrets, secret scopes, and access permissions.
workspace The Workspace API allows you to list, import, export, and delete notebooks and folders.
Compute
cluster-policies You can use cluster policies to control users' ability to configure clusters based on a set of rules.
clusters The Clusters API allows you to create, start, edit, list, terminate, and delete clusters.
global-init-scripts The Global Init Scripts API enables Workspace administrators to configure global initialization scripts for their workspace.
instance-pools Instance Pools API are used to create, edit, delete and list instance pools by using ready-to-use cloud instances which reduces a cluster start and auto-scaling times.
instance-profiles The Instance Profiles API allows admins to add, list, and remove instance profiles that users can launch clusters with.
libraries The Libraries API allows you to install and uninstall libraries and get the status of libraries on a cluster.
policy-compliance-for-clusters The policy compliance APIs allow you to view and manage the policy compliance status of clusters in your workspace.
policy-families View available policy families.
Workflows
jobs The Jobs API allows you to create, edit, and delete jobs.
policy-compliance-for-jobs The compliance APIs allow you to view and manage the policy compliance status of jobs in your workspace.
Delta Live Tables
pipelines The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines.
Machine Learning
experiments Experiments are the primary unit of organization in MLflow; all MLflow runs belong to an experiment.
model-registry Note: This API reference documents APIs for the Workspace Model Registry.
Real-time Serving
serving-endpoints The Serving Endpoints API allows you to create, update, and delete model serving endpoints.
Identity and Access Management
current-user This API allows retrieving information about currently authenticated user or service principal.
groups Groups simplify identity management, making it easier to assign access to Databricks workspace, data, and other securable objects.
permissions Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints.
service-principals Identities for use with jobs, automated tools, and systems such as scripts, apps, and CI/CD platforms.
users User identities recognized by Databricks and represented by email addresses.
Databricks SQL
alerts The alerts API can be used to perform CRUD operations on alerts.
alerts-legacy The alerts API can be used to perform CRUD operations on alerts.
dashboards In general, there is little need to modify dashboards using the API.
data-sources This API is provided to assist you in making new query objects.
queries The queries API can be used to perform CRUD operations on queries.
queries-legacy These endpoints are used for CRUD operations on query definitions.
query-history A service responsible for storing and retrieving the list of queries run against SQL endpoints and serverless compute.
warehouses A SQL warehouse is a compute resource that lets you run SQL commands on data objects within Databricks SQL.
Unity Catalog
artifact-allowlists In Databricks Runtime 13.3 and above, you can add libraries and init scripts to the allowlist in UC so that users can leverage these artifacts on compute configured with shared access mode.
catalogs A catalog is the first layer of Unity Catalogs three-level namespace.
connections Connections allow for creating a connection to an external data source.
credentials A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant.
external-locations An external location is an object that combines a cloud storage path with a storage credential that authorizes access to the cloud storage path.
functions Functions implement User-Defined Functions (UDFs) in Unity Catalog.
grants In Unity Catalog, data is secure by default.
metastores A metastore is the top-level container of objects in Unity Catalog.
model-versions Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
online-tables Online tables provide lower latency and higher QPS access to data from Delta tables.
quality-monitors A monitor computes and monitors data or model quality metrics for a table over time.
registered-models Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.
resource-quotas Unity Catalog enforces resource quotas on all securable objects, which limits the number of resources that can be created.
schemas A schema (also called a database) is the second layer of Unity Catalogs three-level namespace.
storage-credentials A storage credential represents an authentication and authorization mechanism for accessing data stored on your cloud tenant.
system-schemas A system schema is a schema that lives within the system catalog.
table-constraints Primary key and foreign key constraints encode relationships between fields in tables.
tables A table resides in the third layer of Unity Catalogs three-level namespace.
temporary-table-credentials Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locationswhere table data is stored in Databricks.
volumes Volumes are a Unity Catalog (UC) capability for accessing, storing, governing, organizing and processing files.
workspace-bindings A securable in Databricks can be configured as __OPEN__ or __ISOLATED__.
Delta Sharing
providers A data provider is an object representing the organization in the real world who shares the data.
recipient-activation The Recipient Activation API is only applicable in the open sharing model where the recipient object has the authentication type of TOKEN.
recipients A recipient is an object you create using :method:recipients/create to represent an organization which you want to allow access shares.
shares A share is a container instantiated with :method:shares/create.
Settings
ip-access-lists IP Access List enables admins to configure IP access lists.
notification-destinations The notification destinations API lets you programmatically manage a workspace's notification destinations.
settings Workspace Settings API allows users to manage settings at the workspace level.
token-management Enables administrators to get all tokens and delete tokens for other users.
tokens The Token API allows you to create, list, and revoke tokens that can be used to authenticate and access Databricks REST APIs.
workspace-conf This API allows updating known workspace settings for advanced users.
Developer Tools
bundle Databricks Asset Bundles let you express data/AI/analytics projects as code.
sync Synchronize a local directory to a workspace directory
Vector Search
vector-search-endpoints **Endpoint**: Represents the compute resources to host vector search indexes.
vector-search-indexes **Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries.
Dashboards
lakeview These APIs provide specific management operations for Lakeview dashboards.
Marketplace
consumer-fulfillments Fulfillments are entities that allow consumers to preview installations.
consumer-installations Installations are entities that allow consumers to interact with Databricks Marketplace listings.
consumer-listings Listings are the core entities in the Marketplace.
consumer-personalization-requests Personalization Requests allow customers to interact with the individualized Marketplace listing flow.
consumer-providers Providers are the entities that publish listings to the Marketplace.
provider-exchange-filters Marketplace exchanges filters curate which groups can access an exchange.
provider-exchanges Marketplace exchanges allow providers to share their listings with a curated set of customers.
provider-files Marketplace offers a set of file APIs for various purposes such as preview notebooks and provider icons.
provider-listings Listings are the core entities in the Marketplace.
provider-personalization-requests Personalization requests are an alternate to instantly available listings.
provider-provider-analytics-dashboards Manage templated analytics solution for providers.
provider-providers Providers are entities that manage assets in Marketplace.
Apps
apps Apps run directly on a customers Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
apps Apps run directly on a customers Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.
Clean Rooms
clean-room-assets Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room.
clean-room-task-runs Clean room task runs are the executions of notebooks in a clean room.
clean-rooms A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each others data.
Additional Commands:
account Databricks Account Commands
api Perform Databricks API call
auth Authentication related commands
completion Generate the autocompletion script for the specified shell
configure Configure authentication
help Help about any command
labs Manage Databricks Labs installations
version Retrieve information about the current version of this CLI
Flags:
--debug enable debug logging
-h, --help help for databricks
-o, --output type output type: text or json (default text)
-p, --profile string ~/.databrickscfg profile
-t, --target string bundle target to use (if applicable)
-v, --version version for databricks
Use "databricks [command] --help" for more information about a command.

1
acceptance/help/script Normal file
View File

@ -0,0 +1 @@
$CLI

View File

@ -0,0 +1 @@
rm -fr .databricks .gitignore

36
acceptance/script.prepare Normal file
View File

@ -0,0 +1,36 @@
# Prevent CLI from downloading terraform in each test:
export DATABRICKS_TF_EXEC_PATH=/tmp/
errcode() {
# Temporarily disable 'set -e' to prevent the script from exiting on error
set +e
# Execute the provided command with all arguments
"$@"
local exit_code=$?
# Re-enable 'set -e' if it was previously set
set -e
>&2 printf "\nExit code: $exit_code\n"
}
trace() {
>&2 printf "\n>>> %s\n" "$*"
if [[ "$1" == *"="* ]]; then
# If the first argument contains '=', collect all env vars
local env_vars=()
while [[ "$1" == *"="* ]]; do
env_vars+=("$1")
shift
done
# Export environment variables in a subshell and execute the command
(
export "${env_vars[@]}"
"$@"
)
else
# Execute the command normally
"$@"
fi
return $?
}

129
acceptance/server_test.go Normal file
View File

@ -0,0 +1,129 @@
package acceptance_test
import (
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"testing"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/databricks/databricks-sdk-go/service/workspace"
)
type TestServer struct {
*httptest.Server
Mux *http.ServeMux
Port int
}
type HandlerFunc func(r *http.Request) (any, error)
func NewTestServer() *TestServer {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
port := server.Listener.Addr().(*net.TCPAddr).Port
return &TestServer{
Server: server,
Mux: mux,
Port: port,
}
}
func (s *TestServer) Handle(pattern string, handler HandlerFunc) {
s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
resp, err := handler(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
var respBytes []byte
respString, ok := resp.(string)
if ok {
respBytes = []byte(respString)
} else {
respBytes, err = json.MarshalIndent(resp, "", " ")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
if _, err := w.Write(respBytes); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
})
}
func StartServer(t *testing.T) *TestServer {
server := NewTestServer()
t.Cleanup(func() {
server.Close()
})
return server
}
func AddHandlers(server *TestServer) {
server.Handle("/api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
return compute.ListPoliciesResponse{
Policies: []compute.Policy{
{
PolicyId: "5678",
Name: "wrong-cluster-policy",
},
{
PolicyId: "9876",
Name: "some-test-cluster-policy",
},
},
}, nil
})
server.Handle("/api/2.0/instance-pools/list", func(r *http.Request) (any, error) {
return compute.ListInstancePools{
InstancePools: []compute.InstancePoolAndStats{
{
InstancePoolName: "some-test-instance-pool",
InstancePoolId: "1234",
},
},
}, nil
})
server.Handle("/api/2.1/clusters/list", func(r *http.Request) (any, error) {
return compute.ListClustersResponse{
Clusters: []compute.ClusterDetails{
{
ClusterName: "some-test-cluster",
ClusterId: "4321",
},
{
ClusterName: "some-other-cluster",
ClusterId: "9876",
},
},
}, nil
})
server.Handle("/api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
return iam.User{
UserName: "tester@databricks.com",
}, nil
})
server.Handle("/api/2.0/workspace/get-status", func(r *http.Request) (any, error) {
return workspace.ObjectInfo{
ObjectId: 1001,
ObjectType: "DIRECTORY",
Path: "",
ResourceId: "1001",
}, nil
})
}

View File

@ -3,7 +3,6 @@ package artifacts
import (
"context"
"fmt"
"slices"
"github.com/databricks/cli/bundle"

View File

@ -13,8 +13,7 @@ func DetectPackages() bundle.Mutator {
return &autodetect{}
}
type autodetect struct {
}
type autodetect struct{}
func (m *autodetect) Name() string {
return "artifacts.DetectPackages"

View File

@ -96,9 +96,8 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost
// Set the expanded globs back into the configuration.
return dyn.SetByPath(v, base, dyn.V(output))
})
if err != nil {
return diag.FromErr(err)
diags = diags.Extend(diag.FromErr(err))
}
return diags

View File

@ -2,7 +2,6 @@ package artifacts
import (
"context"
"fmt"
"path/filepath"
"testing"
@ -88,16 +87,16 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
))
assert.Len(t, diags, 4)
assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary)
assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[0].Summary)
assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File)
assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String())
assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary)
assert.Equal(t, filepath.Clean("a[.txt")+": syntax error in pattern", diags[1].Summary)
assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File)
assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String())
assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary)
assert.Equal(t, filepath.Clean("../a[.txt")+": syntax error in pattern", diags[2].Summary)
assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File)
assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String())
assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary)
assert.Equal(t, filepath.Clean("subdir/a[.txt")+": syntax error in pattern", diags[3].Summary)
assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File)
assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String())
}

View File

@ -15,8 +15,7 @@ import (
"github.com/databricks/cli/libs/log"
)
type detectPkg struct {
}
type detectPkg struct{}
func DetectPackage() bundle.Mutator {
return &detectPkg{}
@ -42,7 +41,7 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
return nil
}
log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath))
log.Infof(ctx, "Found Python wheel project at %s", b.BundleRootPath)
module := extractModuleName(setupPy)
if b.Config.Artifacts == nil {

View File

@ -16,12 +16,6 @@ type infer struct {
func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
artifact := b.Config.Artifacts[m.name]
// TODO use python.DetectVEnvExecutable once bundle has a way to specify venv path
py, err := python.DetectExecutable(ctx)
if err != nil {
return diag.FromErr(err)
}
// Note: using --build-number (build tag) flag does not help with re-installing
// libraries on all-purpose clusters. The reason is that `pip` ignoring build tag
// when upgrading the library and only look at wheel version.
@ -36,7 +30,9 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// version=datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"),
// ...
//)
artifact.BuildCommand = fmt.Sprintf(`"%s" setup.py bdist_wheel`, py)
py := python.GetExecutable()
artifact.BuildCommand = py + " setup.py bdist_wheel"
return nil
}

View File

@ -8,6 +8,7 @@ package bundle
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
@ -189,7 +190,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
// Make directory if it doesn't exist yet.
dir := filepath.Join(parts...)
err := os.MkdirAll(dir, 0700)
err := os.MkdirAll(dir, 0o700)
if err != nil {
return "", err
}
@ -206,7 +207,7 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) {
}
dir := filepath.Join(cacheDir, internalFolder)
err = os.MkdirAll(dir, 0700)
err = os.MkdirAll(dir, 0o700)
if err != nil {
return dir, err
}
@ -237,7 +238,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
// we call into from this bundle context.
func (b *Bundle) AuthEnv() (map[string]string, error) {
if b.client == nil {
return nil, fmt.Errorf("workspace client not initialized yet")
return nil, errors.New("workspace client not initialized yet")
}
cfg := b.client.Config

View File

@ -2,7 +2,6 @@ package bundle
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
@ -16,7 +15,7 @@ import (
func TestLoadNotExists(t *testing.T) {
b, err := Load(context.Background(), "/doesntexist")
assert.True(t, errors.Is(err, fs.ErrNotExist))
assert.ErrorIs(t, err, fs.ErrNotExist)
assert.Nil(t, b)
}

View File

@ -2,7 +2,7 @@ package config
import (
"context"
"fmt"
"errors"
"github.com/databricks/cli/libs/exec"
)
@ -37,7 +37,7 @@ type Artifact struct {
func (a *Artifact) Build(ctx context.Context) ([]byte, error) {
if a.BuildCommand == "" {
return nil, fmt.Errorf("no build property defined")
return nil, errors.New("no build property defined")
}
var e *exec.Executor

View File

@ -27,9 +27,33 @@ type Experimental struct {
// PyDABs determines whether to load the 'databricks-pydabs' package.
//
// PyDABs allows to define bundle configuration using Python.
// PyDABs is deprecated use Python instead.
PyDABs PyDABs `json:"pydabs,omitempty"`
// Python configures loading of Python code defined with 'databricks-bundles' package.
Python Python `json:"python,omitempty"`
}
type Python struct {
// Resources contains a list of fully qualified function paths to load resources
// defined in Python code.
//
// Example: ["my_project.resources:load_resources"]
Resources []string `json:"resources"`
// Mutators contains a list of fully qualified function paths to mutator functions.
//
// Example: ["my_project.mutators:add_default_cluster"]
Mutators []string `json:"mutators"`
// VEnvPath is path to the virtual environment.
//
// If enabled, Python code will execute within this environment. If disabled,
// it defaults to using the Python interpreter available in the current shell.
VEnvPath string `json:"venv_path,omitempty"`
}
// PyDABs is deprecated use Python instead
type PyDABs struct {
// Enabled is a flag to enable the feature.
Enabled bool `json:"enabled,omitempty"`
@ -47,8 +71,10 @@ type PyDABs struct {
Import []string `json:"import,omitempty"`
}
type Command string
type ScriptHook string
type (
Command string
ScriptHook string
)
// These hook names are subject to change and currently experimental
const (

View File

@ -6,8 +6,10 @@ import (
"github.com/databricks/databricks-sdk-go/service/jobs"
)
var jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"})
var taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"})
var (
jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"})
taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"})
)
func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) {
value := make(map[string]dyn.Value)

Some files were not shown because too many files have changed in this diff Show More