mirror of https://github.com/databricks/cli.git
Merge remote-tracking branch 'origin/main' into cp-encourage-root-path-followup
This commit is contained in:
Lennart Kats
commit
b31ed2dc96
|
|
@ -1 +1 @@
|
|||
a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d
|
||||
779817ed8d63031f5ea761fbd25ee84f38feec0d
|
||||
|
|
@ -140,9 +140,9 @@ func new{{.PascalName}}() *cobra.Command {
|
|||
{{- end}}
|
||||
{{$method := .}}
|
||||
{{ if not .IsJsonOnly }}
|
||||
{{range $request.Fields -}}
|
||||
{{range .AllFields -}}
|
||||
{{- if not .Required -}}
|
||||
{{if .Entity.IsObject }}// TODO: complex arg: {{.Name}}
|
||||
{{if .Entity.IsObject}}{{if not (eq . $method.RequestBodyField) }}// TODO: complex arg: {{.Name}}{{end}}
|
||||
{{else if .Entity.IsAny }}// TODO: any: {{.Name}}
|
||||
{{else if .Entity.ArrayValue }}// TODO: array: {{.Name}}
|
||||
{{else if .Entity.MapValue }}// TODO: map via StringToStringVar: {{.Name}}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ jobs:
|
|||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
- uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
|
||||
with:
|
||||
stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled.
|
||||
stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled.
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
if: "${{ github.event.pull_request.head.repo.fork }}"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Delete old comments
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
steps:
|
||||
- name: Generate GitHub App Token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@v1
|
||||
uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
|
||||
with:
|
||||
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
|
||||
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ jobs:
|
|||
steps:
|
||||
- name: Generate GitHub App Token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@v1
|
||||
uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1
|
||||
with:
|
||||
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
|
||||
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
|
||||
|
|
|
|||
|
|
@ -10,19 +10,65 @@ on:
|
|||
jobs:
|
||||
publish-to-winget-pkgs:
|
||||
runs-on:
|
||||
group: databricks-protected-runner-group
|
||||
labels: windows-server-latest
|
||||
group: databricks-deco-testing-runner-group
|
||||
labels: ubuntu-latest-deco
|
||||
|
||||
environment: release
|
||||
|
||||
steps:
|
||||
- uses: vedantmgoyal2009/winget-releaser@93fd8b606a1672ec3e5c6c3bb19426be68d1a8b0 # https://github.com/vedantmgoyal2009/winget-releaser/releases/tag/v2
|
||||
with:
|
||||
identifier: Databricks.DatabricksCLI
|
||||
installers-regex: 'windows_.*-signed\.zip$' # Only signed Windows releases
|
||||
token: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
|
||||
fork-user: eng-dev-ecosystem-bot
|
||||
- name: Checkout repository and submodules
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
# Use the tag from the input, or the ref name if the input is not provided.
|
||||
# The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
|
||||
release-tag: ${{ inputs.tag || github.ref_name }}
|
||||
# When updating the version of komac, make sure to update the checksum in the next step.
|
||||
# Find both at https://github.com/russellbanks/Komac/releases.
|
||||
- name: Download komac binary
|
||||
run: |
|
||||
curl -s -L -o $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz https://github.com/russellbanks/Komac/releases/download/v2.9.0/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz
|
||||
|
||||
- name: Verify komac binary
|
||||
run: |
|
||||
echo "d07a12831ad5418fee715488542a98ce3c0e591d05c850dd149fe78432be8c4c $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz" | sha256sum -c -
|
||||
|
||||
- name: Untar komac binary to temporary path
|
||||
run: |
|
||||
mkdir -p $RUNNER_TEMP/komac
|
||||
tar -xzf $RUNNER_TEMP/komac-2.9.0-x86_64-unknown-linux-gnu.tar.gz -C $RUNNER_TEMP/komac
|
||||
|
||||
- name: Add komac to PATH
|
||||
run: echo "$RUNNER_TEMP/komac" >> $GITHUB_PATH
|
||||
|
||||
- name: Confirm komac version
|
||||
run: komac --version
|
||||
|
||||
# Use the tag from the input, or the ref name if the input is not provided.
|
||||
# The ref name is equal to the tag name when this workflow is triggered by the "sign-cli" command.
|
||||
- name: Strip "v" prefix from version
|
||||
id: strip_version
|
||||
run: echo "version=$(echo ${{ inputs.tag || github.ref_name }} | sed 's/^v//')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Get URLs of signed Windows binaries
|
||||
id: get_windows_urls
|
||||
run: |
|
||||
urls=$(
|
||||
gh api https://api.github.com/repos/databricks/cli/releases/tags/${{ inputs.tag || github.ref_name }} | \
|
||||
jq -r .assets[].browser_download_url | \
|
||||
grep -E '_windows_.*-signed\.zip$' | \
|
||||
tr '\n' ' '
|
||||
)
|
||||
if [ -z "$urls" ]; then
|
||||
echo "No signed Windows binaries found" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "urls=$urls" >> "$GITHUB_OUTPUT"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish to Winget
|
||||
run: |
|
||||
komac update Databricks.DatabricksCLI \
|
||||
--version ${{ steps.strip_version.outputs.version }} \
|
||||
--submit \
|
||||
--urls ${{ steps.get_windows_urls.outputs.urls }} \
|
||||
env:
|
||||
KOMAC_FORK_OWNER: eng-dev-ecosystem-bot
|
||||
GITHUB_TOKEN: ${{ secrets.ENG_DEV_ECOSYSTEM_BOT_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -45,20 +45,20 @@ jobs:
|
|||
|
||||
steps:
|
||||
- name: Checkout repository and submodules
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
|
||||
with:
|
||||
go-version: 1.23.4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0
|
||||
|
||||
- name: Set go env
|
||||
run: |
|
||||
|
|
@ -71,16 +71,16 @@ jobs:
|
|||
make vendor
|
||||
pip3 install wheel
|
||||
|
||||
- name: Run tests
|
||||
run: make test
|
||||
- name: Run tests with coverage
|
||||
run: make cover
|
||||
|
||||
golangci:
|
||||
linters:
|
||||
needs: cleanups
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
|
||||
with:
|
||||
go-version: 1.23.4
|
||||
# Use different schema from regular job, to avoid overwriting the same key
|
||||
|
|
@ -95,10 +95,15 @@ jobs:
|
|||
# Exit with status code 1 if there are differences (i.e. unformatted files)
|
||||
git diff --exit-code
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
|
||||
with:
|
||||
version: v1.63.4
|
||||
args: --timeout=15m
|
||||
- name: Run ruff
|
||||
uses: astral-sh/ruff-action@31a518504640beb4897d0b9f9e50a2a9196e75ba # v3.0.1
|
||||
with:
|
||||
version: "0.9.1"
|
||||
args: "format --check"
|
||||
|
||||
validate-bundle-schema:
|
||||
needs: cleanups
|
||||
|
|
@ -106,10 +111,10 @@ jobs:
|
|||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
|
||||
with:
|
||||
go-version: 1.23.4
|
||||
# Use different schema from regular job, to avoid overwriting the same key
|
||||
|
|
|
|||
|
|
@ -26,13 +26,13 @@ jobs:
|
|||
|
||||
steps:
|
||||
- name: Checkout repository and submodules
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
|
||||
with:
|
||||
go-version: 1.23.4
|
||||
|
||||
|
|
@ -48,27 +48,27 @@ jobs:
|
|||
|
||||
- name: Run GoReleaser
|
||||
id: releaser
|
||||
uses: goreleaser/goreleaser-action@v6
|
||||
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
|
||||
with:
|
||||
version: ~> v2
|
||||
args: release --snapshot --skip docker
|
||||
|
||||
- name: Upload macOS binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
with:
|
||||
name: cli_darwin_snapshot
|
||||
path: |
|
||||
dist/*_darwin_*/
|
||||
|
||||
- name: Upload Linux binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
with:
|
||||
name: cli_linux_snapshot
|
||||
path: |
|
||||
dist/*_linux_*/
|
||||
|
||||
- name: Upload Windows binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
with:
|
||||
name: cli_windows_snapshot
|
||||
path: |
|
||||
|
|
@ -88,7 +88,7 @@ jobs:
|
|||
# Snapshot release may only be updated for commits to the main branch.
|
||||
if: github.ref == 'refs/heads/main'
|
||||
|
||||
uses: softprops/action-gh-release@v1
|
||||
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v1
|
||||
with:
|
||||
name: Snapshot
|
||||
prerelease: true
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ jobs:
|
|||
|
||||
steps:
|
||||
- name: Checkout repository and submodules
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
|
||||
with:
|
||||
go-version: 1.23.4
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
|
||||
# Log into the GitHub Container Registry. The goreleaser action will create
|
||||
# the docker images and push them to the GitHub Container Registry.
|
||||
- uses: "docker/login-action@v3"
|
||||
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.actor }}"
|
||||
|
|
@ -46,11 +46,11 @@ jobs:
|
|||
# QEMU is required to build cross platform docker images using buildx.
|
||||
# It allows virtualization of the CPU architecture at the application level.
|
||||
- name: Set up QEMU dependency
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
|
||||
|
||||
- name: Run GoReleaser
|
||||
id: releaser
|
||||
uses: goreleaser/goreleaser-action@v6
|
||||
uses: goreleaser/goreleaser-action@9ed2f89a662bf1735a48bc8557fd212fa902bebf # v6.1.0
|
||||
with:
|
||||
version: ~> v2
|
||||
args: release
|
||||
|
|
@ -71,7 +71,7 @@ jobs:
|
|||
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
|
||||
|
||||
- name: Update setup-cli
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
|
||||
script: |
|
||||
|
|
@ -99,7 +99,7 @@ jobs:
|
|||
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
|
||||
|
||||
- name: Update homebrew-tap
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
|
||||
script: |
|
||||
|
|
@ -140,7 +140,7 @@ jobs:
|
|||
echo "VERSION=${VERSION:1}" >> $GITHUB_ENV
|
||||
|
||||
- name: Update CLI version in the VSCode extension
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ secrets.DECO_GITHUB_TOKEN }}
|
||||
script: |
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ dist/
|
|||
|
||||
*.log
|
||||
coverage.txt
|
||||
coverage-acceptance.txt
|
||||
|
||||
__pycache__
|
||||
*.pyc
|
||||
|
|
@ -31,3 +32,4 @@ __pycache__
|
|||
.vscode/tasks.json
|
||||
|
||||
.databricks
|
||||
.ruff_cache
|
||||
|
|
|
|||
|
|
@ -15,12 +15,20 @@ linters:
|
|||
- intrange
|
||||
- mirror
|
||||
- perfsprint
|
||||
- unconvert
|
||||
linters-settings:
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
- shadow
|
||||
settings:
|
||||
printf:
|
||||
funcs:
|
||||
- (github.com/databricks/cli/internal/testutil.TestingT).Infof
|
||||
- (github.com/databricks/cli/internal/testutil.TestingT).Errorf
|
||||
- (github.com/databricks/cli/internal/testutil.TestingT).Fatalf
|
||||
- (github.com/databricks/cli/internal/testutil.TestingT).Skipf
|
||||
gofmt:
|
||||
rewrite-rules:
|
||||
- pattern: 'a[b:len(a)]'
|
||||
|
|
@ -41,6 +49,8 @@ linters-settings:
|
|||
disable:
|
||||
# good check, but we have too many assert.(No)?Errorf? so excluding for now
|
||||
- require-error
|
||||
copyloopvar:
|
||||
check-alias: true
|
||||
issues:
|
||||
exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/
|
||||
max-issues-per-linter: 1000
|
||||
|
|
|
|||
64
CHANGELOG.md
64
CHANGELOG.md
|
|
@ -1,5 +1,69 @@
|
|||
# Version changelog
|
||||
|
||||
## [Release] Release v0.239.1
|
||||
|
||||
CLI:
|
||||
* Added text output templates for apps list and list-deployments ([#2175](https://github.com/databricks/cli/pull/2175)).
|
||||
* Fix duplicate "apps" entry in help output ([#2191](https://github.com/databricks/cli/pull/2191)).
|
||||
|
||||
Bundles:
|
||||
* Allow yaml-anchors in schema ([#2200](https://github.com/databricks/cli/pull/2200)).
|
||||
* Show an error when non-yaml files used in include section ([#2201](https://github.com/databricks/cli/pull/2201)).
|
||||
* Set WorktreeRoot to sync root outside git repo ([#2197](https://github.com/databricks/cli/pull/2197)).
|
||||
* fix: Detailed message for using source-linked deployment with file_path specified ([#2119](https://github.com/databricks/cli/pull/2119)).
|
||||
* Allow using variables in enum fields ([#2199](https://github.com/databricks/cli/pull/2199)).
|
||||
* Add experimental-jobs-as-code template ([#2177](https://github.com/databricks/cli/pull/2177)).
|
||||
* Reading variables from file ([#2171](https://github.com/databricks/cli/pull/2171)).
|
||||
* Fixed an apps message order and added output test ([#2174](https://github.com/databricks/cli/pull/2174)).
|
||||
* Default to forward slash-separated paths for path translation ([#2145](https://github.com/databricks/cli/pull/2145)).
|
||||
* Include a materialized copy of built-in templates ([#2146](https://github.com/databricks/cli/pull/2146)).
|
||||
|
||||
|
||||
|
||||
## [Release] Release v0.239.0
|
||||
|
||||
### New feature announcement
|
||||
|
||||
#### Databricks Apps support
|
||||
|
||||
You can now manage Databricks Apps using DABs by defining an `app` resource in your bundle configuration.
|
||||
For more information see Databricks documentation https://docs.databricks.com/en/dev-tools/bundles/resources.html#app
|
||||
|
||||
#### Referencing complex variables in complex variables
|
||||
|
||||
You can now reference complex variables within other complex variables.
|
||||
For more details see https://github.com/databricks/cli/pull/2157
|
||||
|
||||
CLI:
|
||||
* Filter out system clusters in cluster picker ([#2131](https://github.com/databricks/cli/pull/2131)).
|
||||
* Add command line flags for fields that are not in the API request body ([#2155](https://github.com/databricks/cli/pull/2155)).
|
||||
|
||||
Bundles:
|
||||
* Added support for Databricks Apps in DABs ([#1928](https://github.com/databricks/cli/pull/1928)).
|
||||
* Allow artifact path to be located outside the sync root ([#2128](https://github.com/databricks/cli/pull/2128)).
|
||||
* Retry app deployment if there is an active deployment in progress ([#2153](https://github.com/databricks/cli/pull/2153)).
|
||||
* Resolve variables in a loop ([#2164](https://github.com/databricks/cli/pull/2164)).
|
||||
* Improve resolution of complex variables within complex variables ([#2157](https://github.com/databricks/cli/pull/2157)).
|
||||
* Added output message to warn about slower deployments with apps ([#2161](https://github.com/databricks/cli/pull/2161)).
|
||||
* Patch references to UC schemas to capture dependencies automatically ([#1989](https://github.com/databricks/cli/pull/1989)).
|
||||
* Format default-python template ([#2110](https://github.com/databricks/cli/pull/2110)).
|
||||
* Encourage the use of root_path in production to ensure single deployment ([#1712](https://github.com/databricks/cli/pull/1712)).
|
||||
* Log warnings to stderr for "bundle validate -o json" ([#2109](https://github.com/databricks/cli/pull/2109)).
|
||||
|
||||
API Changes:
|
||||
* Changed `databricks account federation-policy update` command with new required argument order.
|
||||
* Changed `databricks account service-principal-federation-policy update` command with new required argument order.
|
||||
|
||||
OpenAPI commit 779817ed8d63031f5ea761fbd25ee84f38feec0d (2025-01-08)
|
||||
Dependency updates:
|
||||
* Upgrade TF provider to 1.63.0 ([#2162](https://github.com/databricks/cli/pull/2162)).
|
||||
* Bump golangci-lint version to v1.63.4 from v1.63.1 ([#2114](https://github.com/databricks/cli/pull/2114)).
|
||||
* Bump astral-sh/setup-uv from 4 to 5 ([#2116](https://github.com/databricks/cli/pull/2116)).
|
||||
* Bump golang.org/x/oauth2 from 0.24.0 to 0.25.0 ([#2080](https://github.com/databricks/cli/pull/2080)).
|
||||
* Bump github.com/hashicorp/hc-install from 0.9.0 to 0.9.1 ([#2079](https://github.com/databricks/cli/pull/2079)).
|
||||
* Bump golang.org/x/term from 0.27.0 to 0.28.0 ([#2078](https://github.com/databricks/cli/pull/2078)).
|
||||
* Bump github.com/databricks/databricks-sdk-go from 0.54.0 to 0.55.0 ([#2126](https://github.com/databricks/cli/pull/2126)).
|
||||
|
||||
## [Release] Release v0.238.0
|
||||
|
||||
Bundles:
|
||||
|
|
|
|||
19
Makefile
19
Makefile
|
|
@ -1,8 +1,10 @@
|
|||
default: build
|
||||
default: vendor fmt lint
|
||||
|
||||
PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
|
||||
|
||||
GOTESTSUM_FORMAT ?= pkgname-and-test-fails
|
||||
GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped
|
||||
|
||||
|
||||
lint:
|
||||
golangci-lint run --fix
|
||||
|
|
@ -14,17 +16,26 @@ lintcheck:
|
|||
# formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'.
|
||||
# If you need to ensure that formatting & imports are always fixed, do "make fmt lint"
|
||||
fmt:
|
||||
ruff format -q
|
||||
golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./...
|
||||
|
||||
test:
|
||||
gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES}
|
||||
${GOTESTSUM_CMD} -- ${PACKAGES}
|
||||
|
||||
cover:
|
||||
gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES}
|
||||
rm -fr ./acceptance/build/cover/
|
||||
CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} -- -coverprofile=coverage.txt ${PACKAGES}
|
||||
rm -fr ./acceptance/build/cover-merged/
|
||||
mkdir -p acceptance/build/cover-merged/
|
||||
go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
|
||||
go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
|
||||
|
||||
showcover:
|
||||
go tool cover -html=coverage.txt
|
||||
|
||||
acc-showcover:
|
||||
go tool cover -html=coverage-acceptance.txt
|
||||
|
||||
build: vendor
|
||||
go build -mod vendor
|
||||
|
||||
|
|
@ -45,4 +56,4 @@ integration:
|
|||
integration-short:
|
||||
$(INTEGRATION) -short
|
||||
|
||||
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
|
||||
.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
|
||||
|
|
|
|||
4
NOTICE
4
NOTICE
|
|
@ -105,3 +105,7 @@ License - https://github.com/wI2L/jsondiff/blob/master/LICENSE
|
|||
https://github.com/hexops/gotextdiff
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
License - https://github.com/hexops/gotextdiff/blob/main/LICENSE
|
||||
|
||||
https://github.com/BurntSushi/toml
|
||||
Copyright (c) 2013 TOML authors
|
||||
https://github.com/BurntSushi/toml/blob/master/COPYING
|
||||
|
|
|
|||
|
|
@ -17,3 +17,5 @@ For more complex tests one can also use:
|
|||
- `errcode` helper: if the command fails with non-zero code, it appends `Exit code: N` to the output but returns success to caller (bash), allowing continuation of script.
|
||||
- `trace` helper: prints the arguments before executing the command.
|
||||
- custom output files: redirect output to custom file (it must start with `out`), e.g. `$CLI bundle validate > out.txt 2> out.error.txt`.
|
||||
|
||||
See [selftest](./selftest) for a toy test.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
package acceptance_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
|
@ -17,10 +19,26 @@ import (
|
|||
"github.com/databricks/cli/internal/testutil"
|
||||
"github.com/databricks/cli/libs/env"
|
||||
"github.com/databricks/cli/libs/testdiff"
|
||||
"github.com/databricks/databricks-sdk-go"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var KeepTmp = os.Getenv("KEEP_TMP") != ""
|
||||
var KeepTmp bool
|
||||
|
||||
// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
|
||||
// Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
|
||||
// example: var SingleTest = "bundle/variables/empty"
|
||||
var SingleTest = ""
|
||||
|
||||
// If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs
|
||||
// CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py).
|
||||
// Also disables parallelism in tests.
|
||||
var InprocessMode bool
|
||||
|
||||
func init() {
|
||||
flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)")
|
||||
flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
|
||||
}
|
||||
|
||||
const (
|
||||
EntryPointScript = "script"
|
||||
|
|
@ -35,37 +53,100 @@ var Scripts = map[string]bool{
|
|||
}
|
||||
|
||||
func TestAccept(t *testing.T) {
|
||||
testAccept(t, InprocessMode, SingleTest)
|
||||
}
|
||||
|
||||
func TestInprocessMode(t *testing.T) {
|
||||
if InprocessMode {
|
||||
t.Skip("Already tested by TestAccept")
|
||||
}
|
||||
require.Equal(t, 1, testAccept(t, true, "selftest"))
|
||||
}
|
||||
|
||||
func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
|
||||
repls := testdiff.ReplacementsContext{}
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
execPath := BuildCLI(t, cwd)
|
||||
// $CLI is what test scripts are using
|
||||
coverDir := os.Getenv("CLI_GOCOVERDIR")
|
||||
|
||||
if coverDir != "" {
|
||||
require.NoError(t, os.MkdirAll(coverDir, os.ModePerm))
|
||||
coverDir, err = filepath.Abs(coverDir)
|
||||
require.NoError(t, err)
|
||||
t.Logf("Writing coverage to %s", coverDir)
|
||||
}
|
||||
|
||||
execPath := ""
|
||||
|
||||
if InprocessMode {
|
||||
cmdServer := StartCmdServer(t)
|
||||
t.Setenv("CMD_SERVER_URL", cmdServer.URL)
|
||||
execPath = filepath.Join(cwd, "bin", "callserver.py")
|
||||
} else {
|
||||
execPath = BuildCLI(t, cwd, coverDir)
|
||||
}
|
||||
|
||||
t.Setenv("CLI", execPath)
|
||||
repls.SetPath(execPath, "$CLI")
|
||||
|
||||
// Make helper scripts available
|
||||
t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
|
||||
|
||||
server := StartServer(t)
|
||||
AddHandlers(server)
|
||||
// Redirect API access to local server:
|
||||
t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port))
|
||||
t.Setenv("DATABRICKS_TOKEN", "dapi1234")
|
||||
tempHomeDir := t.TempDir()
|
||||
repls.SetPath(tempHomeDir, "$TMPHOME")
|
||||
t.Logf("$TMPHOME=%v", tempHomeDir)
|
||||
|
||||
homeDir := t.TempDir()
|
||||
// Do not read user's ~/.databrickscfg
|
||||
t.Setenv(env.HomeEnvVar(), homeDir)
|
||||
// Prevent CLI from downloading terraform in each test:
|
||||
t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
|
||||
|
||||
repls := testdiff.ReplacementsContext{}
|
||||
repls.Set(execPath, "$CLI")
|
||||
ctx := context.Background()
|
||||
cloudEnv := os.Getenv("CLOUD_ENV")
|
||||
|
||||
if cloudEnv == "" {
|
||||
server := StartServer(t)
|
||||
AddHandlers(server)
|
||||
// Redirect API access to local server:
|
||||
t.Setenv("DATABRICKS_HOST", server.URL)
|
||||
t.Setenv("DATABRICKS_TOKEN", "dapi1234")
|
||||
|
||||
homeDir := t.TempDir()
|
||||
// Do not read user's ~/.databrickscfg
|
||||
t.Setenv(env.HomeEnvVar(), homeDir)
|
||||
}
|
||||
|
||||
workspaceClient, err := databricks.NewWorkspaceClient()
|
||||
require.NoError(t, err)
|
||||
|
||||
user, err := workspaceClient.CurrentUser.Me(ctx)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, user)
|
||||
testdiff.PrepareReplacementsUser(t, &repls, *user)
|
||||
testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient)
|
||||
testdiff.PrepareReplacementsUUID(t, &repls)
|
||||
|
||||
testDirs := getTests(t)
|
||||
require.NotEmpty(t, testDirs)
|
||||
|
||||
if singleTest != "" {
|
||||
testDirs = slices.DeleteFunc(testDirs, func(n string) bool {
|
||||
return n != singleTest
|
||||
})
|
||||
require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs)
|
||||
}
|
||||
|
||||
for _, dir := range testDirs {
|
||||
t.Run(dir, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
runTest(t, dir, repls)
|
||||
testName := strings.ReplaceAll(dir, "\\", "/")
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
if !InprocessMode {
|
||||
t.Parallel()
|
||||
}
|
||||
|
||||
runTest(t, dir, coverDir, repls.Clone())
|
||||
})
|
||||
}
|
||||
|
||||
return len(testDirs)
|
||||
}
|
||||
|
||||
func getTests(t *testing.T) []string {
|
||||
|
|
@ -88,7 +169,14 @@ func getTests(t *testing.T) []string {
|
|||
return testDirs
|
||||
}
|
||||
|
||||
func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
|
||||
func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
|
||||
config, configPath := LoadConfig(t, dir)
|
||||
|
||||
isEnabled, isPresent := config.GOOS[runtime.GOOS]
|
||||
if isPresent && !isEnabled {
|
||||
t.Skipf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath)
|
||||
}
|
||||
|
||||
var tmpDir string
|
||||
var err error
|
||||
if KeepTmp {
|
||||
|
|
@ -101,6 +189,9 @@ func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
|
|||
tmpDir = t.TempDir()
|
||||
}
|
||||
|
||||
repls.SetPathWithParents(tmpDir, "$TMPDIR")
|
||||
repls.Repls = append(repls.Repls, config.Repls...)
|
||||
|
||||
scriptContents := readMergedScriptContents(t, dir)
|
||||
testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
|
||||
|
||||
|
|
@ -111,70 +202,105 @@ func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
|
|||
|
||||
args := []string{"bash", "-euo", "pipefail", EntryPointScript}
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
if coverDir != "" {
|
||||
// Creating individual coverage directory for each test, because writing to the same one
|
||||
// results in sporadic failures like this one (only if tests are running in parallel):
|
||||
// +error: coverage meta-data emit failed: writing ... rename .../tmp.covmeta.b3f... .../covmeta.b3f2c...: no such file or directory
|
||||
coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
|
||||
err := os.MkdirAll(coverDir, os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
|
||||
}
|
||||
|
||||
// Write combined output to a file
|
||||
out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
|
||||
require.NoError(t, err)
|
||||
cmd.Stdout = out
|
||||
cmd.Stderr = out
|
||||
cmd.Dir = tmpDir
|
||||
outB, err := cmd.CombinedOutput()
|
||||
err = cmd.Run()
|
||||
|
||||
out := formatOutput(string(outB), err)
|
||||
out = repls.Replace(out)
|
||||
doComparison(t, filepath.Join(dir, "output.txt"), "script output", out)
|
||||
// Include exit code in output (if non-zero)
|
||||
formatOutput(out, err)
|
||||
require.NoError(t, out.Close())
|
||||
|
||||
for key := range outputs {
|
||||
if key == "output.txt" {
|
||||
// handled above
|
||||
continue
|
||||
}
|
||||
pathNew := filepath.Join(tmpDir, key)
|
||||
newValBytes, err := os.ReadFile(pathNew)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("%s: expected to find this file but could not (%s)", key, tmpDir)
|
||||
} else {
|
||||
t.Errorf("%s: could not read: %s", key, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
pathExpected := filepath.Join(dir, key)
|
||||
newVal := repls.Replace(string(newValBytes))
|
||||
doComparison(t, pathExpected, pathNew, newVal)
|
||||
printedRepls := false
|
||||
|
||||
// Compare expected outputs
|
||||
for relPath := range outputs {
|
||||
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
|
||||
}
|
||||
|
||||
// Make sure there are not unaccounted for new files
|
||||
files, err := os.ReadDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range files {
|
||||
name := f.Name()
|
||||
if _, ok := inputs[name]; ok {
|
||||
files := ListDir(t, tmpDir)
|
||||
for _, relPath := range files {
|
||||
if _, ok := inputs[relPath]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := outputs[name]; ok {
|
||||
if _, ok := outputs[relPath]; ok {
|
||||
continue
|
||||
}
|
||||
t.Errorf("Unexpected output: %s", f)
|
||||
if strings.HasPrefix(name, "out") {
|
||||
if strings.HasPrefix(relPath, "out") {
|
||||
// We have a new file starting with "out"
|
||||
// Show the contents & support overwrite mode for it:
|
||||
pathNew := filepath.Join(tmpDir, name)
|
||||
newVal := testutil.ReadFile(t, pathNew)
|
||||
newVal = repls.Replace(newVal)
|
||||
doComparison(t, filepath.Join(dir, name), filepath.Join(tmpDir, name), newVal)
|
||||
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
|
||||
valueNew = testdiff.NormalizeNewlines(valueNew)
|
||||
valueExpected := string(readIfExists(t, pathExpected))
|
||||
valueExpected = testdiff.NormalizeNewlines(valueExpected)
|
||||
testdiff.AssertEqualTexts(t, pathExpected, pathNew, valueExpected, valueNew)
|
||||
if testdiff.OverwriteMode {
|
||||
if valueNew != "" {
|
||||
t.Logf("Overwriting: %s", pathExpected)
|
||||
testutil.WriteFile(t, pathExpected, valueNew)
|
||||
} else {
|
||||
t.Logf("Removing: %s", pathExpected)
|
||||
_ = os.Remove(pathExpected)
|
||||
func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
|
||||
pathRef := filepath.Join(dirRef, relPath)
|
||||
pathNew := filepath.Join(dirNew, relPath)
|
||||
bufRef, okRef := readIfExists(t, pathRef)
|
||||
bufNew, okNew := readIfExists(t, pathNew)
|
||||
if !okRef && !okNew {
|
||||
t.Errorf("Both files are missing: %s, %s", pathRef, pathNew)
|
||||
return
|
||||
}
|
||||
|
||||
valueRef := testdiff.NormalizeNewlines(string(bufRef))
|
||||
valueNew := testdiff.NormalizeNewlines(string(bufNew))
|
||||
|
||||
// Apply replacements to the new value only.
|
||||
// The reference value is stored after applying replacements.
|
||||
valueNew = repls.Replace(valueNew)
|
||||
|
||||
// The test did not produce an expected output file.
|
||||
if okRef && !okNew {
|
||||
t.Errorf("Missing output file: %s", relPath)
|
||||
testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
|
||||
if testdiff.OverwriteMode {
|
||||
t.Logf("Removing output file: %s", relPath)
|
||||
require.NoError(t, os.Remove(pathRef))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// The test produced an unexpected output file.
|
||||
if !okRef && okNew {
|
||||
t.Errorf("Unexpected output file: %s", relPath)
|
||||
testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
|
||||
if testdiff.OverwriteMode {
|
||||
t.Logf("Writing output file: %s", relPath)
|
||||
testutil.WriteFile(t, pathRef, valueNew)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Compare the reference and new values.
|
||||
equal := testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
|
||||
if !equal && testdiff.OverwriteMode {
|
||||
t.Logf("Overwriting existing output file: %s", relPath)
|
||||
testutil.WriteFile(t, pathRef, valueNew)
|
||||
}
|
||||
|
||||
if !equal && printedRepls != nil && !*printedRepls {
|
||||
*printedRepls = true
|
||||
var items []string
|
||||
for _, item := range repls.Repls {
|
||||
items = append(items, fmt.Sprintf("REPL %s => %s", item.Old, item.New))
|
||||
}
|
||||
t.Log("Available replacements:\n" + strings.Join(items, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -182,17 +308,22 @@ func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) {
|
|||
// Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir.
|
||||
func readMergedScriptContents(t *testing.T, dir string) string {
|
||||
scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript))
|
||||
|
||||
// Wrap script contents in a subshell such that changing the working
|
||||
// directory only affects the main script and not cleanup.
|
||||
scriptContents = "(\n" + scriptContents + ")\n"
|
||||
|
||||
prepares := []string{}
|
||||
cleanups := []string{}
|
||||
|
||||
for {
|
||||
x := readIfExists(t, filepath.Join(dir, CleanupScript))
|
||||
if len(x) > 0 {
|
||||
x, ok := readIfExists(t, filepath.Join(dir, CleanupScript))
|
||||
if ok {
|
||||
cleanups = append(cleanups, string(x))
|
||||
}
|
||||
|
||||
x = readIfExists(t, filepath.Join(dir, PrepareScript))
|
||||
if len(x) > 0 {
|
||||
x, ok = readIfExists(t, filepath.Join(dir, PrepareScript))
|
||||
if ok {
|
||||
prepares = append(prepares, string(x))
|
||||
}
|
||||
|
||||
|
|
@ -210,14 +341,30 @@ func readMergedScriptContents(t *testing.T, dir string) string {
|
|||
return strings.Join(prepares, "\n")
|
||||
}
|
||||
|
||||
func BuildCLI(t *testing.T, cwd string) string {
|
||||
func BuildCLI(t *testing.T, cwd, coverDir string) string {
|
||||
execPath := filepath.Join(cwd, "build", "databricks")
|
||||
if runtime.GOOS == "windows" {
|
||||
execPath += ".exe"
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
|
||||
args := []string{
|
||||
"go", "build",
|
||||
"-mod", "vendor",
|
||||
"-o", execPath,
|
||||
}
|
||||
|
||||
if coverDir != "" {
|
||||
args = append(args, "-cover")
|
||||
}
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
// Get this error on my local Windows:
|
||||
// error obtaining VCS status: exit status 128
|
||||
// Use -buildvcs=false to disable VCS stamping.
|
||||
args = append(args, "-buildvcs=false")
|
||||
}
|
||||
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
cmd.Dir = ".."
|
||||
out, err := cmd.CombinedOutput()
|
||||
|
|
@ -252,29 +399,28 @@ func copyFile(src, dst string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func formatOutput(out string, err error) string {
|
||||
func formatOutput(w io.Writer, err error) {
|
||||
if err == nil {
|
||||
return out
|
||||
return
|
||||
}
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
exitCode := exiterr.ExitCode()
|
||||
out += fmt.Sprintf("\nExit code: %d\n", exitCode)
|
||||
fmt.Fprintf(w, "\nExit code: %d\n", exitCode)
|
||||
} else {
|
||||
out += fmt.Sprintf("\nError: %s\n", err)
|
||||
fmt.Fprintf(w, "\nError: %s\n", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readIfExists(t *testing.T, path string) []byte {
|
||||
func readIfExists(t *testing.T, path string) ([]byte, bool) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err == nil {
|
||||
return data
|
||||
return data, true
|
||||
}
|
||||
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("%s: %s", path, err)
|
||||
}
|
||||
return []byte{}
|
||||
return []byte{}, false
|
||||
}
|
||||
|
||||
func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
|
||||
|
|
@ -289,8 +435,10 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if strings.HasPrefix(name, "out") {
|
||||
outputs[relPath] = true
|
||||
if strings.HasPrefix(relPath, "out") {
|
||||
if !info.IsDir() {
|
||||
outputs[relPath] = true
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
inputs[relPath] = true
|
||||
|
|
@ -309,3 +457,32 @@ func CopyDir(src, dst string, inputs, outputs map[string]bool) error {
|
|||
return copyFile(path, destPath)
|
||||
})
|
||||
}
|
||||
|
||||
func ListDir(t *testing.T, src string) []string {
|
||||
var files []string
|
||||
err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
// Do not FailNow here.
|
||||
// The output comparison is happening after this call which includes output.txt which
|
||||
// includes errors printed by commands which include explanation why a given file cannot be read.
|
||||
t.Errorf("Error when listing %s: path=%s: %s", src, path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(src, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
files = append(files, relPath)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list %s: %s", src, err)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import urllib.request
|
||||
from urllib.parse import urlencode
|
||||
|
||||
env = {}
|
||||
for key, value in os.environ.items():
|
||||
if len(value) > 10_000:
|
||||
sys.stderr.write(f"Dropping key={key} value len={len(value)}\n")
|
||||
continue
|
||||
env[key] = value
|
||||
|
||||
q = {
|
||||
"args": " ".join(sys.argv[1:]),
|
||||
"cwd": os.getcwd(),
|
||||
"env": json.dumps(env),
|
||||
}
|
||||
|
||||
url = os.environ["CMD_SERVER_URL"] + "/?" + urlencode(q)
|
||||
if len(url) > 100_000:
|
||||
sys.exit("url too large")
|
||||
|
||||
resp = urllib.request.urlopen(url)
|
||||
assert resp.status == 200, (resp.status, resp.url, resp.headers)
|
||||
result = json.load(resp)
|
||||
sys.stderr.write(result["stderr"])
|
||||
sys.stdout.write(result["stdout"])
|
||||
exitcode = int(result["exitcode"])
|
||||
sys.exit(exitcode)
|
||||
|
|
@ -4,6 +4,7 @@ Helper to sort blocks in text file. A block is a set of lines separated from oth
|
|||
|
||||
This is to workaround non-determinism in the output.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
blocks = []
|
||||
|
|
@ -11,10 +12,10 @@ blocks = []
|
|||
for line in sys.stdin:
|
||||
if not line.strip():
|
||||
if blocks and blocks[-1]:
|
||||
blocks.append('')
|
||||
blocks.append("")
|
||||
continue
|
||||
if not blocks:
|
||||
blocks.append('')
|
||||
blocks.append("")
|
||||
blocks[-1] += line
|
||||
|
||||
blocks.sort()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
bundle:
|
||||
name: git-permerror
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case.
|
||||
|
||||
>>> chmod 000 .git
|
||||
|
||||
>>> $CLI bundle validate
|
||||
Error: unable to load repository specific gitconfig: open config: permission denied
|
||||
|
||||
Name: git-permerror
|
||||
Target: default
|
||||
Workspace:
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/git-permerror/default
|
||||
|
||||
Found 1 error
|
||||
|
||||
Exit code: 1
|
||||
|
||||
>>> $CLI bundle validate -o json
|
||||
Error: unable to load repository specific gitconfig: open config: permission denied
|
||||
|
||||
|
||||
Exit code: 1
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
||||
>>> withdir subdir/a/b $CLI bundle validate -o json
|
||||
Error: unable to load repository specific gitconfig: open config: permission denied
|
||||
|
||||
|
||||
Exit code: 1
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
||||
|
||||
=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case.
|
||||
|
||||
>>> chmod 000 .git/HEAD
|
||||
|
||||
>>> $CLI bundle validate -o json
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
||||
>>> withdir subdir/a/b $CLI bundle validate -o json
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
||||
|
||||
=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case.
|
||||
|
||||
>>> chmod 000 .git/config
|
||||
|
||||
>>> $CLI bundle validate -o json
|
||||
Error: unable to load repository specific gitconfig: open config: permission denied
|
||||
|
||||
|
||||
Exit code: 1
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
||||
>>> withdir subdir/a/b $CLI bundle validate -o json
|
||||
Error: unable to load repository specific gitconfig: open config: permission denied
|
||||
|
||||
|
||||
Exit code: 1
|
||||
{
|
||||
"bundle_root_path": ".",
|
||||
"inferred": true
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
mkdir myrepo
|
||||
cd myrepo
|
||||
cp ../databricks.yml .
|
||||
git-repo-init
|
||||
mkdir -p subdir/a/b
|
||||
|
||||
printf "=== No permission to access .git. Badness: inferred flag is set to true even though we did not infer branch. bundle_root_path is not correct in subdir case.\n"
|
||||
trace chmod 000 .git
|
||||
errcode trace $CLI bundle validate
|
||||
errcode trace $CLI bundle validate -o json | jq .bundle.git
|
||||
errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
|
||||
|
||||
printf "\n\n=== No permissions to read .git/HEAD. Badness: warning is not shown. inferred is incorrectly set to true. bundle_root_path is not correct in subdir case.\n"
|
||||
chmod 700 .git
|
||||
trace chmod 000 .git/HEAD
|
||||
errcode trace $CLI bundle validate -o json | jq .bundle.git
|
||||
errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
|
||||
|
||||
printf "\n\n=== No permissions to read .git/config. Badness: inferred is incorretly set to true. bundle_root_path is not correct is subdir case.\n"
|
||||
chmod 666 .git/HEAD
|
||||
trace chmod 000 .git/config
|
||||
errcode trace $CLI bundle validate -o json | jq .bundle.git
|
||||
errcode trace withdir subdir/a/b $CLI bundle validate -o json | jq .bundle.git
|
||||
|
||||
rm -fr .git
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct"
|
||||
|
||||
[GOOS]
|
||||
# This test relies on chmod which does not work on Windows
|
||||
windows = false
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
|
||||
>>> $CLI bundle deploy --help
|
||||
Deploy bundle
|
||||
|
||||
Usage:
|
||||
databricks bundle deploy [flags]
|
||||
|
||||
Flags:
|
||||
--auto-approve Skip interactive approvals that might be required for deployment.
|
||||
-c, --cluster-id string Override cluster in the deployment with the given cluster ID.
|
||||
--fail-on-active-runs Fail if there are running jobs or pipelines in the deployment.
|
||||
--force Force-override Git branch validation.
|
||||
--force-lock Force acquisition of deployment lock.
|
||||
-h, --help help for deploy
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle deploy --help
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
|
||||
>>> $CLI bundle deployment --help
|
||||
Deployment related commands
|
||||
|
||||
Usage:
|
||||
databricks bundle deployment [command]
|
||||
|
||||
Available Commands:
|
||||
bind Bind bundle-defined resources to existing resources
|
||||
unbind Unbind bundle-defined resources from its managed remote resource
|
||||
|
||||
Flags:
|
||||
-h, --help help for deployment
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
||||
Use "databricks bundle deployment [command] --help" for more information about a command.
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle deployment --help
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
>>> $CLI bundle destroy --help
|
||||
Destroy deployed bundle resources
|
||||
|
||||
Usage:
|
||||
databricks bundle destroy [flags]
|
||||
|
||||
Flags:
|
||||
--auto-approve Skip interactive approvals for deleting resources and files
|
||||
--force-lock Force acquisition of deployment lock.
|
||||
-h, --help help for destroy
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle destroy --help
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
|
||||
>>> $CLI bundle generate dashboard --help
|
||||
Generate configuration for a dashboard
|
||||
|
||||
Usage:
|
||||
databricks bundle generate dashboard [flags]
|
||||
|
||||
Flags:
|
||||
-s, --dashboard-dir string directory to write the dashboard representation to (default "src")
|
||||
--existing-id string ID of the dashboard to generate configuration for
|
||||
--existing-path string workspace path of the dashboard to generate configuration for
|
||||
-f, --force force overwrite existing files in the output directory
|
||||
-h, --help help for dashboard
|
||||
--resource string resource key of dashboard to watch for changes
|
||||
-d, --resource-dir string directory to write the configuration to (default "resources")
|
||||
--watch watch for changes to the dashboard and update the configuration
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
--key string resource key to use for the generated configuration
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle generate dashboard --help
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
|
||||
>>> $CLI bundle generate job --help
|
||||
Generate bundle configuration for a job
|
||||
|
||||
Usage:
|
||||
databricks bundle generate job [flags]
|
||||
|
||||
Flags:
|
||||
-d, --config-dir string Dir path where the output config will be stored (default "resources")
|
||||
--existing-job-id int Job ID of the job to generate config for
|
||||
-f, --force Force overwrite existing files in the output directory
|
||||
-h, --help help for job
|
||||
-s, --source-dir string Dir path where the downloaded files will be stored (default "src")
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
--key string resource key to use for the generated configuration
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle generate job --help
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
|
||||
>>> $CLI bundle generate pipeline --help
|
||||
Generate bundle configuration for a pipeline
|
||||
|
||||
Usage:
|
||||
databricks bundle generate pipeline [flags]
|
||||
|
||||
Flags:
|
||||
-d, --config-dir string Dir path where the output config will be stored (default "resources")
|
||||
--existing-pipeline-id string ID of the pipeline to generate config for
|
||||
-f, --force Force overwrite existing files in the output directory
|
||||
-h, --help help for pipeline
|
||||
-s, --source-dir string Dir path where the downloaded files will be stored (default "src")
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
--key string resource key to use for the generated configuration
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle generate pipeline --help
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
|
||||
>>> $CLI bundle generate --help
|
||||
Generate bundle configuration
|
||||
|
||||
Usage:
|
||||
databricks bundle generate [command]
|
||||
|
||||
Available Commands:
|
||||
app Generate bundle configuration for a Databricks app
|
||||
dashboard Generate configuration for a dashboard
|
||||
job Generate bundle configuration for a job
|
||||
pipeline Generate bundle configuration for a pipeline
|
||||
|
||||
Flags:
|
||||
-h, --help help for generate
|
||||
--key string resource key to use for the generated configuration
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
||||
Use "databricks bundle generate [command] --help" for more information about a command.
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle generate --help
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
|
||||
>>> $CLI bundle init --help
|
||||
Initialize using a bundle template.
|
||||
|
||||
TEMPLATE_PATH optionally specifies which template to use. It can be one of the following:
|
||||
- default-python: The default Python template for Notebooks / Delta Live Tables / Workflows
|
||||
- default-sql: The default SQL template for .sql files that run with Databricks SQL
|
||||
- dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)
|
||||
- mlops-stacks: The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)
|
||||
- a local file system path with a template directory
|
||||
- a Git repository URL, e.g. https://github.com/my/repository
|
||||
|
||||
See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more information on templates.
|
||||
|
||||
Usage:
|
||||
databricks bundle init [TEMPLATE_PATH] [flags]
|
||||
|
||||
Flags:
|
||||
--branch string Git branch to use for template initialization
|
||||
--config-file string JSON file containing key value pairs of input parameters required for template initialization.
|
||||
-h, --help help for init
|
||||
--output-dir string Directory to write the initialized template to.
|
||||
--tag string Git tag to use for template initialization
|
||||
--template-dir string Directory path within a Git repository containing the template.
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle init --help
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
|
||||
>>> $CLI bundle open --help
|
||||
Open a resource in the browser
|
||||
|
||||
Usage:
|
||||
databricks bundle open [flags]
|
||||
|
||||
Flags:
|
||||
--force-pull Skip local cache and load the state from the remote workspace
|
||||
-h, --help help for open
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle open --help
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
|
||||
>>> $CLI bundle run --help
|
||||
Run the job or pipeline identified by KEY.
|
||||
|
||||
The KEY is the unique identifier of the resource to run. In addition to
|
||||
customizing the run using any of the available flags, you can also specify
|
||||
keyword or positional arguments as shown in these examples:
|
||||
|
||||
databricks bundle run my_job -- --key1 value1 --key2 value2
|
||||
|
||||
Or:
|
||||
|
||||
databricks bundle run my_job -- value1 value2 value3
|
||||
|
||||
If the specified job uses job parameters or the job has a notebook task with
|
||||
parameters, the first example applies and flag names are mapped to the
|
||||
parameter names.
|
||||
|
||||
If the specified job does not use job parameters and the job has a Python file
|
||||
task or a Python wheel task, the second example applies.
|
||||
|
||||
Usage:
|
||||
databricks bundle run [flags] KEY
|
||||
|
||||
Job Flags:
|
||||
--params stringToString comma separated k=v pairs for job parameters (default [])
|
||||
|
||||
Job Task Flags:
|
||||
Note: please prefer use of job-level parameters (--param) over task-level parameters.
|
||||
For more information, see https://docs.databricks.com/en/workflows/jobs/create-run-jobs.html#pass-parameters-to-a-databricks-job-task
|
||||
--dbt-commands strings A list of commands to execute for jobs with DBT tasks.
|
||||
--jar-params strings A list of parameters for jobs with Spark JAR tasks.
|
||||
--notebook-params stringToString A map from keys to values for jobs with notebook tasks. (default [])
|
||||
--pipeline-params stringToString A map from keys to values for jobs with pipeline tasks. (default [])
|
||||
--python-named-params stringToString A map from keys to values for jobs with Python wheel tasks. (default [])
|
||||
--python-params strings A list of parameters for jobs with Python tasks.
|
||||
--spark-submit-params strings A list of parameters for jobs with Spark submit tasks.
|
||||
--sql-params stringToString A map from keys to values for jobs with SQL tasks. (default [])
|
||||
|
||||
Pipeline Flags:
|
||||
--full-refresh strings List of tables to reset and recompute.
|
||||
--full-refresh-all Perform a full graph reset and recompute.
|
||||
--refresh strings List of tables to update.
|
||||
--refresh-all Perform a full graph update.
|
||||
--validate-only Perform an update to validate graph correctness.
|
||||
|
||||
Flags:
|
||||
-h, --help help for run
|
||||
--no-wait Don't wait for the run to complete.
|
||||
--restart Restart the run if it is already running.
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle run --help
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
|
||||
>>> $CLI bundle schema --help
|
||||
Generate JSON Schema for bundle configuration
|
||||
|
||||
Usage:
|
||||
databricks bundle schema [flags]
|
||||
|
||||
Flags:
|
||||
-h, --help help for schema
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle schema --help
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
|
||||
>>> $CLI bundle summary --help
|
||||
Summarize resources deployed by this bundle
|
||||
|
||||
Usage:
|
||||
databricks bundle summary [flags]
|
||||
|
||||
Flags:
|
||||
--force-pull Skip local cache and load the state from the remote workspace
|
||||
-h, --help help for summary
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle summary --help
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
|
||||
>>> $CLI bundle sync --help
|
||||
Synchronize bundle tree to the workspace
|
||||
|
||||
Usage:
|
||||
databricks bundle sync [flags]
|
||||
|
||||
Flags:
|
||||
--full perform full synchronization (default is incremental)
|
||||
-h, --help help for sync
|
||||
--interval duration file system polling interval (for --watch) (default 1s)
|
||||
--output type type of the output format
|
||||
--watch watch local file system for changes
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle sync --help
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
|
||||
>>> $CLI bundle validate --help
|
||||
Validate configuration
|
||||
|
||||
Usage:
|
||||
databricks bundle validate [flags]
|
||||
|
||||
Flags:
|
||||
-h, --help help for validate
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle validate --help
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
|
||||
>>> $CLI bundle --help
|
||||
Databricks Asset Bundles let you express data/AI/analytics projects as code.
|
||||
|
||||
Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html
|
||||
|
||||
Usage:
|
||||
databricks bundle [command]
|
||||
|
||||
Available Commands:
|
||||
deploy Deploy bundle
|
||||
deployment Deployment related commands
|
||||
destroy Destroy deployed bundle resources
|
||||
generate Generate bundle configuration
|
||||
init Initialize using a bundle template
|
||||
open Open a resource in the browser
|
||||
run Run a job or pipeline update
|
||||
schema Generate JSON Schema for bundle configuration
|
||||
summary Summarize resources deployed by this bundle
|
||||
sync Synchronize bundle tree to the workspace
|
||||
validate Validate configuration
|
||||
|
||||
Flags:
|
||||
-h, --help help for bundle
|
||||
--var strings set values for variables defined in bundle config. Example: --var="foo=bar"
|
||||
|
||||
Global Flags:
|
||||
--debug enable debug logging
|
||||
-o, --output type output type: text or json (default text)
|
||||
-p, --profile string ~/.databrickscfg profile
|
||||
-t, --target string bundle target to use (if applicable)
|
||||
|
||||
Use "databricks bundle [command] --help" for more information about a command.
|
||||
|
|
@ -0,0 +1 @@
|
|||
trace $CLI bundle --help
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
bundle:
|
||||
name: non_yaml_in_includes
|
||||
|
||||
include:
|
||||
- test.py
|
||||
- resources/*.yml
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
Error: Files in the 'include' configuration section must be YAML files.
|
||||
in databricks.yml:5:4
|
||||
|
||||
The file test.py in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.
|
||||
|
||||
Name: non_yaml_in_includes
|
||||
|
||||
Found 1 error
|
||||
|
||||
Exit code: 1
|
||||
|
|
@ -0,0 +1 @@
|
|||
$CLI bundle validate
|
||||
|
|
@ -0,0 +1 @@
|
|||
print("Hello world")
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
"foo": {
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"format": "MULTI_TASK",
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
"foo": {
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"format": "MULTI_TASK",
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ targets:
|
|||
jobs:
|
||||
foo:
|
||||
job_clusters:
|
||||
# This does not work because merging is done before resolution
|
||||
- job_cluster_key: "${var.mykey}"
|
||||
new_cluster:
|
||||
node_type_id: i3.xlarge
|
||||
|
|
|
|||
|
|
@ -4,22 +4,17 @@
|
|||
"foo": {
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/development/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"format": "MULTI_TASK",
|
||||
"job_clusters": [
|
||||
{
|
||||
"job_cluster_key": "key",
|
||||
"new_cluster": {
|
||||
"spark_version": "13.3.x-scala2.12"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "key",
|
||||
"new_cluster": {
|
||||
"node_type_id": "i3.xlarge",
|
||||
"num_workers": 1
|
||||
"num_workers": 1,
|
||||
"spark_version": "13.3.x-scala2.12"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -36,8 +31,8 @@
|
|||
Name: override_job_cluster
|
||||
Target: development
|
||||
Workspace:
|
||||
User: tester@databricks.com
|
||||
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/development
|
||||
|
||||
Validation OK!
|
||||
|
||||
|
|
@ -46,22 +41,17 @@ Validation OK!
|
|||
"foo": {
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"format": "MULTI_TASK",
|
||||
"job_clusters": [
|
||||
{
|
||||
"job_cluster_key": "key",
|
||||
"new_cluster": {
|
||||
"spark_version": "13.3.x-scala2.12"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "key",
|
||||
"new_cluster": {
|
||||
"node_type_id": "i3.2xlarge",
|
||||
"num_workers": 4
|
||||
"num_workers": 4,
|
||||
"spark_version": "13.3.x-scala2.12"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
|
@ -78,7 +68,7 @@ Validation OK!
|
|||
Name: override_job_cluster
|
||||
Target: staging
|
||||
Workspace:
|
||||
User: tester@databricks.com
|
||||
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/override_job_cluster/staging
|
||||
|
||||
Validation OK!
|
||||
|
|
|
|||
|
|
@ -69,8 +69,8 @@ Error: file ./test1.py not found
|
|||
Name: override_job_tasks
|
||||
Target: staging
|
||||
Workspace:
|
||||
User: tester@databricks.com
|
||||
Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_tasks/staging
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/override_job_tasks/staging
|
||||
|
||||
Found 1 error
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Warning: expected map, found string
|
|||
Name: merge-string-map
|
||||
Target: dev
|
||||
Workspace:
|
||||
User: tester@databricks.com
|
||||
Path: /Workspace/Users/tester@databricks.com/.bundle/merge-string-map/dev
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/merge-string-map/dev
|
||||
|
||||
Found 1 warning
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
],
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/development/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/development/state/metadata.json"
|
||||
},
|
||||
"name": "job",
|
||||
"permissions": []
|
||||
|
|
@ -36,7 +36,7 @@
|
|||
],
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_pipeline_cluster/staging/state/metadata.json"
|
||||
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/override_pipeline_cluster/staging/state/metadata.json"
|
||||
},
|
||||
"name": "job",
|
||||
"permissions": []
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
bundle:
|
||||
name: path_translation_nominal
|
||||
name: fallback
|
||||
|
||||
include:
|
||||
- "resources/*.yml"
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
[
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"notebook_task": {
|
||||
"notebook_path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook"
|
||||
},
|
||||
"task_key": "notebook_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"spark_python_task": {
|
||||
"python_file": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file.py"
|
||||
},
|
||||
"task_key": "spark_python_example"
|
||||
},
|
||||
{
|
||||
"dbt_task": {
|
||||
"commands": [
|
||||
"dbt run",
|
||||
"dbt run"
|
||||
],
|
||||
"project_directory": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/dbt_project"
|
||||
},
|
||||
"job_cluster_key": "default",
|
||||
"task_key": "dbt_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"sql_task": {
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/sql.sql"
|
||||
},
|
||||
"warehouse_id": "cafef00d"
|
||||
},
|
||||
"task_key": "sql_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "dist/wheel1.whl"
|
||||
},
|
||||
{
|
||||
"whl": "dist/wheel2.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"package_name": "my_package"
|
||||
},
|
||||
"task_key": "python_wheel_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"libraries": [
|
||||
{
|
||||
"jar": "target/jar1.jar"
|
||||
},
|
||||
{
|
||||
"jar": "target/jar2.jar"
|
||||
}
|
||||
],
|
||||
"spark_jar_task": {
|
||||
"main_class_name": "com.example.Main"
|
||||
},
|
||||
"task_key": "spark_jar_example"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[
|
||||
{
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file1.py"
|
||||
}
|
||||
},
|
||||
{
|
||||
"notebook": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/file2.py"
|
||||
}
|
||||
},
|
||||
{
|
||||
"notebook": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/fallback/development/files/src/notebook2"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
|
||||
>>> $CLI bundle validate -t development -o json
|
||||
|
||||
>>> $CLI bundle validate -t error
|
||||
Error: notebook this value is overridden not found. Local notebook references are expected
|
||||
to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
|
||||
|
||||
Name: fallback
|
||||
Target: error
|
||||
Workspace:
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/fallback/error
|
||||
|
||||
Found 1 error
|
||||
|
||||
Exit code: 1
|
||||
|
|
@ -4,33 +4,45 @@ resources:
|
|||
name: "placeholder"
|
||||
tasks:
|
||||
- task_key: notebook_example
|
||||
job_cluster_key: default
|
||||
notebook_task:
|
||||
notebook_path: "this value is overridden"
|
||||
|
||||
- task_key: spark_python_example
|
||||
job_cluster_key: default
|
||||
spark_python_task:
|
||||
python_file: "this value is overridden"
|
||||
|
||||
- task_key: dbt_example
|
||||
job_cluster_key: default
|
||||
dbt_task:
|
||||
project_directory: "this value is overridden"
|
||||
commands:
|
||||
- "dbt run"
|
||||
|
||||
- task_key: sql_example
|
||||
job_cluster_key: default
|
||||
sql_task:
|
||||
file:
|
||||
path: "this value is overridden"
|
||||
warehouse_id: cafef00d
|
||||
|
||||
- task_key: python_wheel_example
|
||||
job_cluster_key: default
|
||||
python_wheel_task:
|
||||
package_name: my_package
|
||||
libraries:
|
||||
- whl: ../dist/wheel1.whl
|
||||
|
||||
- task_key: spark_jar_example
|
||||
job_cluster_key: default
|
||||
spark_jar_task:
|
||||
main_class_name: com.example.Main
|
||||
libraries:
|
||||
- jar: ../target/jar1.jar
|
||||
|
||||
# Include a job cluster for completeness
|
||||
job_clusters:
|
||||
- job_cluster_key: default
|
||||
new_cluster:
|
||||
spark_version: 15.4.x-scala2.12
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
errcode trace $CLI bundle validate -t development -o json > output.tmp.json
|
||||
|
||||
# Capture job tasks
|
||||
jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
|
||||
|
||||
# Capture pipeline libraries
|
||||
jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
|
||||
|
||||
# Expect failure for the "error" target
|
||||
errcode trace $CLI bundle validate -t error
|
||||
|
|
@ -0,0 +1 @@
|
|||
rm -f output.tmp.json
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
bundle:
|
||||
name: path_translation_fallback
|
||||
name: nominal
|
||||
|
||||
include:
|
||||
- "resources/*.yml"
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
[
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"notebook_task": {
|
||||
"notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
|
||||
},
|
||||
"task_key": "notebook_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"spark_python_task": {
|
||||
"python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
|
||||
},
|
||||
"task_key": "spark_python_example"
|
||||
},
|
||||
{
|
||||
"dbt_task": {
|
||||
"commands": [
|
||||
"dbt run",
|
||||
"dbt run"
|
||||
],
|
||||
"project_directory": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/dbt_project"
|
||||
},
|
||||
"job_cluster_key": "default",
|
||||
"task_key": "dbt_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"sql_task": {
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/sql.sql"
|
||||
},
|
||||
"warehouse_id": "cafef00d"
|
||||
},
|
||||
"task_key": "sql_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "dist/wheel1.whl"
|
||||
},
|
||||
{
|
||||
"whl": "dist/wheel2.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"package_name": "my_package"
|
||||
},
|
||||
"task_key": "python_wheel_example"
|
||||
},
|
||||
{
|
||||
"job_cluster_key": "default",
|
||||
"libraries": [
|
||||
{
|
||||
"jar": "target/jar1.jar"
|
||||
},
|
||||
{
|
||||
"jar": "target/jar2.jar"
|
||||
}
|
||||
],
|
||||
"spark_jar_task": {
|
||||
"main_class_name": "com.example.Main"
|
||||
},
|
||||
"task_key": "spark_jar_example"
|
||||
},
|
||||
{
|
||||
"for_each_task": {
|
||||
"task": {
|
||||
"notebook_task": {
|
||||
"notebook_path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook"
|
||||
}
|
||||
}
|
||||
},
|
||||
"job_cluster_key": "default",
|
||||
"task_key": "for_each_notebook_example"
|
||||
},
|
||||
{
|
||||
"for_each_task": {
|
||||
"task": {
|
||||
"job_cluster_key": "default",
|
||||
"spark_python_task": {
|
||||
"python_file": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file.py"
|
||||
}
|
||||
}
|
||||
},
|
||||
"task_key": "for_each_spark_python_example"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
[
|
||||
{
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file1.py"
|
||||
}
|
||||
},
|
||||
{
|
||||
"notebook": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"file": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/file2.py"
|
||||
}
|
||||
},
|
||||
{
|
||||
"notebook": {
|
||||
"path": "/Workspace/Users/$USERNAME/.bundle/nominal/development/files/src/notebook2"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
|
||||
>>> $CLI bundle validate -t development -o json
|
||||
|
||||
>>> $CLI bundle validate -t error
|
||||
Error: notebook this value is overridden not found. Local notebook references are expected
|
||||
to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
|
||||
|
||||
Name: nominal
|
||||
Target: error
|
||||
Workspace:
|
||||
User: $USERNAME
|
||||
Path: /Workspace/Users/$USERNAME/.bundle/nominal/error
|
||||
|
||||
Found 1 error
|
||||
|
||||
Exit code: 1
|
||||
|
|
@ -4,38 +4,45 @@ resources:
|
|||
name: "placeholder"
|
||||
tasks:
|
||||
- task_key: notebook_example
|
||||
job_cluster_key: default
|
||||
notebook_task:
|
||||
notebook_path: "this value is overridden"
|
||||
|
||||
- task_key: spark_python_example
|
||||
job_cluster_key: default
|
||||
spark_python_task:
|
||||
python_file: "this value is overridden"
|
||||
|
||||
- task_key: dbt_example
|
||||
job_cluster_key: default
|
||||
dbt_task:
|
||||
project_directory: "this value is overridden"
|
||||
commands:
|
||||
- "dbt run"
|
||||
|
||||
- task_key: sql_example
|
||||
job_cluster_key: default
|
||||
sql_task:
|
||||
file:
|
||||
path: "this value is overridden"
|
||||
warehouse_id: cafef00d
|
||||
|
||||
- task_key: python_wheel_example
|
||||
job_cluster_key: default
|
||||
python_wheel_task:
|
||||
package_name: my_package
|
||||
libraries:
|
||||
- whl: ../dist/wheel1.whl
|
||||
|
||||
- task_key: spark_jar_example
|
||||
job_cluster_key: default
|
||||
spark_jar_task:
|
||||
main_class_name: com.example.Main
|
||||
libraries:
|
||||
- jar: ../target/jar1.jar
|
||||
|
||||
- task_key: for_each_notebook_example
|
||||
job_cluster_key: default
|
||||
for_each_task:
|
||||
task:
|
||||
notebook_task:
|
||||
|
|
@ -44,5 +51,12 @@ resources:
|
|||
- task_key: for_each_spark_python_example
|
||||
for_each_task:
|
||||
task:
|
||||
job_cluster_key: default
|
||||
spark_python_task:
|
||||
python_file: "this value is overridden"
|
||||
|
||||
# Include a job cluster for completeness
|
||||
job_clusters:
|
||||
- job_cluster_key: default
|
||||
new_cluster:
|
||||
spark_version: 15.4.x-scala2.12
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
errcode trace $CLI bundle validate -t development -o json > output.tmp.json
|
||||
|
||||
# Capture job tasks
|
||||
jq '.resources.jobs.my_job.tasks' output.tmp.json > output.job.json
|
||||
|
||||
# Capture pipeline libraries
|
||||
jq '.resources.pipelines.my_pipeline.libraries' output.tmp.json > output.pipeline.json
|
||||
|
||||
# Expect failure for the "error" target
|
||||
errcode trace $CLI bundle validate -t error
|
||||
|
|
@ -0,0 +1 @@
|
|||
rm -f output.tmp.json
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue