mirror of https://github.com/databricks/cli.git
Compare commits
No commits in common. "c296299ad29b2a74a4b8d542d5da4d3f5927ddea" and "50af744a974f7ba3bc6ad1dbb406a4c007862628" have entirely different histories.
c296299ad2
...
50af744a97
|
@ -5,13 +5,14 @@
|
||||||
},
|
},
|
||||||
"batch": {
|
"batch": {
|
||||||
".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go",
|
".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go",
|
||||||
".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
|
".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go",
|
||||||
|
".codegen/lookup.go.tmpl": "bundle/config/variable/lookup.go"
|
||||||
},
|
},
|
||||||
"toolchain": {
|
"toolchain": {
|
||||||
"required": ["go"],
|
"required": ["go"],
|
||||||
"post_generate": [
|
"post_generate": [
|
||||||
"go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
|
"go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
|
||||||
"make schema",
|
"go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
|
||||||
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
|
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
|
||||||
"echo 'go.sum linguist-generated=true' >> ./.gitattributes",
|
"echo 'go.sum linguist-generated=true' >> ./.gitattributes",
|
||||||
"echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"
|
"echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d
|
d25296d2f4aa7bd6195c816fdf82e0f960f775da
|
|
@ -0,0 +1,134 @@
|
||||||
|
// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
|
||||||
|
|
||||||
|
package variable
|
||||||
|
|
||||||
|
{{ $allowlist :=
|
||||||
|
list
|
||||||
|
"alerts"
|
||||||
|
"clusters"
|
||||||
|
"cluster-policies"
|
||||||
|
"clusters"
|
||||||
|
"dashboards"
|
||||||
|
"instance-pools"
|
||||||
|
"jobs"
|
||||||
|
"metastores"
|
||||||
|
"pipelines"
|
||||||
|
"service-principals"
|
||||||
|
"queries"
|
||||||
|
"warehouses"
|
||||||
|
}}
|
||||||
|
|
||||||
|
{{ $customField :=
|
||||||
|
dict
|
||||||
|
"service-principals" "ApplicationId"
|
||||||
|
}}
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/databricks/databricks-sdk-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Lookup struct {
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
{{.Singular.PascalName}} string `json:"{{.Singular.SnakeName}},omitempty"`
|
||||||
|
|
||||||
|
{{end}}
|
||||||
|
{{- end}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func LookupFromMap(m map[string]any) *Lookup {
|
||||||
|
l := &Lookup{}
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
if v, ok := m["{{.Singular.SnakeName}}"]; ok {
|
||||||
|
l.{{.Singular.PascalName}} = v.(string)
|
||||||
|
}
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
||||||
|
if err := l.validate(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
r := allResolvers()
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
if l.{{.Singular.PascalName}} != "" {
|
||||||
|
return r.{{.Singular.PascalName}}(ctx, w, l.{{.Singular.PascalName}})
|
||||||
|
}
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("no valid lookup fields provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lookup) String() string {
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
if l.{{.Singular.PascalName}} != "" {
|
||||||
|
return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}})
|
||||||
|
}
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lookup) validate() error {
|
||||||
|
// Validate that only one field is set
|
||||||
|
count := 0
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
if l.{{.Singular.PascalName}} != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
|
||||||
|
if count != 1 {
|
||||||
|
return fmt.Errorf("exactly one lookup field must be provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(l.String(), "${var") {
|
||||||
|
return fmt.Errorf("lookup fields cannot contain variable references")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error)
|
||||||
|
type resolvers struct {
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
{{.Singular.PascalName}} resolverFunc
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func allResolvers() *resolvers {
|
||||||
|
r := &resolvers{}
|
||||||
|
{{range .Services -}}
|
||||||
|
{{- if in $allowlist .KebabName -}}
|
||||||
|
r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["{{.Singular.PascalName}}"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .NamedIdMap.IdPath 0).PascalName) }}), nil
|
||||||
|
}
|
||||||
|
{{end -}}
|
||||||
|
{{- end}}
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
|
@ -411,5 +411,5 @@ func new{{.PascalName}}() *cobra.Command {
|
||||||
{{- define "request-body-obj" -}}
|
{{- define "request-body-obj" -}}
|
||||||
{{- $method := .Method -}}
|
{{- $method := .Method -}}
|
||||||
{{- $field := .Field -}}
|
{{- $field := .Field -}}
|
||||||
{{$method.CamelName}}Req{{ if (and $method.RequestBodyField (and (not $field.IsPath) (not $field.IsQuery))) }}.{{$method.RequestBodyField.PascalName}}{{end}}.{{$field.PascalName}}
|
{{$method.CamelName}}Req{{ if (and $method.RequestBodyField (not $field.IsPath)) }}.{{$method.RequestBodyField.PascalName}}{{end}}.{{$field.PascalName}}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
|
@ -1,8 +0,0 @@
|
||||||
# Enable gofumpt and goimports in golangci-lint (#1999)
|
|
||||||
2e018cfaec200a02ee2bd5b389e7da3c6f15f460
|
|
||||||
|
|
||||||
# Enable errcheck everywhere and fix or silent remaining issues (#1987)
|
|
||||||
8d5351c1c3d7befda4baae5d6adb99367aa50b3c
|
|
||||||
|
|
||||||
# Add error checking in tests and enable errcheck there (#1980)
|
|
||||||
1b2be1b2cb4b7909df2a8ad4cb6a0f43e8fcf0c6
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
bundle/config/variable/lookup.go linguist-generated=true
|
||||||
cmd/account/access-control/access-control.go linguist-generated=true
|
cmd/account/access-control/access-control.go linguist-generated=true
|
||||||
cmd/account/billable-usage/billable-usage.go linguist-generated=true
|
cmd/account/billable-usage/billable-usage.go linguist-generated=true
|
||||||
cmd/account/budgets/budgets.go linguist-generated=true
|
cmd/account/budgets/budgets.go linguist-generated=true
|
||||||
|
@ -8,7 +9,6 @@ cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=
|
||||||
cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
|
cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true
|
||||||
cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
|
cmd/account/encryption-keys/encryption-keys.go linguist-generated=true
|
||||||
cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
|
cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true
|
||||||
cmd/account/federation-policy/federation-policy.go linguist-generated=true
|
|
||||||
cmd/account/groups/groups.go linguist-generated=true
|
cmd/account/groups/groups.go linguist-generated=true
|
||||||
cmd/account/ip-access-lists/ip-access-lists.go linguist-generated=true
|
cmd/account/ip-access-lists/ip-access-lists.go linguist-generated=true
|
||||||
cmd/account/log-delivery/log-delivery.go linguist-generated=true
|
cmd/account/log-delivery/log-delivery.go linguist-generated=true
|
||||||
|
@ -20,7 +20,6 @@ cmd/account/o-auth-published-apps/o-auth-published-apps.go linguist-generated=tr
|
||||||
cmd/account/personal-compute/personal-compute.go linguist-generated=true
|
cmd/account/personal-compute/personal-compute.go linguist-generated=true
|
||||||
cmd/account/private-access/private-access.go linguist-generated=true
|
cmd/account/private-access/private-access.go linguist-generated=true
|
||||||
cmd/account/published-app-integration/published-app-integration.go linguist-generated=true
|
cmd/account/published-app-integration/published-app-integration.go linguist-generated=true
|
||||||
cmd/account/service-principal-federation-policy/service-principal-federation-policy.go linguist-generated=true
|
|
||||||
cmd/account/service-principal-secrets/service-principal-secrets.go linguist-generated=true
|
cmd/account/service-principal-secrets/service-principal-secrets.go linguist-generated=true
|
||||||
cmd/account/service-principals/service-principals.go linguist-generated=true
|
cmd/account/service-principals/service-principals.go linguist-generated=true
|
||||||
cmd/account/settings/settings.go linguist-generated=true
|
cmd/account/settings/settings.go linguist-generated=true
|
||||||
|
@ -39,9 +38,6 @@ cmd/workspace/apps/apps.go linguist-generated=true
|
||||||
cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true
|
cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true
|
||||||
cmd/workspace/automatic-cluster-update/automatic-cluster-update.go linguist-generated=true
|
cmd/workspace/automatic-cluster-update/automatic-cluster-update.go linguist-generated=true
|
||||||
cmd/workspace/catalogs/catalogs.go linguist-generated=true
|
cmd/workspace/catalogs/catalogs.go linguist-generated=true
|
||||||
cmd/workspace/clean-room-assets/clean-room-assets.go linguist-generated=true
|
|
||||||
cmd/workspace/clean-room-task-runs/clean-room-task-runs.go linguist-generated=true
|
|
||||||
cmd/workspace/clean-rooms/clean-rooms.go linguist-generated=true
|
|
||||||
cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true
|
cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true
|
||||||
cmd/workspace/clusters/clusters.go linguist-generated=true
|
cmd/workspace/clusters/clusters.go linguist-generated=true
|
||||||
cmd/workspace/cmd.go linguist-generated=true
|
cmd/workspace/cmd.go linguist-generated=true
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
* @pietern @andrewnester @shreyas-goenka @denik
|
|
|
@ -1,32 +0,0 @@
|
||||||
name: integration-approve
|
|
||||||
|
|
||||||
on:
|
|
||||||
merge_group:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
# Trigger for merge groups.
|
|
||||||
#
|
|
||||||
# Statuses and checks apply to specific commits (by hash).
|
|
||||||
# Enforcement of required checks is done both at the PR level and the merge queue level.
|
|
||||||
# In case of multiple commits in a single PR, the hash of the squashed commit
|
|
||||||
# will not match the one for the latest (approved) commit in the PR.
|
|
||||||
#
|
|
||||||
# We auto approve the check for the merge queue for two reasons:
|
|
||||||
#
|
|
||||||
# * Queue times out due to duration of tests.
|
|
||||||
# * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
|
|
||||||
#
|
|
||||||
trigger:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Auto-approve squashed commit
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
gh api -X POST -H "Accept: application/vnd.github+json" \
|
|
||||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
||||||
/repos/${{ github.repository }}/statuses/${{ github.sha }} \
|
|
||||||
-f 'state=success' \
|
|
||||||
-f 'context=Integration Tests Check'
|
|
|
@ -1,33 +0,0 @@
|
||||||
name: integration-main
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
# Trigger for pushes to the main branch.
|
|
||||||
#
|
|
||||||
# This workflow triggers the integration test workflow in a different repository.
|
|
||||||
# It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
|
|
||||||
trigger:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
environment: "test-trigger-is"
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Generate GitHub App Token
|
|
||||||
id: generate-token
|
|
||||||
uses: actions/create-github-app-token@v1
|
|
||||||
with:
|
|
||||||
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
|
|
||||||
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
|
|
||||||
owner: ${{ secrets.ORG_NAME }}
|
|
||||||
repositories: ${{secrets.REPO_NAME}}
|
|
||||||
|
|
||||||
- name: Trigger Workflow in Another Repo
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
|
||||||
run: |
|
|
||||||
gh workflow run cli-isolated-nightly.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
|
|
||||||
--ref main \
|
|
||||||
-f commit_sha=${{ github.event.after }}
|
|
|
@ -1,56 +0,0 @@
|
||||||
name: integration-pr
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
types: [opened, synchronize]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check-token:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
environment: "test-trigger-is"
|
|
||||||
|
|
||||||
outputs:
|
|
||||||
has_token: ${{ steps.set-token-status.outputs.has_token }}
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set
|
|
||||||
id: set-token-status
|
|
||||||
run: |
|
|
||||||
if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then
|
|
||||||
echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets."
|
|
||||||
echo "::set-output name=has_token::false"
|
|
||||||
else
|
|
||||||
echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
|
|
||||||
echo "::set-output name=has_token::true"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Trigger for pull requests.
|
|
||||||
#
|
|
||||||
# This workflow triggers the integration test workflow in a different repository.
|
|
||||||
# It requires secrets from the "test-trigger-is" environment, which are only available to authorized users.
|
|
||||||
# It depends on the "check-token" workflow to confirm access to this environment to avoid failures.
|
|
||||||
trigger:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
environment: "test-trigger-is"
|
|
||||||
|
|
||||||
if: needs.check-token.outputs.has_token == 'true'
|
|
||||||
needs: check-token
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Generate GitHub App Token
|
|
||||||
id: generate-token
|
|
||||||
uses: actions/create-github-app-token@v1
|
|
||||||
with:
|
|
||||||
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
|
|
||||||
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
|
|
||||||
owner: ${{ secrets.ORG_NAME }}
|
|
||||||
repositories: ${{secrets.REPO_NAME}}
|
|
||||||
|
|
||||||
- name: Trigger Workflow in Another Repo
|
|
||||||
env:
|
|
||||||
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
|
||||||
run: |
|
|
||||||
gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
|
|
||||||
--ref main \
|
|
||||||
-f pull_request_number=${{ github.event.pull_request.number }} \
|
|
||||||
-f commit_sha=${{ github.event.pull_request.head.sha }}
|
|
|
@ -0,0 +1,78 @@
|
||||||
|
name: integration
|
||||||
|
|
||||||
|
on:
|
||||||
|
|
||||||
|
pull_request:
|
||||||
|
types: [opened, synchronize]
|
||||||
|
|
||||||
|
merge_group:
|
||||||
|
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-token:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: "test-trigger-is"
|
||||||
|
outputs:
|
||||||
|
has_token: ${{ steps.set-token-status.outputs.has_token }}
|
||||||
|
steps:
|
||||||
|
- name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set
|
||||||
|
id: set-token-status
|
||||||
|
run: |
|
||||||
|
if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then
|
||||||
|
echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets."
|
||||||
|
echo "::set-output name=has_token::false"
|
||||||
|
else
|
||||||
|
echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets."
|
||||||
|
echo "::set-output name=has_token::true"
|
||||||
|
fi
|
||||||
|
|
||||||
|
trigger-tests:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: check-token
|
||||||
|
if: github.event_name == 'pull_request' && needs.check-token.outputs.has_token == 'true'
|
||||||
|
environment: "test-trigger-is"
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Generate GitHub App Token
|
||||||
|
id: generate-token
|
||||||
|
uses: actions/create-github-app-token@v1
|
||||||
|
with:
|
||||||
|
app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
|
||||||
|
private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
|
||||||
|
owner: ${{ secrets.ORG_NAME }}
|
||||||
|
repositories: ${{secrets.REPO_NAME}}
|
||||||
|
|
||||||
|
- name: Trigger Workflow in Another Repo
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||||
|
run: |
|
||||||
|
gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \
|
||||||
|
--ref main \
|
||||||
|
-f pull_request_number=${{ github.event.pull_request.number }} \
|
||||||
|
-f commit_sha=${{ github.event.pull_request.head.sha }}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Statuses and checks apply to specific commits (by hash).
|
||||||
|
# Enforcement of required checks is done both at the PR level and the merge queue level.
|
||||||
|
# In case of multiple commits in a single PR, the hash of the squashed commit
|
||||||
|
# will not match the one for the latest (approved) commit in the PR.
|
||||||
|
# We auto approve the check for the merge queue for two reasons:
|
||||||
|
# * Queue times out due to duration of tests.
|
||||||
|
# * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing.
|
||||||
|
auto-approve:
|
||||||
|
if: github.event_name == 'merge_group'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Mark Check
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
gh api -X POST -H "Accept: application/vnd.github+json" \
|
||||||
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||||
|
/repos/${{ github.repository }}/statuses/${{ github.sha }} \
|
||||||
|
-f 'state=success' \
|
||||||
|
-f 'context=Integration Tests Check'
|
|
@ -33,21 +33,19 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: 1.23.4
|
go-version: 1.23.2
|
||||||
|
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.9'
|
python-version: '3.9'
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
uses: astral-sh/setup-uv@v4
|
|
||||||
|
|
||||||
- name: Set go env
|
- name: Set go env
|
||||||
run: |
|
run: |
|
||||||
echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
|
echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
|
||||||
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
|
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
|
||||||
go install gotest.tools/gotestsum@v1.12.0
|
go install gotest.tools/gotestsum@latest
|
||||||
|
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||||
|
|
||||||
- name: Pull external libraries
|
- name: Pull external libraries
|
||||||
run: |
|
run: |
|
||||||
|
@ -55,28 +53,42 @@ jobs:
|
||||||
pip3 install wheel
|
pip3 install wheel
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: make testonly
|
run: make test
|
||||||
|
|
||||||
golangci:
|
- name: Publish test coverage
|
||||||
name: lint
|
uses: codecov/codecov-action@v4
|
||||||
|
|
||||||
|
fmt:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- name: Checkout
|
||||||
- uses: actions/setup-go@v5
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: 1.23.4
|
go-version: 1.23.2
|
||||||
|
|
||||||
|
# No need to download cached dependencies when running gofmt.
|
||||||
|
cache: false
|
||||||
|
|
||||||
|
- name: Install goimports
|
||||||
|
run: |
|
||||||
|
go install golang.org/x/tools/cmd/goimports@latest
|
||||||
|
|
||||||
|
- name: Run make fmt
|
||||||
|
run: |
|
||||||
|
make fmt
|
||||||
|
|
||||||
- name: Run go mod tidy
|
- name: Run go mod tidy
|
||||||
run: |
|
run: |
|
||||||
go mod tidy
|
go mod tidy
|
||||||
|
|
||||||
- name: Fail on differences
|
- name: Fail on differences
|
||||||
run: |
|
run: |
|
||||||
# Exit with status code 1 if there are differences (i.e. unformatted files)
|
# Exit with status code 1 if there are differences (i.e. unformatted files)
|
||||||
git diff --exit-code
|
git diff --exit-code
|
||||||
- name: golangci-lint
|
|
||||||
uses: golangci/golangci-lint-action@v6
|
|
||||||
with:
|
|
||||||
version: v1.62.2
|
|
||||||
args: --timeout=15m
|
|
||||||
|
|
||||||
validate-bundle-schema:
|
validate-bundle-schema:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
@ -88,7 +100,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: 1.23.4
|
go-version: 1.23.2
|
||||||
|
|
||||||
# Github repo: https://github.com/ajv-validator/ajv-cli
|
# Github repo: https://github.com/ajv-validator/ajv-cli
|
||||||
- name: Install ajv-cli
|
- name: Install ajv-cli
|
||||||
|
@ -99,19 +111,14 @@ jobs:
|
||||||
# By default the ajv-cli runs in strict mode which will fail if the schema
|
# By default the ajv-cli runs in strict mode which will fail if the schema
|
||||||
# itself is not valid. Strict mode is more strict than the JSON schema
|
# itself is not valid. Strict mode is more strict than the JSON schema
|
||||||
# specification. See for details: https://ajv.js.org/options.html#strict-mode-options
|
# specification. See for details: https://ajv.js.org/options.html#strict-mode-options
|
||||||
# The ajv-cli is configured to use the markdownDescription keyword which is not part of the JSON schema specification,
|
|
||||||
# but is used in editors like VSCode to render markdown in the description field
|
|
||||||
- name: Validate bundle schema
|
- name: Validate bundle schema
|
||||||
run: |
|
run: |
|
||||||
go run main.go bundle schema > schema.json
|
go run main.go bundle schema > schema.json
|
||||||
|
|
||||||
# Add markdownDescription keyword to ajv
|
|
||||||
echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js
|
|
||||||
|
|
||||||
for file in ./bundle/internal/schema/testdata/pass/*.yml; do
|
for file in ./bundle/internal/schema/testdata/pass/*.yml; do
|
||||||
ajv test -s schema.json -d $file --valid -c=./keywords.js
|
ajv test -s schema.json -d $file --valid
|
||||||
done
|
done
|
||||||
|
|
||||||
for file in ./bundle/internal/schema/testdata/fail/*.yml; do
|
for file in ./bundle/internal/schema/testdata/fail/*.yml; do
|
||||||
ajv test -s schema.json -d $file --invalid -c=./keywords.js
|
ajv test -s schema.json -d $file --invalid
|
||||||
done
|
done
|
||||||
|
|
|
@ -5,7 +5,6 @@ on:
|
||||||
branches:
|
branches:
|
||||||
- "main"
|
- "main"
|
||||||
- "demo-*"
|
- "demo-*"
|
||||||
- "bugbash-*"
|
|
||||||
|
|
||||||
# Confirm that snapshot builds work if this file is modified.
|
# Confirm that snapshot builds work if this file is modified.
|
||||||
pull_request:
|
pull_request:
|
||||||
|
@ -31,7 +30,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: 1.23.4
|
go-version: 1.23.2
|
||||||
|
|
||||||
# The default cache key for this action considers only the `go.sum` file.
|
# The default cache key for this action considers only the `go.sum` file.
|
||||||
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
||||||
|
|
|
@ -22,7 +22,7 @@ jobs:
|
||||||
- name: Setup Go
|
- name: Setup Go
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
with:
|
with:
|
||||||
go-version: 1.23.4
|
go-version: 1.23.2
|
||||||
|
|
||||||
# The default cache key for this action considers only the `go.sum` file.
|
# The default cache key for this action considers only the `go.sum` file.
|
||||||
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
||||||
|
|
|
@ -1,38 +0,0 @@
|
||||||
linters:
|
|
||||||
disable-all: true
|
|
||||||
enable:
|
|
||||||
- bodyclose
|
|
||||||
- errcheck
|
|
||||||
- gosimple
|
|
||||||
- govet
|
|
||||||
- ineffassign
|
|
||||||
- staticcheck
|
|
||||||
- unused
|
|
||||||
- gofmt
|
|
||||||
- gofumpt
|
|
||||||
- goimports
|
|
||||||
linters-settings:
|
|
||||||
govet:
|
|
||||||
enable-all: true
|
|
||||||
disable:
|
|
||||||
- fieldalignment
|
|
||||||
- shadow
|
|
||||||
gofmt:
|
|
||||||
rewrite-rules:
|
|
||||||
- pattern: 'a[b:len(a)]'
|
|
||||||
replacement: 'a[b:]'
|
|
||||||
- pattern: 'interface{}'
|
|
||||||
replacement: 'any'
|
|
||||||
errcheck:
|
|
||||||
exclude-functions:
|
|
||||||
- (*github.com/spf13/cobra.Command).RegisterFlagCompletionFunc
|
|
||||||
- (*github.com/spf13/cobra.Command).MarkFlagRequired
|
|
||||||
- (*github.com/spf13/pflag.FlagSet).MarkDeprecated
|
|
||||||
- (*github.com/spf13/pflag.FlagSet).MarkHidden
|
|
||||||
gofumpt:
|
|
||||||
module-path: github.com/databricks/cli
|
|
||||||
extra-rules: true
|
|
||||||
#goimports:
|
|
||||||
# local-prefixes: github.com/databricks/cli
|
|
||||||
issues:
|
|
||||||
exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/
|
|
|
@ -3,18 +3,11 @@
|
||||||
"editor.insertSpaces": false,
|
"editor.insertSpaces": false,
|
||||||
"editor.formatOnSave": true
|
"editor.formatOnSave": true
|
||||||
},
|
},
|
||||||
"go.lintTool": "golangci-lint",
|
|
||||||
"go.lintFlags": [
|
|
||||||
"--fast"
|
|
||||||
],
|
|
||||||
"go.useLanguageServer": true,
|
|
||||||
"gopls": {
|
|
||||||
"formatting.gofumpt": true
|
|
||||||
},
|
|
||||||
"files.trimTrailingWhitespace": true,
|
"files.trimTrailingWhitespace": true,
|
||||||
"files.insertFinalNewline": true,
|
"files.insertFinalNewline": true,
|
||||||
"files.trimFinalNewlines": true,
|
"files.trimFinalNewlines": true,
|
||||||
"python.envFile": "${workspaceRoot}/.env",
|
"python.envFile": "${workspaceRoot}/.env",
|
||||||
|
"databricks.python.envFile": "${workspaceFolder}/.env",
|
||||||
"python.analysis.stubPath": ".vscode",
|
"python.analysis.stubPath": ".vscode",
|
||||||
"jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
|
"jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
|
||||||
"jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------"
|
"jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------"
|
||||||
|
|
73
CHANGELOG.md
73
CHANGELOG.md
|
@ -1,78 +1,5 @@
|
||||||
# Version changelog
|
# Version changelog
|
||||||
|
|
||||||
## [Release] Release v0.237.0
|
|
||||||
|
|
||||||
Bundles:
|
|
||||||
* Allow overriding compute for non-development mode targets ([#1899](https://github.com/databricks/cli/pull/1899)).
|
|
||||||
* Show an error when using a cluster override with 'mode: production' ([#1994](https://github.com/databricks/cli/pull/1994)).
|
|
||||||
|
|
||||||
API Changes:
|
|
||||||
* Added `databricks account federation-policy` command group.
|
|
||||||
* Added `databricks account service-principal-federation-policy` command group.
|
|
||||||
* Added `databricks aibi-dashboard-embedding-access-policy delete` command.
|
|
||||||
* Added `databricks aibi-dashboard-embedding-approved-domains delete` command.
|
|
||||||
|
|
||||||
OpenAPI commit a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d (2024-12-16)
|
|
||||||
Dependency updates:
|
|
||||||
* Upgrade TF provider to 1.62.0 ([#2030](https://github.com/databricks/cli/pull/2030)).
|
|
||||||
* Upgrade Go SDK to 0.54.0 ([#2029](https://github.com/databricks/cli/pull/2029)).
|
|
||||||
* Bump TF codegen dependencies to latest ([#1961](https://github.com/databricks/cli/pull/1961)).
|
|
||||||
* Bump golang.org/x/term from 0.26.0 to 0.27.0 ([#1983](https://github.com/databricks/cli/pull/1983)).
|
|
||||||
* Bump golang.org/x/sync from 0.9.0 to 0.10.0 ([#1984](https://github.com/databricks/cli/pull/1984)).
|
|
||||||
* Bump github.com/databricks/databricks-sdk-go from 0.52.0 to 0.53.0 ([#1985](https://github.com/databricks/cli/pull/1985)).
|
|
||||||
* Bump golang.org/x/crypto from 0.24.0 to 0.31.0 ([#2006](https://github.com/databricks/cli/pull/2006)).
|
|
||||||
* Bump golang.org/x/crypto from 0.30.0 to 0.31.0 in /bundle/internal/tf/codegen ([#2005](https://github.com/databricks/cli/pull/2005)).
|
|
||||||
|
|
||||||
## [Release] Release v0.236.0
|
|
||||||
|
|
||||||
**New features for Databricks Asset Bundles:**
|
|
||||||
|
|
||||||
This release adds support for managing Unity Catalog volumes as part of your bundle configuration.
|
|
||||||
|
|
||||||
Bundles:
|
|
||||||
* Add DABs support for Unity Catalog volumes ([#1762](https://github.com/databricks/cli/pull/1762)).
|
|
||||||
* Support lookup by name of notification destinations ([#1922](https://github.com/databricks/cli/pull/1922)).
|
|
||||||
* Extend "notebook not found" error to warn about missing extension ([#1920](https://github.com/databricks/cli/pull/1920)).
|
|
||||||
* Skip sync warning if no sync paths are defined ([#1926](https://github.com/databricks/cli/pull/1926)).
|
|
||||||
* Add validation for single node clusters ([#1909](https://github.com/databricks/cli/pull/1909)).
|
|
||||||
* Fix segfault in bundle summary command ([#1937](https://github.com/databricks/cli/pull/1937)).
|
|
||||||
* Add the `bundle_uuid` helper function for templates ([#1947](https://github.com/databricks/cli/pull/1947)).
|
|
||||||
* Add default value for `volume_type` for DABs ([#1952](https://github.com/databricks/cli/pull/1952)).
|
|
||||||
* Properly read Git metadata when running inside workspace ([#1945](https://github.com/databricks/cli/pull/1945)).
|
|
||||||
* Upgrade TF provider to 1.59.0 ([#1960](https://github.com/databricks/cli/pull/1960)).
|
|
||||||
|
|
||||||
Internal:
|
|
||||||
* Breakout variable lookup into separate files and tests ([#1921](https://github.com/databricks/cli/pull/1921)).
|
|
||||||
* Add golangci-lint v1.62.2 ([#1953](https://github.com/databricks/cli/pull/1953)).
|
|
||||||
|
|
||||||
Dependency updates:
|
|
||||||
* Bump golang.org/x/term from 0.25.0 to 0.26.0 ([#1907](https://github.com/databricks/cli/pull/1907)).
|
|
||||||
* Bump github.com/Masterminds/semver/v3 from 3.3.0 to 3.3.1 ([#1930](https://github.com/databricks/cli/pull/1930)).
|
|
||||||
* Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 ([#1932](https://github.com/databricks/cli/pull/1932)).
|
|
||||||
* Bump github.com/databricks/databricks-sdk-go from 0.51.0 to 0.52.0 ([#1931](https://github.com/databricks/cli/pull/1931)).
|
|
||||||
## [Release] Release v0.235.0
|
|
||||||
|
|
||||||
**Note:** the `bundle generate` command now uses the `.<resource-type>.yml`
|
|
||||||
sub-extension for the configuration files it writes. Existing configuration
|
|
||||||
files that do not use this sub-extension are renamed to include it.
|
|
||||||
|
|
||||||
Bundles:
|
|
||||||
* Make `TableName` field part of quality monitor schema ([#1903](https://github.com/databricks/cli/pull/1903)).
|
|
||||||
* Do not prepend paths starting with ~ or variable reference ([#1905](https://github.com/databricks/cli/pull/1905)).
|
|
||||||
* Fix workspace extensions filer accidentally reading notebooks ([#1891](https://github.com/databricks/cli/pull/1891)).
|
|
||||||
* Fix template initialization when running on Databricks ([#1912](https://github.com/databricks/cli/pull/1912)).
|
|
||||||
* Source-linked deployments for bundles in the workspace ([#1884](https://github.com/databricks/cli/pull/1884)).
|
|
||||||
* Added integration test to deploy bundle to /Shared root path ([#1914](https://github.com/databricks/cli/pull/1914)).
|
|
||||||
* Update filenames used by bundle generate to use `.<resource-type>.yml` ([#1901](https://github.com/databricks/cli/pull/1901)).
|
|
||||||
|
|
||||||
Internal:
|
|
||||||
* Extract functionality to detect if the CLI is running on DBR ([#1889](https://github.com/databricks/cli/pull/1889)).
|
|
||||||
* Consolidate test helpers for `io/fs` ([#1906](https://github.com/databricks/cli/pull/1906)).
|
|
||||||
* Use `fs.FS` interface to read template ([#1910](https://github.com/databricks/cli/pull/1910)).
|
|
||||||
* Use `filer.Filer` to write template instantiation ([#1911](https://github.com/databricks/cli/pull/1911)).
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## [Release] Release v0.234.0
|
## [Release] Release v0.234.0
|
||||||
|
|
||||||
Bundles:
|
Bundles:
|
||||||
|
|
31
Makefile
31
Makefile
|
@ -1,16 +1,16 @@
|
||||||
default: build
|
default: build
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
@echo "✓ Formatting source code with goimports ..."
|
||||||
|
@goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
|
||||||
|
@echo "✓ Formatting source code with gofmt ..."
|
||||||
|
@gofmt -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
|
||||||
|
|
||||||
lint: vendor
|
lint: vendor
|
||||||
@echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..."
|
@echo "✓ Linting source code with https://staticcheck.io/ ..."
|
||||||
@./lint.sh ./...
|
@staticcheck ./...
|
||||||
|
|
||||||
lintcheck: vendor
|
test: lint
|
||||||
@echo "✓ Linting source code with https://golangci-lint.run/ ..."
|
|
||||||
@golangci-lint run ./...
|
|
||||||
|
|
||||||
test: lint testonly
|
|
||||||
|
|
||||||
testonly:
|
|
||||||
@echo "✓ Running tests ..."
|
@echo "✓ Running tests ..."
|
||||||
@gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./...
|
@gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./...
|
||||||
|
|
||||||
|
@ -29,17 +29,6 @@ snapshot:
|
||||||
vendor:
|
vendor:
|
||||||
@echo "✓ Filling vendor folder with library code ..."
|
@echo "✓ Filling vendor folder with library code ..."
|
||||||
@go mod vendor
|
@go mod vendor
|
||||||
|
|
||||||
schema:
|
|
||||||
@echo "✓ Generating json-schema ..."
|
|
||||||
@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json
|
|
||||||
|
|
||||||
INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
|
.PHONY: build vendor coverage test lint fmt
|
||||||
|
|
||||||
integration:
|
|
||||||
$(INTEGRATION)
|
|
||||||
|
|
||||||
integration-short:
|
|
||||||
$(INTEGRATION) -short
|
|
||||||
|
|
||||||
.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short
|
|
||||||
|
|
12
NOTICE
12
NOTICE
|
@ -73,6 +73,10 @@ fatih/color - https://github.com/fatih/color
|
||||||
Copyright (c) 2013 Fatih Arslan
|
Copyright (c) 2013 Fatih Arslan
|
||||||
License - https://github.com/fatih/color/blob/main/LICENSE.md
|
License - https://github.com/fatih/color/blob/main/LICENSE.md
|
||||||
|
|
||||||
|
ghodss/yaml - https://github.com/ghodss/yaml
|
||||||
|
Copyright (c) 2014 Sam Ghods
|
||||||
|
License - https://github.com/ghodss/yaml/blob/master/LICENSE
|
||||||
|
|
||||||
Masterminds/semver - https://github.com/Masterminds/semver
|
Masterminds/semver - https://github.com/Masterminds/semver
|
||||||
Copyright (C) 2014-2019, Matt Butcher and Matt Farina
|
Copyright (C) 2014-2019, Matt Butcher and Matt Farina
|
||||||
License - https://github.com/Masterminds/semver/blob/master/LICENSE.txt
|
License - https://github.com/Masterminds/semver/blob/master/LICENSE.txt
|
||||||
|
@ -97,11 +101,3 @@ License - https://github.com/stretchr/testify/blob/master/LICENSE
|
||||||
whilp/git-urls - https://github.com/whilp/git-urls
|
whilp/git-urls - https://github.com/whilp/git-urls
|
||||||
Copyright (c) 2020 Will Maier
|
Copyright (c) 2020 Will Maier
|
||||||
License - https://github.com/whilp/git-urls/blob/master/LICENSE
|
License - https://github.com/whilp/git-urls/blob/master/LICENSE
|
||||||
|
|
||||||
github.com/wI2L/jsondiff v0.6.1
|
|
||||||
Copyright (c) 2020-2024 William Poussier <william.poussier@gmail.com>
|
|
||||||
License - https://github.com/wI2L/jsondiff/blob/master/LICENSE
|
|
||||||
|
|
||||||
https://github.com/hexops/gotextdiff
|
|
||||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
|
||||||
License - https://github.com/hexops/gotextdiff/blob/main/LICENSE
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ package artifacts
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
|
|
|
@ -13,7 +13,8 @@ func DetectPackages() bundle.Mutator {
|
||||||
return &autodetect{}
|
return &autodetect{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type autodetect struct{}
|
type autodetect struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (m *autodetect) Name() string {
|
func (m *autodetect) Name() string {
|
||||||
return "artifacts.DetectPackages"
|
return "artifacts.DetectPackages"
|
||||||
|
|
|
@ -96,6 +96,7 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost
|
||||||
// Set the expanded globs back into the configuration.
|
// Set the expanded globs back into the configuration.
|
||||||
return dyn.SetByPath(v, base, dyn.V(output))
|
return dyn.SetByPath(v, base, dyn.V(output))
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diag.FromErr(err)
|
return diag.FromErr(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,13 +21,18 @@ func (m *cleanUp) Name() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
client, uploadPath, diags := libraries.GetFilerForLibraries(ctx, b)
|
uploadPath, err := libraries.GetUploadBasePath(b)
|
||||||
if diags.HasError() {
|
if err != nil {
|
||||||
return diags
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := libraries.GetFilerForLibraries(b.WorkspaceClient(), uploadPath)
|
||||||
|
if err != nil {
|
||||||
|
return diag.FromErr(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// We intentionally ignore the error because it is not critical to the deployment
|
// We intentionally ignore the error because it is not critical to the deployment
|
||||||
err := client.Delete(ctx, ".", filer.DeleteRecursively)
|
err = client.Delete(ctx, ".", filer.DeleteRecursively)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err)
|
log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,7 +15,8 @@ import (
|
||||||
"github.com/databricks/cli/libs/log"
|
"github.com/databricks/cli/libs/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type detectPkg struct{}
|
type detectPkg struct {
|
||||||
|
}
|
||||||
|
|
||||||
func DetectPackage() bundle.Mutator {
|
func DetectPackage() bundle.Mutator {
|
||||||
return &detectPkg{}
|
return &detectPkg{}
|
||||||
|
@ -41,7 +42,7 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof(ctx, "Found Python wheel project at %s", b.BundleRootPath)
|
log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath))
|
||||||
module := extractModuleName(setupPy)
|
module := extractModuleName(setupPy)
|
||||||
|
|
||||||
if b.Config.Artifacts == nil {
|
if b.Config.Artifacts == nil {
|
||||||
|
|
|
@ -16,6 +16,12 @@ type infer struct {
|
||||||
func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
artifact := b.Config.Artifacts[m.name]
|
artifact := b.Config.Artifacts[m.name]
|
||||||
|
|
||||||
|
// TODO use python.DetectVEnvExecutable once bundle has a way to specify venv path
|
||||||
|
py, err := python.DetectExecutable(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
// Note: using --build-number (build tag) flag does not help with re-installing
|
// Note: using --build-number (build tag) flag does not help with re-installing
|
||||||
// libraries on all-purpose clusters. The reason is that `pip` ignoring build tag
|
// libraries on all-purpose clusters. The reason is that `pip` ignoring build tag
|
||||||
// when upgrading the library and only look at wheel version.
|
// when upgrading the library and only look at wheel version.
|
||||||
|
@ -30,9 +36,7 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
// version=datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"),
|
// version=datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"),
|
||||||
// ...
|
// ...
|
||||||
//)
|
//)
|
||||||
|
artifact.BuildCommand = fmt.Sprintf(`"%s" setup.py bdist_wheel`, py)
|
||||||
py := python.GetExecutable()
|
|
||||||
artifact.BuildCommand = fmt.Sprintf(`%s setup.py bdist_wheel`, py)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,6 +17,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle/env"
|
"github.com/databricks/cli/bundle/env"
|
||||||
"github.com/databricks/cli/bundle/metadata"
|
"github.com/databricks/cli/bundle/metadata"
|
||||||
"github.com/databricks/cli/libs/fileset"
|
"github.com/databricks/cli/libs/fileset"
|
||||||
|
"github.com/databricks/cli/libs/git"
|
||||||
"github.com/databricks/cli/libs/locker"
|
"github.com/databricks/cli/libs/locker"
|
||||||
"github.com/databricks/cli/libs/log"
|
"github.com/databricks/cli/libs/log"
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
|
@ -48,10 +49,6 @@ type Bundle struct {
|
||||||
// Exclusively use this field for filesystem operations.
|
// Exclusively use this field for filesystem operations.
|
||||||
SyncRoot vfs.Path
|
SyncRoot vfs.Path
|
||||||
|
|
||||||
// Path to the root of git worktree containing the bundle.
|
|
||||||
// https://git-scm.com/docs/git-worktree
|
|
||||||
WorktreeRoot vfs.Path
|
|
||||||
|
|
||||||
// Config contains the bundle configuration.
|
// Config contains the bundle configuration.
|
||||||
// It is loaded from the bundle configuration files and mutators may update it.
|
// It is loaded from the bundle configuration files and mutators may update it.
|
||||||
Config config.Root
|
Config config.Root
|
||||||
|
@ -186,7 +183,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
|
||||||
|
|
||||||
// Make directory if it doesn't exist yet.
|
// Make directory if it doesn't exist yet.
|
||||||
dir := filepath.Join(parts...)
|
dir := filepath.Join(parts...)
|
||||||
err := os.MkdirAll(dir, 0o700)
|
err := os.MkdirAll(dir, 0700)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
@ -203,7 +200,7 @@ func (b *Bundle) InternalDir(ctx context.Context) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
dir := filepath.Join(cacheDir, internalFolder)
|
dir := filepath.Join(cacheDir, internalFolder)
|
||||||
err = os.MkdirAll(dir, 0o700)
|
err = os.MkdirAll(dir, 0700)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dir, err
|
return dir, err
|
||||||
}
|
}
|
||||||
|
@ -226,6 +223,15 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
|
||||||
return append(b.Config.Sync.Include, filepath.ToSlash(filepath.Join(internalDirRel, "*.*"))), nil
|
return append(b.Config.Sync.Include, filepath.ToSlash(filepath.Join(internalDirRel, "*.*"))), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *Bundle) GitRepository() (*git.Repository, error) {
|
||||||
|
_, err := vfs.FindLeafInTree(b.BundleRoot, ".git")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unable to locate repository root: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return git.NewRepository(b.BundleRoot)
|
||||||
|
}
|
||||||
|
|
||||||
// AuthEnv returns a map with environment variables and their values
|
// AuthEnv returns a map with environment variables and their values
|
||||||
// derived from the workspace client configuration that was resolved
|
// derived from the workspace client configuration that was resolved
|
||||||
// in the context of this bundle.
|
// in the context of this bundle.
|
||||||
|
|
|
@ -32,10 +32,6 @@ func (r ReadOnlyBundle) SyncRoot() vfs.Path {
|
||||||
return r.b.SyncRoot
|
return r.b.SyncRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r ReadOnlyBundle) WorktreeRoot() vfs.Path {
|
|
||||||
return r.b.WorktreeRoot
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
|
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
|
||||||
return r.b.WorkspaceClient()
|
return r.b.WorkspaceClient()
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,8 +49,4 @@ type Bundle struct {
|
||||||
|
|
||||||
// Databricks CLI version constraints required to run the bundle.
|
// Databricks CLI version constraints required to run the bundle.
|
||||||
DatabricksCliVersion string `json:"databricks_cli_version,omitempty"`
|
DatabricksCliVersion string `json:"databricks_cli_version,omitempty"`
|
||||||
|
|
||||||
// A stable generated UUID for the bundle. This is normally serialized by
|
|
||||||
// Databricks first party template when a user runs bundle init.
|
|
||||||
Uuid string `json:"uuid,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -47,10 +47,8 @@ type PyDABs struct {
|
||||||
Import []string `json:"import,omitempty"`
|
Import []string `json:"import,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type (
|
type Command string
|
||||||
Command string
|
type ScriptHook string
|
||||||
ScriptHook string
|
|
||||||
)
|
|
||||||
|
|
||||||
// These hook names are subject to change and currently experimental
|
// These hook names are subject to change and currently experimental
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -6,10 +6,8 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"})
|
||||||
jobOrder = yamlsaver.NewOrder([]string{"name", "job_clusters", "compute", "tasks"})
|
var taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"})
|
||||||
taskOrder = yamlsaver.NewOrder([]string{"task_key", "depends_on", "existing_cluster_id", "new_cluster", "job_cluster_key"})
|
|
||||||
)
|
|
||||||
|
|
||||||
func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) {
|
func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) {
|
||||||
value := make(map[string]dyn.Value)
|
value := make(map[string]dyn.Value)
|
||||||
|
|
|
@ -27,7 +27,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
||||||
var out []bundle.Mutator
|
var out []bundle.Mutator
|
||||||
|
|
||||||
// Map with files we've already seen to avoid loading them twice.
|
// Map with files we've already seen to avoid loading them twice.
|
||||||
seen := map[string]bool{}
|
var seen = map[string]bool{}
|
||||||
|
|
||||||
for _, file := range config.FileNames {
|
for _, file := range config.FileNames {
|
||||||
seen[file] = true
|
seen[file] = true
|
||||||
|
|
|
@ -9,7 +9,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/textutil"
|
"github.com/databricks/cli/libs/textutil"
|
||||||
|
@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
dashboard.DisplayName = prefix + dashboard.DisplayName
|
dashboard.DisplayName = prefix + dashboard.DisplayName
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
|
|
||||||
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if !isDatabricksWorkspace {
|
|
||||||
target := b.Config.Bundle.Target
|
|
||||||
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
|
||||||
diags = diags.Append(
|
|
||||||
diag.Diagnostic{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
Paths: []dyn.Path{
|
|
||||||
path,
|
|
||||||
},
|
|
||||||
Locations: b.Config.GetLocations(path[2:].String()),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
disabled := false
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &disabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,16 +2,12 @@ package mutator_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -73,7 +69,7 @@ func TestApplyPresetsPrefix(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyPresetsPrefixForSchema(t *testing.T) {
|
func TestApplyPresetsPrefixForUcSchema(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
prefix string
|
prefix string
|
||||||
|
@ -129,36 +125,6 @@ func TestApplyPresetsPrefixForSchema(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyPresetsVolumesShouldNotBePrefixed(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Volumes: map[string]*resources.Volume{
|
|
||||||
"volume1": {
|
|
||||||
CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
|
|
||||||
Name: "volume1",
|
|
||||||
CatalogName: "catalog1",
|
|
||||||
SchemaName: "schema1",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Presets: config.Presets{
|
|
||||||
NamePrefix: "[prefix]",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diag := bundle.Apply(ctx, b, mutator.ApplyPresets())
|
|
||||||
|
|
||||||
if diag.HasError() {
|
|
||||||
t.Fatalf("unexpected error: %v", diag)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplyPresetsTags(t *testing.T) {
|
func TestApplyPresetsTags(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
@ -398,87 +364,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
testContext := context.Background()
|
|
||||||
enabled := true
|
|
||||||
disabled := false
|
|
||||||
workspacePath := "/Workspace/user.name@company.com"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
bundlePath string
|
|
||||||
ctx context.Context
|
|
||||||
name string
|
|
||||||
initialValue *bool
|
|
||||||
expectedValue *bool
|
|
||||||
expectedWarning string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &enabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle not in Workspace, databricks runtime",
|
|
||||||
bundlePath: "/Users/user.name@company.com",
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset enabled, bundle in Workspace, not databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, false),
|
|
||||||
initialValue: &enabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset disabled, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: &disabled,
|
|
||||||
expectedValue: &disabled,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "preset nil, bundle in Workspace, databricks runtime",
|
|
||||||
bundlePath: workspacePath,
|
|
||||||
ctx: dbr.MockRuntime(testContext, true),
|
|
||||||
initialValue: nil,
|
|
||||||
expectedValue: nil,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRootPath: tt.bundlePath,
|
|
||||||
Config: config.Root{
|
|
||||||
Presets: config.Presets{
|
|
||||||
SourceLinkedDeployment: tt.initialValue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
|
|
||||||
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
|
|
||||||
if diags.HasError() {
|
|
||||||
t.Fatalf("unexpected error: %v", diags)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectedWarning != "" {
|
|
||||||
require.Equal(t, tt.expectedWarning, diags[0].Summary)
|
|
||||||
require.NotEmpty(t, diags[0].Locations)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
@ -42,6 +42,7 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn
|
||||||
var diags diag.Diagnostics
|
var diags diag.Diagnostics
|
||||||
computeIdPath := p.Append(dyn.Key("compute_id"))
|
computeIdPath := p.Append(dyn.Key("compute_id"))
|
||||||
computeId, err := dyn.GetByPath(v, computeIdPath)
|
computeId, err := dyn.GetByPath(v, computeIdPath)
|
||||||
|
|
||||||
// If the "compute_id" key is not set, we don't need to do anything.
|
// If the "compute_id" key is not set, we don't need to do anything.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v, nil
|
return v, nil
|
||||||
|
|
|
@ -1,44 +0,0 @@
|
||||||
package mutator
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
)
|
|
||||||
|
|
||||||
type configureVolumeDefaults struct{}
|
|
||||||
|
|
||||||
func ConfigureVolumeDefaults() bundle.Mutator {
|
|
||||||
return &configureVolumeDefaults{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *configureVolumeDefaults) Name() string {
|
|
||||||
return "ConfigureVolumeDefaults"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *configureVolumeDefaults) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
|
||||||
var diags diag.Diagnostics
|
|
||||||
|
|
||||||
pattern := dyn.NewPattern(
|
|
||||||
dyn.Key("resources"),
|
|
||||||
dyn.Key("volumes"),
|
|
||||||
dyn.AnyKey(),
|
|
||||||
)
|
|
||||||
|
|
||||||
// Configure defaults for all volumes.
|
|
||||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
|
||||||
var err error
|
|
||||||
v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("volume_type")), dyn.V("MANAGED"))
|
|
||||||
if err != nil {
|
|
||||||
return dyn.InvalidValue, err
|
|
||||||
}
|
|
||||||
return v, nil
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
diags = diags.Extend(diag.FromErr(err))
|
|
||||||
return diags
|
|
||||||
}
|
|
|
@ -1,75 +0,0 @@
|
||||||
package mutator_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestConfigureVolumeDefaultsVolumeType(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Volumes: map[string]*resources.Volume{
|
|
||||||
"v1": {
|
|
||||||
// Empty string is skipped.
|
|
||||||
// See below for how it is set.
|
|
||||||
CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
|
|
||||||
VolumeType: "",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"v2": {
|
|
||||||
// Non-empty string is skipped.
|
|
||||||
CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{
|
|
||||||
VolumeType: "already-set",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"v3": {
|
|
||||||
// No volume type set.
|
|
||||||
},
|
|
||||||
"v4": nil,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// We can't set an empty string in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.volumes.v1.volume_type", dyn.V(""))
|
|
||||||
})
|
|
||||||
|
|
||||||
diags := bundle.Apply(context.Background(), b, mutator.ConfigureVolumeDefaults())
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
var v dyn.Value
|
|
||||||
var err error
|
|
||||||
|
|
||||||
// Set to empty string; unchanged.
|
|
||||||
v, err = dyn.Get(b.Config.Value(), "resources.volumes.v1.volume_type")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "", v.MustString())
|
|
||||||
|
|
||||||
// Set to non-empty string; unchanged.
|
|
||||||
v, err = dyn.Get(b.Config.Value(), "resources.volumes.v2.volume_type")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "already-set", v.MustString())
|
|
||||||
|
|
||||||
// Not set; set to default.
|
|
||||||
v, err = dyn.Get(b.Config.Value(), "resources.volumes.v3.volume_type")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "MANAGED", v.MustString())
|
|
||||||
|
|
||||||
// No valid volume; No change.
|
|
||||||
_, err = dyn.Get(b.Config.Value(), "resources.volumes.v4.volume_type")
|
|
||||||
assert.True(t, dyn.IsCannotTraverseNilError(err))
|
|
||||||
}
|
|
|
@ -17,7 +17,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func touchEmptyFile(t *testing.T, path string) {
|
func touchEmptyFile(t *testing.T, path string) {
|
||||||
err := os.MkdirAll(filepath.Dir(path), 0o700)
|
err := os.MkdirAll(filepath.Dir(path), 0700)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
f, err := os.Create(path)
|
f, err := os.Create(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
|
@ -28,7 +28,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
||||||
}
|
}
|
||||||
|
|
||||||
currentUser := b.Config.Workspace.CurrentUser
|
currentUser := b.Config.Workspace.CurrentUser
|
||||||
if currentUser == nil || currentUser.User == nil || currentUser.UserName == "" {
|
if currentUser == nil || currentUser.UserName == "" {
|
||||||
return diag.Errorf("unable to expand workspace root: current user not set")
|
return diag.Errorf("unable to expand workspace root: current user not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,8 @@ import (
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
)
|
)
|
||||||
|
|
||||||
type initializeURLs struct{}
|
type initializeURLs struct {
|
||||||
|
}
|
||||||
|
|
||||||
// InitializeURLs makes sure the URL field of each resource is configured.
|
// InitializeURLs makes sure the URL field of each resource is configured.
|
||||||
// NOTE: since this depends on an extra API call, this mutator adds some extra
|
// NOTE: since this depends on an extra API call, this mutator adds some extra
|
||||||
|
@ -31,14 +32,11 @@ func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
|
||||||
}
|
}
|
||||||
orgId := strconv.FormatInt(workspaceId, 10)
|
orgId := strconv.FormatInt(workspaceId, 10)
|
||||||
host := b.WorkspaceClient().Config.CanonicalHostName()
|
host := b.WorkspaceClient().Config.CanonicalHostName()
|
||||||
err = initializeForWorkspace(b, orgId, host)
|
initializeForWorkspace(b, orgId, host)
|
||||||
if err != nil {
|
|
||||||
return diag.FromErr(err)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func initializeForWorkspace(b *bundle.Bundle, orgId, host string) error {
|
func initializeForWorkspace(b *bundle.Bundle, orgId string, host string) error {
|
||||||
baseURL, err := url.Parse(host)
|
baseURL, err := url.Parse(host)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -110,8 +110,7 @@ func TestInitializeURLs(t *testing.T) {
|
||||||
"dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456",
|
"dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456",
|
||||||
}
|
}
|
||||||
|
|
||||||
err := initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/")
|
initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/")
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, group := range b.Config.Resources.AllResources() {
|
for _, group := range b.Config.Resources.AllResources() {
|
||||||
for key, r := range group.Resources {
|
for key, r := range group.Resources {
|
||||||
|
@ -134,8 +133,7 @@ func TestInitializeURLsWithoutOrgId(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := initializeForWorkspace(b, "123456", "https://adb-123456.azuredatabricks.net/")
|
initializeForWorkspace(b, "123456", "https://adb-123456.azuredatabricks.net/")
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.Equal(t, "https://adb-123456.azuredatabricks.net/jobs/1", b.Config.Resources.Jobs["job1"].URL)
|
require.Equal(t, "https://adb-123456.azuredatabricks.net/jobs/1", b.Config.Resources.Jobs["job1"].URL)
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,12 @@ package mutator
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/git"
|
"github.com/databricks/cli/libs/git"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type loadGitDetails struct{}
|
type loadGitDetails struct{}
|
||||||
|
@ -23,42 +21,50 @@ func (m *loadGitDetails) Name() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
var diags diag.Diagnostics
|
// Load relevant git repository
|
||||||
info, err := git.FetchRepositoryInfo(ctx, b.BundleRoot.Native(), b.WorkspaceClient())
|
repo, err := git.NewRepository(b.BundleRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
return diag.FromErr(err)
|
||||||
diags = append(diags, diag.WarningFromErr(err)...)
|
}
|
||||||
|
|
||||||
|
// Read branch name of current checkout
|
||||||
|
branch, err := repo.CurrentBranch()
|
||||||
|
if err == nil {
|
||||||
|
b.Config.Bundle.Git.ActualBranch = branch
|
||||||
|
if b.Config.Bundle.Git.Branch == "" {
|
||||||
|
// Only load branch if there's no user defined value
|
||||||
|
b.Config.Bundle.Git.Inferred = true
|
||||||
|
b.Config.Bundle.Git.Branch = branch
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if info.WorktreeRoot == "" {
|
|
||||||
b.WorktreeRoot = b.BundleRoot
|
|
||||||
} else {
|
} else {
|
||||||
b.WorktreeRoot = vfs.MustNew(info.WorktreeRoot)
|
log.Warnf(ctx, "failed to load current branch: %s", err)
|
||||||
}
|
|
||||||
|
|
||||||
b.Config.Bundle.Git.ActualBranch = info.CurrentBranch
|
|
||||||
if b.Config.Bundle.Git.Branch == "" {
|
|
||||||
// Only load branch if there's no user defined value
|
|
||||||
b.Config.Bundle.Git.Inferred = true
|
|
||||||
b.Config.Bundle.Git.Branch = info.CurrentBranch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// load commit hash if undefined
|
// load commit hash if undefined
|
||||||
if b.Config.Bundle.Git.Commit == "" {
|
if b.Config.Bundle.Git.Commit == "" {
|
||||||
b.Config.Bundle.Git.Commit = info.LatestCommit
|
commit, err := repo.LatestCommit()
|
||||||
|
if err != nil {
|
||||||
|
log.Warnf(ctx, "failed to load latest commit: %s", err)
|
||||||
|
} else {
|
||||||
|
b.Config.Bundle.Git.Commit = commit
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// load origin url if undefined
|
// load origin url if undefined
|
||||||
if b.Config.Bundle.Git.OriginURL == "" {
|
if b.Config.Bundle.Git.OriginURL == "" {
|
||||||
b.Config.Bundle.Git.OriginURL = info.OriginURL
|
remoteUrl := repo.OriginUrl()
|
||||||
|
b.Config.Bundle.Git.OriginURL = remoteUrl
|
||||||
}
|
}
|
||||||
|
|
||||||
relBundlePath, err := filepath.Rel(b.WorktreeRoot.Native(), b.BundleRoot.Native())
|
// Compute relative path of the bundle root from the Git repo root.
|
||||||
|
absBundlePath, err := filepath.Abs(b.BundleRootPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
diags = append(diags, diag.FromErr(err)...)
|
return diag.FromErr(err)
|
||||||
} else {
|
|
||||||
b.Config.Bundle.Git.BundleRootPath = filepath.ToSlash(relBundlePath)
|
|
||||||
}
|
}
|
||||||
return diags
|
// repo.Root() returns the absolute path of the repo
|
||||||
|
relBundlePath, err := filepath.Rel(repo.Root(), absBundlePath)
|
||||||
|
if err != nil {
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
b.Config.Bundle.Git.BundleRootPath = filepath.ToSlash(relBundlePath)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,6 +26,7 @@ func DefaultMutators() []bundle.Mutator {
|
||||||
ComputeIdToClusterId(),
|
ComputeIdToClusterId(),
|
||||||
InitializeVariables(),
|
InitializeVariables(),
|
||||||
DefineDefaultTarget(),
|
DefineDefaultTarget(),
|
||||||
|
LoadGitDetails(),
|
||||||
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
|
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
|
||||||
|
|
||||||
// Note: This mutator must run before the target overrides are merged.
|
// Note: This mutator must run before the target overrides are merged.
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/libs/cmdio"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/env"
|
"github.com/databricks/cli/libs/env"
|
||||||
)
|
)
|
||||||
|
@ -23,7 +22,7 @@ func (m *overrideCompute) Name() string {
|
||||||
|
|
||||||
func overrideJobCompute(j *resources.Job, compute string) {
|
func overrideJobCompute(j *resources.Job, compute string) {
|
||||||
for i := range j.Tasks {
|
for i := range j.Tasks {
|
||||||
task := &j.Tasks[i]
|
var task = &j.Tasks[i]
|
||||||
|
|
||||||
if task.ForEachTask != nil {
|
if task.ForEachTask != nil {
|
||||||
task = &task.ForEachTask.Task
|
task = &task.ForEachTask.Task
|
||||||
|
@ -39,32 +38,18 @@ func overrideJobCompute(j *resources.Job, compute string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
var diags diag.Diagnostics
|
if b.Config.Bundle.Mode != config.Development {
|
||||||
|
|
||||||
if b.Config.Bundle.Mode == config.Production {
|
|
||||||
if b.Config.Bundle.ClusterId != "" {
|
if b.Config.Bundle.ClusterId != "" {
|
||||||
// Overriding compute via a command-line flag for production works, but is not recommended.
|
return diag.Errorf("cannot override compute for an target that does not use 'mode: development'")
|
||||||
diags = diags.Extend(diag.Diagnostics{{
|
|
||||||
Summary: "Setting a cluster override for a target that uses 'mode: production' is not recommended",
|
|
||||||
Detail: "It is recommended to always use the same compute for production target for consistency.",
|
|
||||||
Severity: diag.Warning,
|
|
||||||
}})
|
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" {
|
if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" {
|
||||||
// For historical reasons, we allow setting the cluster ID via the DATABRICKS_CLUSTER_ID
|
|
||||||
// when development mode is used. Sometimes, this is done by accident, so we log an info message.
|
|
||||||
if b.Config.Bundle.Mode == config.Development {
|
|
||||||
cmdio.LogString(ctx, "Setting a cluster override because DATABRICKS_CLUSTER_ID is set. It is recommended to use --cluster-id instead, which works in any target mode.")
|
|
||||||
} else {
|
|
||||||
// We don't allow using DATABRICKS_CLUSTER_ID in any other mode, it's too error-prone.
|
|
||||||
return diag.Warningf("The DATABRICKS_CLUSTER_ID variable is set but is ignored since the current target does not use 'mode: development'")
|
|
||||||
}
|
|
||||||
b.Config.Bundle.ClusterId = v
|
b.Config.Bundle.ClusterId = v
|
||||||
}
|
}
|
||||||
|
|
||||||
if b.Config.Bundle.ClusterId == "" {
|
if b.Config.Bundle.ClusterId == "" {
|
||||||
return diags
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
r := b.Config.Resources
|
r := b.Config.Resources
|
||||||
|
@ -72,5 +57,5 @@ func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag
|
||||||
overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId)
|
overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId)
|
||||||
}
|
}
|
||||||
|
|
||||||
return diags
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,14 +8,13 @@ import (
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/libs/diag"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOverrideComputeModeDevelopment(t *testing.T) {
|
func TestOverrideDevelopment(t *testing.T) {
|
||||||
t.Setenv("DATABRICKS_CLUSTER_ID", "")
|
t.Setenv("DATABRICKS_CLUSTER_ID", "")
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
|
@ -63,13 +62,10 @@ func TestOverrideComputeModeDevelopment(t *testing.T) {
|
||||||
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[3].JobClusterKey)
|
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[3].JobClusterKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideComputeModeDefaultIgnoresVariable(t *testing.T) {
|
func TestOverrideDevelopmentEnv(t *testing.T) {
|
||||||
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Bundle: config.Bundle{
|
|
||||||
Mode: "",
|
|
||||||
},
|
|
||||||
Resources: config.Resources{
|
Resources: config.Resources{
|
||||||
Jobs: map[string]*resources.Job{
|
Jobs: map[string]*resources.Job{
|
||||||
"job1": {JobSettings: &jobs.JobSettings{
|
"job1": {JobSettings: &jobs.JobSettings{
|
||||||
|
@ -90,12 +86,11 @@ func TestOverrideComputeModeDefaultIgnoresVariable(t *testing.T) {
|
||||||
|
|
||||||
m := mutator.OverrideCompute()
|
m := mutator.OverrideCompute()
|
||||||
diags := bundle.Apply(context.Background(), b, m)
|
diags := bundle.Apply(context.Background(), b, m)
|
||||||
require.Len(t, diags, 1)
|
require.NoError(t, diags.Error())
|
||||||
assert.Equal(t, "The DATABRICKS_CLUSTER_ID variable is set but is ignored since the current target does not use 'mode: development'", diags[0].Summary)
|
|
||||||
assert.Equal(t, "cluster2", b.Config.Resources.Jobs["job1"].Tasks[1].ExistingClusterId)
|
assert.Equal(t, "cluster2", b.Config.Resources.Jobs["job1"].Tasks[1].ExistingClusterId)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideComputePipelineTask(t *testing.T) {
|
func TestOverridePipelineTask(t *testing.T) {
|
||||||
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
|
@ -120,7 +115,7 @@ func TestOverrideComputePipelineTask(t *testing.T) {
|
||||||
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[0].ExistingClusterId)
|
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[0].ExistingClusterId)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideComputeForEachTask(t *testing.T) {
|
func TestOverrideForEachTask(t *testing.T) {
|
||||||
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
|
@ -145,11 +140,10 @@ func TestOverrideComputeForEachTask(t *testing.T) {
|
||||||
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[0].ForEachTask.Task)
|
assert.Empty(t, b.Config.Resources.Jobs["job1"].Tasks[0].ForEachTask.Task)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideComputeModeProduction(t *testing.T) {
|
func TestOverrideProduction(t *testing.T) {
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Bundle: config.Bundle{
|
Bundle: config.Bundle{
|
||||||
Mode: config.Production,
|
|
||||||
ClusterId: "newClusterID",
|
ClusterId: "newClusterID",
|
||||||
},
|
},
|
||||||
Resources: config.Resources{
|
Resources: config.Resources{
|
||||||
|
@ -172,19 +166,13 @@ func TestOverrideComputeModeProduction(t *testing.T) {
|
||||||
|
|
||||||
m := mutator.OverrideCompute()
|
m := mutator.OverrideCompute()
|
||||||
diags := bundle.Apply(context.Background(), b, m)
|
diags := bundle.Apply(context.Background(), b, m)
|
||||||
require.Len(t, diags, 1)
|
require.True(t, diags.HasError())
|
||||||
assert.Equal(t, "Setting a cluster override for a target that uses 'mode: production' is not recommended", diags[0].Summary)
|
|
||||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
|
||||||
assert.Equal(t, "newClusterID", b.Config.Resources.Jobs["job1"].Tasks[0].ExistingClusterId)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOverrideComputeModeProductionIgnoresVariable(t *testing.T) {
|
func TestOverrideProductionEnv(t *testing.T) {
|
||||||
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
t.Setenv("DATABRICKS_CLUSTER_ID", "newClusterId")
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Bundle: config.Bundle{
|
|
||||||
Mode: config.Production,
|
|
||||||
},
|
|
||||||
Resources: config.Resources{
|
Resources: config.Resources{
|
||||||
Jobs: map[string]*resources.Job{
|
Jobs: map[string]*resources.Job{
|
||||||
"job1": {JobSettings: &jobs.JobSettings{
|
"job1": {JobSettings: &jobs.JobSettings{
|
||||||
|
@ -205,7 +193,5 @@ func TestOverrideComputeModeProductionIgnoresVariable(t *testing.T) {
|
||||||
|
|
||||||
m := mutator.OverrideCompute()
|
m := mutator.OverrideCompute()
|
||||||
diags := bundle.Apply(context.Background(), b, m)
|
diags := bundle.Apply(context.Background(), b, m)
|
||||||
require.Len(t, diags, 1)
|
require.NoError(t, diags.Error())
|
||||||
assert.Equal(t, "The DATABRICKS_CLUSTER_ID variable is set but is ignored since the current target does not use 'mode: development'", diags[0].Summary)
|
|
||||||
assert.Equal(t, "cluster2", b.Config.Resources.Jobs["job1"].Tasks[1].ExistingClusterId)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -95,7 +95,7 @@ func jobRewritePatterns() []jobRewritePattern {
|
||||||
// VisitJobPaths visits all paths in job resources and applies a function to each path.
|
// VisitJobPaths visits all paths in job resources and applies a function to each path.
|
||||||
func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
|
func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
|
||||||
var err error
|
var err error
|
||||||
newValue := value
|
var newValue = value
|
||||||
|
|
||||||
for _, rewritePattern := range jobRewritePatterns() {
|
for _, rewritePattern := range jobRewritePatterns() {
|
||||||
newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
@ -105,6 +105,7 @@ func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
|
||||||
|
|
||||||
return fn(p, rewritePattern.kind, v)
|
return fn(p, rewritePattern.kind, v)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dyn.InvalidValue, err
|
return dyn.InvalidValue, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,12 +57,14 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
|
||||||
|
|
||||||
return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil
|
return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dyn.InvalidValue, err
|
return dyn.InvalidValue, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return v, nil
|
return v, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diag.FromErr(err)
|
return diag.FromErr(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/iamutil"
|
"github.com/databricks/cli/libs/iamutil"
|
||||||
|
@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
|
||||||
t.TriggerPauseStatus = config.Paused
|
t.TriggerPauseStatus = config.Paused
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
|
|
||||||
isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
|
|
||||||
if isInWorkspace && dbr.RunsOnRuntime(ctx) {
|
|
||||||
enabled := true
|
|
||||||
t.SourceLinkedDeployment = &enabled
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
|
||||||
enabled := true
|
enabled := true
|
||||||
t.PipelinesDevelopment = &enabled
|
t.PipelinesDevelopment = &enabled
|
||||||
|
|
|
@ -3,17 +3,14 @@ package mutator
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
"runtime"
|
"strings"
|
||||||
"slices"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
|
||||||
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
|
@ -131,9 +128,6 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
||||||
Schemas: map[string]*resources.Schema{
|
Schemas: map[string]*resources.Schema{
|
||||||
"schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}},
|
"schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}},
|
||||||
},
|
},
|
||||||
Volumes: map[string]*resources.Volume{
|
|
||||||
"volume1": {CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{Name: "volume1"}},
|
|
||||||
},
|
|
||||||
Clusters: map[string]*resources.Cluster{
|
Clusters: map[string]*resources.Cluster{
|
||||||
"cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}},
|
"cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}},
|
||||||
},
|
},
|
||||||
|
@ -146,7 +140,6 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
|
|
||||||
// Use AWS implementation for testing.
|
// Use AWS implementation for testing.
|
||||||
Tagging: tags.ForCloud(&sdkconfig.Config{
|
Tagging: tags.ForCloud(&sdkconfig.Config{
|
||||||
Host: "https://company.cloud.databricks.com",
|
Host: "https://company.cloud.databricks.com",
|
||||||
|
@ -314,8 +307,6 @@ func TestProcessTargetModeDefault(t *testing.T) {
|
||||||
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
||||||
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
||||||
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
||||||
assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name)
|
|
||||||
assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name)
|
|
||||||
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -360,8 +351,6 @@ func TestProcessTargetModeProduction(t *testing.T) {
|
||||||
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
||||||
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
||||||
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
||||||
assert.Equal(t, "schema1", b.Config.Resources.Schemas["schema1"].Name)
|
|
||||||
assert.Equal(t, "volume1", b.Config.Resources.Volumes["volume1"].Name)
|
|
||||||
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -395,17 +384,10 @@ func TestAllResourcesMocked(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make sure that we at rename all non UC resources
|
// Make sure that we at least rename all resources
|
||||||
func TestAllNonUcResourcesAreRenamed(t *testing.T) {
|
func TestAllResourcesRenamed(t *testing.T) {
|
||||||
b := mockBundle(config.Development)
|
b := mockBundle(config.Development)
|
||||||
|
|
||||||
// UC resources should not have a prefix added to their name. Right now
|
|
||||||
// this list only contains the Volume resource since we have yet to remove
|
|
||||||
// prefixing support for UC schemas and registered models.
|
|
||||||
ucFields := []reflect.Type{
|
|
||||||
reflect.TypeOf(&resources.Volume{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
|
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
|
||||||
diags := bundle.Apply(context.Background(), b, m)
|
diags := bundle.Apply(context.Background(), b, m)
|
||||||
require.NoError(t, diags.Error())
|
require.NoError(t, diags.Error())
|
||||||
|
@ -418,14 +400,14 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
|
||||||
for _, key := range field.MapKeys() {
|
for _, key := range field.MapKeys() {
|
||||||
resource := field.MapIndex(key)
|
resource := field.MapIndex(key)
|
||||||
nameField := resource.Elem().FieldByName("Name")
|
nameField := resource.Elem().FieldByName("Name")
|
||||||
if !nameField.IsValid() || nameField.Kind() != reflect.String {
|
if nameField.IsValid() && nameField.Kind() == reflect.String {
|
||||||
continue
|
assert.True(
|
||||||
}
|
t,
|
||||||
|
strings.Contains(nameField.String(), "dev"),
|
||||||
if slices.Contains(ucFields, resource.Type()) {
|
"process_target_mode should rename '%s' in '%s'",
|
||||||
assert.NotContains(t, nameField.String(), "dev", "process_target_mode should not rename '%s' in '%s'", key, resources.Type().Field(i).Name)
|
key,
|
||||||
} else {
|
resources.Type().Field(i).Name,
|
||||||
assert.Contains(t, nameField.String(), "dev", "process_target_mode should rename '%s' in '%s'", key, resources.Type().Field(i).Name)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -540,32 +522,3 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
|
||||||
|
|
||||||
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentEnabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, true)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSourceLinkedDeploymentDisabled(t *testing.T) {
|
|
||||||
b, diags := processSourceLinkedBundle(t, false)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
b := mockBundle(config.Development)
|
|
||||||
|
|
||||||
workspacePath := "/Workspace/lennart@company.com/"
|
|
||||||
b.SyncRootPath = workspacePath
|
|
||||||
b.Config.Presets.SourceLinkedDeployment = &presetEnabled
|
|
||||||
|
|
||||||
ctx := dbr.MockRuntime(context.Background(), true)
|
|
||||||
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
|
|
||||||
diags := bundle.Apply(ctx, b, m)
|
|
||||||
return b, diags
|
|
||||||
}
|
|
||||||
|
|
|
@ -30,6 +30,7 @@ type parsePythonDiagnosticsTest struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParsePythonDiagnostics(t *testing.T) {
|
func TestParsePythonDiagnostics(t *testing.T) {
|
||||||
|
|
||||||
testCases := []parsePythonDiagnosticsTest{
|
testCases := []parsePythonDiagnosticsTest{
|
||||||
{
|
{
|
||||||
name: "short error with location",
|
name: "short error with location",
|
||||||
|
|
|
@ -9,11 +9,12 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/logger"
|
"github.com/databricks/databricks-sdk-go/logger"
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
|
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/python"
|
"github.com/databricks/cli/libs/python"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle/env"
|
"github.com/databricks/cli/bundle/env"
|
||||||
|
@ -93,10 +94,11 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
|
||||||
|
|
||||||
// mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics'
|
// mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics'
|
||||||
var mutateDiags diag.Diagnostics
|
var mutateDiags diag.Diagnostics
|
||||||
mutateDiagsHasError := errors.New("unexpected error")
|
var mutateDiagsHasError = errors.New("unexpected error")
|
||||||
|
|
||||||
err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
|
err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
|
||||||
pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
|
pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
|
return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -139,7 +141,7 @@ func createCacheDir(ctx context.Context) (string, error) {
|
||||||
// use 'default' as target name
|
// use 'default' as target name
|
||||||
cacheDir := filepath.Join(tempDir, "default", "pydabs")
|
cacheDir := filepath.Join(tempDir, "default", "pydabs")
|
||||||
|
|
||||||
err := os.MkdirAll(cacheDir, 0o700)
|
err := os.MkdirAll(cacheDir, 0700)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
@ -150,7 +152,7 @@ func createCacheDir(ctx context.Context) (string, error) {
|
||||||
return os.MkdirTemp("", "-pydabs")
|
return os.MkdirTemp("", "-pydabs")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
|
func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, rootPath string, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
|
||||||
inputPath := filepath.Join(cacheDir, "input.json")
|
inputPath := filepath.Join(cacheDir, "input.json")
|
||||||
outputPath := filepath.Join(cacheDir, "output.json")
|
outputPath := filepath.Join(cacheDir, "output.json")
|
||||||
diagnosticsPath := filepath.Join(cacheDir, "diagnostics.json")
|
diagnosticsPath := filepath.Join(cacheDir, "diagnostics.json")
|
||||||
|
@ -261,10 +263,10 @@ func writeInputFile(inputPath string, input dyn.Value) error {
|
||||||
return fmt.Errorf("failed to marshal input: %w", err)
|
return fmt.Errorf("failed to marshal input: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return os.WriteFile(inputPath, rootConfigJson, 0o600)
|
return os.WriteFile(inputPath, rootConfigJson, 0600)
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
|
func loadOutputFile(rootPath string, outputPath string) (dyn.Value, diag.Diagnostics) {
|
||||||
outputFile, err := os.Open(outputPath)
|
outputFile, err := os.Open(outputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err))
|
return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err))
|
||||||
|
@ -379,7 +381,7 @@ func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
|
||||||
|
|
||||||
return right, nil
|
return right, nil
|
||||||
},
|
},
|
||||||
VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) {
|
VisitUpdate: func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) {
|
||||||
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String())
|
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String())
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -428,7 +430,7 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
|
||||||
|
|
||||||
return right, nil
|
return right, nil
|
||||||
},
|
},
|
||||||
VisitUpdate: func(valuePath dyn.Path, left, right dyn.Value) (dyn.Value, error) {
|
VisitUpdate: func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) {
|
||||||
if !valuePath.HasPrefix(jobsPath) {
|
if !valuePath.HasPrefix(jobsPath) {
|
||||||
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String())
|
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (update)", valuePath.String())
|
||||||
}
|
}
|
||||||
|
|
|
@ -106,6 +106,7 @@ func TestPythonMutator_load(t *testing.T) {
|
||||||
Column: 5,
|
Column: 5,
|
||||||
},
|
},
|
||||||
}, diags[0].Locations)
|
}, diags[0].Locations)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPythonMutator_load_disallowed(t *testing.T) {
|
func TestPythonMutator_load_disallowed(t *testing.T) {
|
||||||
|
@ -541,7 +542,7 @@ func TestLoadDiagnosticsFile_nonExistent(t *testing.T) {
|
||||||
|
|
||||||
func TestInterpreterPath(t *testing.T) {
|
func TestInterpreterPath(t *testing.T) {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
assert.Equal(t, "venv\\Scripts\\python.exe", interpreterPath("venv"))
|
assert.Equal(t, "venv\\Scripts\\python3.exe", interpreterPath("venv"))
|
||||||
} else {
|
} else {
|
||||||
assert.Equal(t, "venv/bin/python3", interpreterPath("venv"))
|
assert.Equal(t, "venv/bin/python3", interpreterPath("venv"))
|
||||||
}
|
}
|
||||||
|
@ -587,7 +588,7 @@ or activate the environment before running CLI commands:
|
||||||
assert.Equal(t, expected, out)
|
assert.Equal(t, expected, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
func withProcessStub(t *testing.T, args []string, output, diagnostics string) context.Context {
|
func withProcessStub(t *testing.T, args []string, output string, diagnostics string) context.Context {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
ctx, stub := process.WithStub(ctx)
|
ctx, stub := process.WithStub(ctx)
|
||||||
|
|
||||||
|
@ -610,10 +611,10 @@ func withProcessStub(t *testing.T, args []string, output, diagnostics string) co
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
if reflect.DeepEqual(actual.Args, args) {
|
if reflect.DeepEqual(actual.Args, args) {
|
||||||
err := os.WriteFile(outputPath, []byte(output), 0o600)
|
err := os.WriteFile(outputPath, []byte(output), 0600)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = os.WriteFile(diagnosticsPath, []byte(diagnostics), 0o600)
|
err = os.WriteFile(diagnosticsPath, []byte(diagnostics), 0600)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -625,7 +626,7 @@ func withProcessStub(t *testing.T, args []string, output, diagnostics string) co
|
||||||
return ctx
|
return ctx
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadYaml(name, content string) *bundle.Bundle {
|
func loadYaml(name string, content string) *bundle.Bundle {
|
||||||
v, diag := config.LoadFromBytes(name, []byte(content))
|
v, diag := config.LoadFromBytes(name, []byte(content))
|
||||||
|
|
||||||
if diag.Error() != nil {
|
if diag.Error() != nil {
|
||||||
|
@ -649,17 +650,17 @@ func withFakeVEnv(t *testing.T, venvPath string) {
|
||||||
|
|
||||||
interpreterPath := interpreterPath(venvPath)
|
interpreterPath := interpreterPath(venvPath)
|
||||||
|
|
||||||
err = os.MkdirAll(filepath.Dir(interpreterPath), 0o755)
|
err = os.MkdirAll(filepath.Dir(interpreterPath), 0755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.WriteFile(interpreterPath, []byte(""), 0o755)
|
err = os.WriteFile(interpreterPath, []byte(""), 0755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.WriteFile(filepath.Join(venvPath, "pyvenv.cfg"), []byte(""), 0o755)
|
err = os.WriteFile(filepath.Join(venvPath, "pyvenv.cfg"), []byte(""), 0755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
@ -673,7 +674,7 @@ func withFakeVEnv(t *testing.T, venvPath string) {
|
||||||
|
|
||||||
func interpreterPath(venvPath string) string {
|
func interpreterPath(venvPath string) string {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
return filepath.Join(venvPath, "Scripts", "python.exe")
|
return filepath.Join(venvPath, "Scripts", "python3.exe")
|
||||||
} else {
|
} else {
|
||||||
return filepath.Join(venvPath, "bin", "python3")
|
return filepath.Join(venvPath, "bin", "python3")
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,7 +36,8 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle)
|
||||||
return fmt.Errorf("failed to resolve %s, err: %w", v.Lookup, err)
|
return fmt.Errorf("failed to resolve %s, err: %w", v.Lookup, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return v.Set(id)
|
v.Set(id)
|
||||||
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -108,8 +108,7 @@ func TestNoLookupIfVariableIsSet(t *testing.T) {
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
m := mocks.NewMockWorkspaceClient(t)
|
||||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||||
|
|
||||||
err := b.Config.Variables["my-cluster-id"].Set("random value")
|
b.Config.Variables["my-cluster-id"].Set("random value")
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
|
diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
|
||||||
require.NoError(t, diags.Error())
|
require.NoError(t, diags.Error())
|
||||||
|
|
|
@ -32,12 +32,11 @@ func ResolveVariableReferencesInLookup() bundle.Mutator {
|
||||||
}
|
}
|
||||||
|
|
||||||
func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
|
func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
|
||||||
return &resolveVariableReferences{
|
return &resolveVariableReferences{prefixes: []string{
|
||||||
prefixes: []string{
|
"bundle",
|
||||||
"bundle",
|
"workspace",
|
||||||
"workspace",
|
"variables",
|
||||||
"variables",
|
},
|
||||||
},
|
|
||||||
pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
|
pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
|
||||||
lookupFn: lookupForComplexVariables,
|
lookupFn: lookupForComplexVariables,
|
||||||
skipFn: skipResolvingInNonComplexVariables,
|
skipFn: skipResolvingInNonComplexVariables,
|
||||||
|
@ -174,6 +173,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
|
||||||
return dyn.InvalidValue, dynvar.ErrSkipResolution
|
return dyn.InvalidValue, dynvar.ErrSkipResolution
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dyn.InvalidValue, err
|
return dyn.InvalidValue, err
|
||||||
}
|
}
|
||||||
|
@ -184,6 +184,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
|
||||||
diags = diags.Extend(normaliseDiags)
|
diags = diags.Extend(normaliseDiags)
|
||||||
return root, nil
|
return root, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
diags = diags.Extend(diag.FromErr(err))
|
diags = diags.Extend(diag.FromErr(err))
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,6 +63,7 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
|
||||||
return v, nil
|
return v, nil
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diag.FromErr(err)
|
return diag.FromErr(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -81,4 +81,5 @@ func TestNoWorkspacePrefixUsed(t *testing.T) {
|
||||||
require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar)
|
require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar)
|
||||||
require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath)
|
require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath)
|
||||||
require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar)
|
require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,8 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type setRunAs struct{}
|
type setRunAs struct {
|
||||||
|
}
|
||||||
|
|
||||||
// This mutator does two things:
|
// This mutator does two things:
|
||||||
//
|
//
|
||||||
|
@ -29,7 +30,7 @@ func (m *setRunAs) Name() string {
|
||||||
return "SetRunAs"
|
return "SetRunAs"
|
||||||
}
|
}
|
||||||
|
|
||||||
func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser, runAsUser string) diag.Diagnostics {
|
func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser string, runAsUser string) diag.Diagnostics {
|
||||||
return diag.Diagnostics{{
|
return diag.Diagnostics{{
|
||||||
Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+
|
Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+
|
||||||
"Current identity: %s. Run as identity: %s.\n"+
|
"Current identity: %s. Run as identity: %s.\n"+
|
||||||
|
|
|
@ -42,7 +42,6 @@ func allResourceTypes(t *testing.T) []string {
|
||||||
"quality_monitors",
|
"quality_monitors",
|
||||||
"registered_models",
|
"registered_models",
|
||||||
"schemas",
|
"schemas",
|
||||||
"volumes",
|
|
||||||
},
|
},
|
||||||
resourceTypes,
|
resourceTypes,
|
||||||
)
|
)
|
||||||
|
@ -142,7 +141,6 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
||||||
"registered_models",
|
"registered_models",
|
||||||
"experiments",
|
"experiments",
|
||||||
"schemas",
|
"schemas",
|
||||||
"volumes",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
base := config.Root{
|
base := config.Root{
|
||||||
|
|
|
@ -65,6 +65,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
|
||||||
|
|
||||||
// We should have had a value to set for the variable at this point.
|
// We should have had a value to set for the variable at this point.
|
||||||
return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
|
return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
|
|
@ -35,7 +35,7 @@ func (m *syncInferRoot) Name() string {
|
||||||
// If the path does not exist, it returns an empty string.
|
// If the path does not exist, it returns an empty string.
|
||||||
//
|
//
|
||||||
// See "sync_infer_root_internal_test.go" for examples.
|
// See "sync_infer_root_internal_test.go" for examples.
|
||||||
func (m *syncInferRoot) computeRoot(path, root string) string {
|
func (m *syncInferRoot) computeRoot(path string, root string) string {
|
||||||
for !filepath.IsLocal(path) {
|
for !filepath.IsLocal(path) {
|
||||||
// Break if we have reached the root of the filesystem.
|
// Break if we have reached the root of the filesystem.
|
||||||
dir := filepath.Dir(root)
|
dir := filepath.Dir(root)
|
||||||
|
|
|
@ -11,7 +11,6 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/notebook"
|
"github.com/databricks/cli/libs/notebook"
|
||||||
|
@ -104,13 +103,8 @@ func (t *translateContext) rewritePath(
|
||||||
return fmt.Errorf("path %s is not contained in sync root path", localPath)
|
return fmt.Errorf("path %s is not contained in sync root path", localPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
var workspacePath string
|
// Prefix remote path with its remote root path.
|
||||||
if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) {
|
remotePath := path.Join(t.b.Config.Workspace.FilePath, filepath.ToSlash(localRelPath))
|
||||||
workspacePath = t.b.SyncRootPath
|
|
||||||
} else {
|
|
||||||
workspacePath = t.b.Config.Workspace.FilePath
|
|
||||||
}
|
|
||||||
remotePath := path.Join(workspacePath, filepath.ToSlash(localRelPath))
|
|
||||||
|
|
||||||
// Convert local path into workspace path via specified function.
|
// Convert local path into workspace path via specified function.
|
||||||
interp, err := fn(*p, localPath, localRelPath, remotePath)
|
interp, err := fn(*p, localPath, localRelPath, remotePath)
|
||||||
|
@ -126,33 +120,7 @@ func (t *translateContext) rewritePath(
|
||||||
func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||||
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
|
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
if filepath.Ext(localFullPath) != notebook.ExtensionNone {
|
return "", fmt.Errorf("notebook %s not found", literal)
|
||||||
return "", fmt.Errorf("notebook %s not found", literal)
|
|
||||||
}
|
|
||||||
|
|
||||||
extensions := []string{
|
|
||||||
notebook.ExtensionPython,
|
|
||||||
notebook.ExtensionR,
|
|
||||||
notebook.ExtensionScala,
|
|
||||||
notebook.ExtensionSql,
|
|
||||||
notebook.ExtensionJupyter,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check whether a file with a notebook extension already exists. This
|
|
||||||
// way we can provide a more targeted error message.
|
|
||||||
for _, ext := range extensions {
|
|
||||||
literalWithExt := literal + ext
|
|
||||||
localRelPathWithExt := filepath.ToSlash(localRelPath + ext)
|
|
||||||
if _, err := fs.Stat(t.b.SyncRoot, localRelPathWithExt); err == nil {
|
|
||||||
return "", fmt.Errorf(`notebook %s not found. Did you mean %s?
|
|
||||||
Local notebook references are expected to contain one of the following
|
|
||||||
file extensions: [%s]`, literal, literalWithExt, strings.Join(extensions, ", "))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return a generic error message if no matching possible file is found.
|
|
||||||
return "", fmt.Errorf(`notebook %s not found. Local notebook references are expected
|
|
||||||
to contain one of the following file extensions: [%s]`, literal, strings.Join(extensions, ", "))
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("unable to determine if %s is a notebook: %w", localFullPath, err)
|
return "", fmt.Errorf("unable to determine if %s is a notebook: %w", localFullPath, err)
|
||||||
|
@ -275,8 +243,8 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
}
|
}
|
||||||
|
|
||||||
func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) {
|
func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) {
|
||||||
fallback := make(map[string]string)
|
var fallback = make(map[string]string)
|
||||||
pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey())
|
var pattern = dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey())
|
||||||
|
|
||||||
// Previous behavior was to use a resource's location as the base path to resolve
|
// Previous behavior was to use a resource's location as the base path to resolve
|
||||||
// relative paths in its definition. With the introduction of [dyn.Value] throughout,
|
// relative paths in its definition. With the introduction of [dyn.Value] throughout,
|
||||||
|
|
|
@ -2,10 +2,8 @@ package mutator_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -28,13 +26,12 @@ import (
|
||||||
func touchNotebookFile(t *testing.T, path string) {
|
func touchNotebookFile(t *testing.T, path string) {
|
||||||
f, err := os.Create(path)
|
f, err := os.Create(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
_, err = f.WriteString("# Databricks notebook source\n")
|
f.WriteString("# Databricks notebook source\n")
|
||||||
require.NoError(t, err)
|
|
||||||
f.Close()
|
f.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
func touchEmptyFile(t *testing.T, path string) {
|
func touchEmptyFile(t *testing.T, path string) {
|
||||||
err := os.MkdirAll(filepath.Dir(path), 0o700)
|
err := os.MkdirAll(filepath.Dir(path), 0700)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
f, err := os.Create(path)
|
f, err := os.Create(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -510,59 +507,6 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
|
||||||
assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
|
assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) {
|
|
||||||
for _, ext := range []string{
|
|
||||||
".py",
|
|
||||||
".r",
|
|
||||||
".scala",
|
|
||||||
".sql",
|
|
||||||
".ipynb",
|
|
||||||
"",
|
|
||||||
} {
|
|
||||||
t.Run("case_"+ext, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
if ext != "" {
|
|
||||||
touchEmptyFile(t, filepath.Join(dir, "foo"+ext))
|
|
||||||
}
|
|
||||||
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRootPath: dir,
|
|
||||||
SyncRoot: vfs.MustNew(dir),
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Pipelines: map[string]*resources.Pipeline{
|
|
||||||
"pipeline": {
|
|
||||||
PipelineSpec: &pipelines.PipelineSpec{
|
|
||||||
Libraries: []pipelines.PipelineLibrary{
|
|
||||||
{
|
|
||||||
Notebook: &pipelines.NotebookLibrary{
|
|
||||||
Path: "./foo",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
|
|
||||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
|
||||||
|
|
||||||
if ext == "" {
|
|
||||||
assert.EqualError(t, diags.Error(), `notebook ./foo not found. Local notebook references are expected
|
|
||||||
to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]`)
|
|
||||||
} else {
|
|
||||||
assert.EqualError(t, diags.Error(), fmt.Sprintf(`notebook ./foo not found. Did you mean ./foo%s?
|
|
||||||
Local notebook references are expected to contain one of the following
|
|
||||||
file extensions: [.py, .r, .scala, .sql, .ipynb]`, ext))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPipelineFileDoesNotExistError(t *testing.T) {
|
func TestPipelineFileDoesNotExistError(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
@ -843,163 +787,3 @@ func TestTranslatePathWithComplexVariables(t *testing.T) {
|
||||||
b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
|
b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
|
|
||||||
}
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
touchNotebookFile(t, filepath.Join(dir, "my_job_notebook.py"))
|
|
||||||
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
|
|
||||||
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
|
|
||||||
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
|
|
||||||
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
|
|
||||||
|
|
||||||
enabled := true
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRootPath: dir,
|
|
||||||
SyncRoot: vfs.MustNew(dir),
|
|
||||||
Config: config.Root{
|
|
||||||
Workspace: config.Workspace{
|
|
||||||
FilePath: "/bundle",
|
|
||||||
},
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"job": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
Tasks: []jobs.Task{
|
|
||||||
{
|
|
||||||
NotebookTask: &jobs.NotebookTask{
|
|
||||||
NotebookPath: "my_job_notebook.py",
|
|
||||||
},
|
|
||||||
Libraries: []compute.Library{
|
|
||||||
{Whl: "./dist/task.whl"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
NotebookTask: &jobs.NotebookTask{
|
|
||||||
NotebookPath: "/Users/jane.doe@databricks.com/absolute_remote.py",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
NotebookTask: &jobs.NotebookTask{
|
|
||||||
NotebookPath: "my_job_notebook.py",
|
|
||||||
},
|
|
||||||
Libraries: []compute.Library{
|
|
||||||
{Requirements: "requirements.txt"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SparkPythonTask: &jobs.SparkPythonTask{
|
|
||||||
PythonFile: "my_python_file.py",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SparkJarTask: &jobs.SparkJarTask{
|
|
||||||
MainClassName: "HelloWorld",
|
|
||||||
},
|
|
||||||
Libraries: []compute.Library{
|
|
||||||
{Jar: "./dist/task.jar"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
SparkJarTask: &jobs.SparkJarTask{
|
|
||||||
MainClassName: "HelloWorldRemote",
|
|
||||||
},
|
|
||||||
Libraries: []compute.Library{
|
|
||||||
{Jar: "dbfs:/bundle/dist/task_remote.jar"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Pipelines: map[string]*resources.Pipeline{
|
|
||||||
"pipeline": {
|
|
||||||
PipelineSpec: &pipelines.PipelineSpec{
|
|
||||||
Libraries: []pipelines.PipelineLibrary{
|
|
||||||
{
|
|
||||||
Notebook: &pipelines.NotebookLibrary{
|
|
||||||
Path: "my_pipeline_notebook.py",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Notebook: &pipelines.NotebookLibrary{
|
|
||||||
Path: "/Users/jane.doe@databricks.com/absolute_remote.py",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
File: &pipelines.FileLibrary{
|
|
||||||
Path: "my_python_file.py",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Presets: config.Presets{
|
|
||||||
SourceLinkedDeployment: &enabled,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
|
||||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
// updated to source path
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join(dir, "my_job_notebook"),
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join(dir, "requirements.txt"),
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join(dir, "my_python_file.py"),
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[3].SparkPythonTask.PythonFile,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join(dir, "my_pipeline_notebook"),
|
|
||||||
b.Config.Resources.Pipelines["pipeline"].Libraries[0].Notebook.Path,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join(dir, "my_python_file.py"),
|
|
||||||
b.Config.Resources.Pipelines["pipeline"].Libraries[2].File.Path,
|
|
||||||
)
|
|
||||||
|
|
||||||
// left as is
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join("dist", "task.whl"),
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/Users/jane.doe@databricks.com/absolute_remote.py",
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[1].NotebookTask.NotebookPath,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
filepath.Join("dist", "task.jar"),
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[4].Libraries[0].Jar,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"dbfs:/bundle/dist/task_remote.jar",
|
|
||||||
b.Config.Resources.Jobs["job"].Tasks[5].Libraries[0].Jar,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/Users/jane.doe@databricks.com/absolute_remote.py",
|
|
||||||
b.Config.Resources.Pipelines["pipeline"].Libraries[1].Notebook.Path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
|
@ -15,7 +15,8 @@ func VerifyCliVersion() bundle.Mutator {
|
||||||
return &verifyCliVersion{}
|
return &verifyCliVersion{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type verifyCliVersion struct{}
|
type verifyCliVersion struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
// No constraints specified, skip the check.
|
// No constraints specified, skip the check.
|
||||||
|
|
|
@ -1,9 +1,7 @@
|
||||||
package config
|
package config
|
||||||
|
|
||||||
const (
|
const Paused = "PAUSED"
|
||||||
Paused = "PAUSED"
|
const Unpaused = "UNPAUSED"
|
||||||
Unpaused = "UNPAUSED"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Presets struct {
|
type Presets struct {
|
||||||
// NamePrefix to prepend to all resource names.
|
// NamePrefix to prepend to all resource names.
|
||||||
|
@ -19,11 +17,6 @@ type Presets struct {
|
||||||
// JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs.
|
// JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs.
|
||||||
JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"`
|
JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"`
|
||||||
|
|
||||||
// SourceLinkedDeployment indicates whether source-linked deployment is enabled. Works only in Databricks Workspace
|
|
||||||
// When set to true, resources created during deployment will point to source files in the workspace instead of their workspace copies.
|
|
||||||
// File synchronization to ${workspace.file_path} is skipped.
|
|
||||||
SourceLinkedDeployment *bool `json:"source_linked_deployment,omitempty"`
|
|
||||||
|
|
||||||
// Tags to add to all resources.
|
// Tags to add to all resources.
|
||||||
Tags map[string]string `json:"tags,omitempty"`
|
Tags map[string]string `json:"tags,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,7 +20,6 @@ type Resources struct {
|
||||||
RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"`
|
RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"`
|
||||||
QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"`
|
QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"`
|
||||||
Schemas map[string]*resources.Schema `json:"schemas,omitempty"`
|
Schemas map[string]*resources.Schema `json:"schemas,omitempty"`
|
||||||
Volumes map[string]*resources.Volume `json:"volumes,omitempty"`
|
|
||||||
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
|
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
|
||||||
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
|
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
|
||||||
}
|
}
|
||||||
|
@ -42,9 +41,6 @@ type ConfigResource interface {
|
||||||
|
|
||||||
// InitializeURL initializes the URL field of the resource.
|
// InitializeURL initializes the URL field of the resource.
|
||||||
InitializeURL(baseURL url.URL)
|
InitializeURL(baseURL url.URL)
|
||||||
|
|
||||||
// IsNil returns true if the resource is nil, for example, when it was removed from the bundle.
|
|
||||||
IsNil() bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResourceGroup represents a group of resources of the same type.
|
// ResourceGroup represents a group of resources of the same type.
|
||||||
|
@ -61,9 +57,6 @@ func collectResourceMap[T ConfigResource](
|
||||||
) ResourceGroup {
|
) ResourceGroup {
|
||||||
resources := make(map[string]ConfigResource)
|
resources := make(map[string]ConfigResource)
|
||||||
for key, resource := range input {
|
for key, resource := range input {
|
||||||
if resource.IsNil() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resources[key] = resource
|
resources[key] = resource
|
||||||
}
|
}
|
||||||
return ResourceGroup{
|
return ResourceGroup{
|
||||||
|
@ -86,7 +79,6 @@ func (r *Resources) AllResources() []ResourceGroup {
|
||||||
collectResourceMap(descriptions["schemas"], r.Schemas),
|
collectResourceMap(descriptions["schemas"], r.Schemas),
|
||||||
collectResourceMap(descriptions["clusters"], r.Clusters),
|
collectResourceMap(descriptions["clusters"], r.Clusters),
|
||||||
collectResourceMap(descriptions["dashboards"], r.Dashboards),
|
collectResourceMap(descriptions["dashboards"], r.Dashboards),
|
||||||
collectResourceMap(descriptions["volumes"], r.Volumes),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -191,11 +183,5 @@ func SupportedResources() map[string]ResourceDescription {
|
||||||
SingularTitle: "Dashboard",
|
SingularTitle: "Dashboard",
|
||||||
PluralTitle: "Dashboards",
|
PluralTitle: "Dashboards",
|
||||||
},
|
},
|
||||||
"volumes": {
|
|
||||||
SingularName: "volume",
|
|
||||||
PluralName: "volumes",
|
|
||||||
SingularTitle: "Volume",
|
|
||||||
PluralTitle: "Volumes",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,3 @@ func (s *Cluster) GetName() string {
|
||||||
func (s *Cluster) GetURL() string {
|
func (s *Cluster) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Cluster) IsNil() bool {
|
|
||||||
return s.ClusterSpec == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -79,7 +79,3 @@ func (r *Dashboard) GetName() string {
|
||||||
func (r *Dashboard) GetURL() string {
|
func (r *Dashboard) GetURL() string {
|
||||||
return r.URL
|
return r.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Dashboard) IsNil() bool {
|
|
||||||
return r.Dashboard == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -63,7 +63,3 @@ func (j *Job) GetName() string {
|
||||||
func (j *Job) GetURL() string {
|
func (j *Job) GetURL() string {
|
||||||
return j.URL
|
return j.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *Job) IsNil() bool {
|
|
||||||
return j.JobSettings == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -58,7 +58,3 @@ func (s *MlflowExperiment) GetName() string {
|
||||||
func (s *MlflowExperiment) GetURL() string {
|
func (s *MlflowExperiment) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MlflowExperiment) IsNil() bool {
|
|
||||||
return s.Experiment == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -58,7 +58,3 @@ func (s *MlflowModel) GetName() string {
|
||||||
func (s *MlflowModel) GetURL() string {
|
func (s *MlflowModel) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MlflowModel) IsNil() bool {
|
|
||||||
return s.Model == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -66,7 +66,3 @@ func (s *ModelServingEndpoint) GetName() string {
|
||||||
func (s *ModelServingEndpoint) GetURL() string {
|
func (s *ModelServingEndpoint) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ModelServingEndpoint) IsNil() bool {
|
|
||||||
return s.CreateServingEndpoint == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -58,7 +58,3 @@ func (p *Pipeline) GetName() string {
|
||||||
func (s *Pipeline) GetURL() string {
|
func (s *Pipeline) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Pipeline) IsNil() bool {
|
|
||||||
return s.PipelineSpec == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -62,7 +62,3 @@ func (s *QualityMonitor) GetName() string {
|
||||||
func (s *QualityMonitor) GetURL() string {
|
func (s *QualityMonitor) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *QualityMonitor) IsNil() bool {
|
|
||||||
return s.CreateMonitor == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -68,7 +68,3 @@ func (s *RegisteredModel) GetName() string {
|
||||||
func (s *RegisteredModel) GetURL() string {
|
func (s *RegisteredModel) GetURL() string {
|
||||||
return s.URL
|
return s.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *RegisteredModel) IsNil() bool {
|
|
||||||
return s.CreateRegisteredModelRequest == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -56,7 +56,3 @@ func (s *Schema) UnmarshalJSON(b []byte) error {
|
||||||
func (s Schema) MarshalJSON() ([]byte, error) {
|
func (s Schema) MarshalJSON() ([]byte, error) {
|
||||||
return marshal.Marshal(s)
|
return marshal.Marshal(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Schema) IsNil() bool {
|
|
||||||
return s.CreateSchema == nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,62 +0,0 @@
|
||||||
package resources
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
"github.com/databricks/databricks-sdk-go/marshal"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Volume struct {
|
|
||||||
// List of grants to apply on this volume.
|
|
||||||
Grants []Grant `json:"grants,omitempty"`
|
|
||||||
|
|
||||||
// Full name of the volume (catalog_name.schema_name.volume_name). This value is read from
|
|
||||||
// the terraform state after deployment succeeds.
|
|
||||||
ID string `json:"id,omitempty" bundle:"readonly"`
|
|
||||||
|
|
||||||
*catalog.CreateVolumeRequestContent
|
|
||||||
|
|
||||||
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
|
|
||||||
URL string `json:"url,omitempty" bundle:"internal"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) UnmarshalJSON(b []byte) error {
|
|
||||||
return marshal.Unmarshal(b, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v Volume) MarshalJSON() ([]byte, error) {
|
|
||||||
return marshal.Marshal(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
|
|
||||||
return false, fmt.Errorf("volume.Exists() is not supported")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) TerraformResourceName() string {
|
|
||||||
return "databricks_volume"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) InitializeURL(baseURL url.URL) {
|
|
||||||
if v.ID == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
baseURL.Path = fmt.Sprintf("explore/data/volumes/%s", strings.ReplaceAll(v.ID, ".", "/"))
|
|
||||||
v.URL = baseURL.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) GetURL() string {
|
|
||||||
return v.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) GetName() string {
|
|
||||||
return v.Name
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *Volume) IsNil() bool {
|
|
||||||
return v.CreateVolumeRequestContent == nil
|
|
||||||
}
|
|
|
@ -49,8 +49,7 @@ func TestCustomMarshallerIsImplemented(t *testing.T) {
|
||||||
// Eg: resource.Job implements MarshalJSON
|
// Eg: resource.Job implements MarshalJSON
|
||||||
v := reflect.Zero(vt.Elem()).Interface()
|
v := reflect.Zero(vt.Elem()).Interface()
|
||||||
assert.NotPanics(t, func() {
|
assert.NotPanics(t, func() {
|
||||||
_, err := json.Marshal(v)
|
json.Marshal(v)
|
||||||
assert.NoError(t, err)
|
|
||||||
}, "Resource %s does not have a custom marshaller", field.Name)
|
}, "Resource %s does not have a custom marshaller", field.Name)
|
||||||
|
|
||||||
// Unmarshalling a *resourceStruct will panic if the resource does not have a custom unmarshaller
|
// Unmarshalling a *resourceStruct will panic if the resource does not have a custom unmarshaller
|
||||||
|
@ -59,8 +58,7 @@ func TestCustomMarshallerIsImplemented(t *testing.T) {
|
||||||
// Eg: *resource.Job implements UnmarshalJSON
|
// Eg: *resource.Job implements UnmarshalJSON
|
||||||
v = reflect.New(vt.Elem()).Interface()
|
v = reflect.New(vt.Elem()).Interface()
|
||||||
assert.NotPanics(t, func() {
|
assert.NotPanics(t, func() {
|
||||||
err := json.Unmarshal([]byte("{}"), v)
|
json.Unmarshal([]byte("{}"), v)
|
||||||
assert.NoError(t, err)
|
|
||||||
}, "Resource %s does not have a custom unmarshaller", field.Name)
|
}, "Resource %s does not have a custom unmarshaller", field.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -100,7 +100,7 @@ func TestRootMergeTargetOverridesWithMode(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
require.NoError(t, root.initializeDynamicValue())
|
root.initializeDynamicValue()
|
||||||
require.NoError(t, root.MergeTargetOverrides("development"))
|
require.NoError(t, root.MergeTargetOverrides("development"))
|
||||||
assert.Equal(t, Development, root.Bundle.Mode)
|
assert.Equal(t, Development, root.Bundle.Mode)
|
||||||
}
|
}
|
||||||
|
@ -133,7 +133,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
|
||||||
"complex": {
|
"complex": {
|
||||||
Type: variable.VariableTypeComplex,
|
Type: variable.VariableTypeComplex,
|
||||||
Description: "complex var",
|
Description: "complex var",
|
||||||
Default: map[string]any{
|
Default: map[string]interface{}{
|
||||||
"key": "value",
|
"key": "value",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -148,7 +148,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
|
||||||
"complex": {
|
"complex": {
|
||||||
Type: "wrong",
|
Type: "wrong",
|
||||||
Description: "wrong",
|
Description: "wrong",
|
||||||
Default: map[string]any{
|
Default: map[string]interface{}{
|
||||||
"key1": "value1",
|
"key1": "value1",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -156,7 +156,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
require.NoError(t, root.initializeDynamicValue())
|
root.initializeDynamicValue()
|
||||||
require.NoError(t, root.MergeTargetOverrides("development"))
|
require.NoError(t, root.MergeTargetOverrides("development"))
|
||||||
assert.Equal(t, "bar", root.Variables["foo"].Default)
|
assert.Equal(t, "bar", root.Variables["foo"].Default)
|
||||||
assert.Equal(t, "foo var", root.Variables["foo"].Description)
|
assert.Equal(t, "foo var", root.Variables["foo"].Description)
|
||||||
|
@ -164,10 +164,11 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
|
||||||
assert.Equal(t, "foo2", root.Variables["foo2"].Default)
|
assert.Equal(t, "foo2", root.Variables["foo2"].Default)
|
||||||
assert.Equal(t, "foo2 var", root.Variables["foo2"].Description)
|
assert.Equal(t, "foo2 var", root.Variables["foo2"].Description)
|
||||||
|
|
||||||
assert.Equal(t, map[string]any{
|
assert.Equal(t, map[string]interface{}{
|
||||||
"key1": "value1",
|
"key1": "value1",
|
||||||
}, root.Variables["complex"].Default)
|
}, root.Variables["complex"].Default)
|
||||||
assert.Equal(t, "complex var", root.Variables["complex"].Description)
|
assert.Equal(t, "complex var", root.Variables["complex"].Description)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIsFullVariableOverrideDef(t *testing.T) {
|
func TestIsFullVariableOverrideDef(t *testing.T) {
|
||||||
|
@ -251,4 +252,5 @@ func TestIsFullVariableOverrideDef(t *testing.T) {
|
||||||
for i, tc := range testCases {
|
for i, tc := range testCases {
|
||||||
assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i)
|
assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,19 +13,14 @@ func FilesToSync() bundle.ReadOnlyMutator {
|
||||||
return &filesToSync{}
|
return &filesToSync{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type filesToSync struct{}
|
type filesToSync struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (v *filesToSync) Name() string {
|
func (v *filesToSync) Name() string {
|
||||||
return "validate:files_to_sync"
|
return "validate:files_to_sync"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||||
// The user may be intentional about not synchronizing any files.
|
|
||||||
// In this case, we should not show any warnings.
|
|
||||||
if len(rb.Config().Sync.Paths) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
sync, err := files.GetSync(ctx, rb)
|
sync, err := files.GetSync(ctx, rb)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diag.FromErr(err)
|
return diag.FromErr(err)
|
||||||
|
@ -36,7 +31,6 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.
|
||||||
return diag.FromErr(err)
|
return diag.FromErr(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If there are files to sync, we don't need to show any warnings.
|
|
||||||
if len(fl) != 0 {
|
if len(fl) != 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,107 +0,0 @@
|
||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
"github.com/databricks/cli/internal/testutil"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
|
||||||
"github.com/databricks/cli/libs/vfs"
|
|
||||||
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/iam"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/workspace"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesToSync_NoPaths(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Sync: config.Sync{
|
|
||||||
Paths: []string{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
rb := bundle.ReadOnly(b)
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
}
|
|
||||||
|
|
||||||
func setupBundleForFilesToSyncTest(t *testing.T) *bundle.Bundle {
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
testutil.Touch(t, dir, "file1")
|
|
||||||
testutil.Touch(t, dir, "file2")
|
|
||||||
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
BundleRootPath: dir,
|
|
||||||
BundleRoot: vfs.MustNew(dir),
|
|
||||||
SyncRootPath: dir,
|
|
||||||
SyncRoot: vfs.MustNew(dir),
|
|
||||||
WorktreeRoot: vfs.MustNew(dir),
|
|
||||||
Config: config.Root{
|
|
||||||
Bundle: config.Bundle{
|
|
||||||
Target: "default",
|
|
||||||
},
|
|
||||||
Workspace: config.Workspace{
|
|
||||||
FilePath: "/this/doesnt/matter",
|
|
||||||
CurrentUser: &config.User{
|
|
||||||
User: &iam.User{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Sync: config.Sync{
|
|
||||||
// Paths are relative to [SyncRootPath].
|
|
||||||
Paths: []string{"."},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
m.WorkspaceClient.Config = &sdkconfig.Config{
|
|
||||||
Host: "https://foo.com",
|
|
||||||
}
|
|
||||||
|
|
||||||
// The initialization logic in [sync.New] performs a check on the destination path.
|
|
||||||
// Removing this check at initialization time is tbd...
|
|
||||||
m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(mock.Anything, "/this/doesnt/matter").Return(&workspace.ObjectInfo{
|
|
||||||
ObjectType: workspace.ObjectTypeDirectory,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesToSync_EverythingIgnored(t *testing.T) {
|
|
||||||
b := setupBundleForFilesToSyncTest(t)
|
|
||||||
|
|
||||||
// Ignore all files.
|
|
||||||
testutil.WriteFile(t, filepath.Join(b.BundleRootPath, ".gitignore"), "*\n.*\n")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
rb := bundle.ReadOnly(b)
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
|
||||||
require.Equal(t, 1, len(diags))
|
|
||||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
|
||||||
assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesToSync_EverythingExcluded(t *testing.T) {
|
|
||||||
b := setupBundleForFilesToSyncTest(t)
|
|
||||||
|
|
||||||
// Exclude all files.
|
|
||||||
b.Config.Sync.Exclude = []string{"*"}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
rb := bundle.ReadOnly(b)
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
|
||||||
require.Equal(t, 1, len(diags))
|
|
||||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
|
||||||
assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary)
|
|
||||||
}
|
|
|
@ -15,7 +15,8 @@ import (
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
type folderPermissions struct{}
|
type folderPermissions struct {
|
||||||
|
}
|
||||||
|
|
||||||
// Apply implements bundle.ReadOnlyMutator.
|
// Apply implements bundle.ReadOnlyMutator.
|
||||||
func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) diag.Diagnostics {
|
func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||||
|
|
|
@ -13,7 +13,8 @@ func JobClusterKeyDefined() bundle.ReadOnlyMutator {
|
||||||
return &jobClusterKeyDefined{}
|
return &jobClusterKeyDefined{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type jobClusterKeyDefined struct{}
|
type jobClusterKeyDefined struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (v *jobClusterKeyDefined) Name() string {
|
func (v *jobClusterKeyDefined) Name() string {
|
||||||
return "validate:job_cluster_key_defined"
|
return "validate:job_cluster_key_defined"
|
||||||
|
|
|
@ -17,7 +17,8 @@ func JobTaskClusterSpec() bundle.ReadOnlyMutator {
|
||||||
return &jobTaskClusterSpec{}
|
return &jobTaskClusterSpec{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type jobTaskClusterSpec struct{}
|
type jobTaskClusterSpec struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (v *jobTaskClusterSpec) Name() string {
|
func (v *jobTaskClusterSpec) Name() string {
|
||||||
return "validate:job_task_cluster_spec"
|
return "validate:job_task_cluster_spec"
|
||||||
|
|
|
@ -1,137 +0,0 @@
|
||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/cli/libs/dyn/convert"
|
|
||||||
"github.com/databricks/cli/libs/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Validates that any single node clusters defined in the bundle are correctly configured.
|
|
||||||
func SingleNodeCluster() bundle.ReadOnlyMutator {
|
|
||||||
return &singleNodeCluster{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type singleNodeCluster struct{}
|
|
||||||
|
|
||||||
func (m *singleNodeCluster) Name() string {
|
|
||||||
return "validate:SingleNodeCluster"
|
|
||||||
}
|
|
||||||
|
|
||||||
const singleNodeWarningDetail = `num_workers should be 0 only for single-node clusters. To create a
|
|
||||||
valid single node cluster please ensure that the following properties
|
|
||||||
are correctly set in the cluster specification:
|
|
||||||
|
|
||||||
spark_conf:
|
|
||||||
spark.databricks.cluster.profile: singleNode
|
|
||||||
spark.master: local[*]
|
|
||||||
|
|
||||||
custom_tags:
|
|
||||||
ResourceClass: SingleNode
|
|
||||||
`
|
|
||||||
|
|
||||||
const singleNodeWarningSummary = `Single node cluster is not correctly configured`
|
|
||||||
|
|
||||||
func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool {
|
|
||||||
// Check if the user has explicitly set the num_workers to 0. Skip the warning
|
|
||||||
// if that's not the case.
|
|
||||||
numWorkers, ok := v.Get("num_workers").AsInt()
|
|
||||||
if !ok || numWorkers > 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convenient type that contains the common fields from compute.ClusterSpec and
|
|
||||||
// pipelines.PipelineCluster that we are interested in.
|
|
||||||
type ClusterConf struct {
|
|
||||||
SparkConf map[string]string `json:"spark_conf"`
|
|
||||||
CustomTags map[string]string `json:"custom_tags"`
|
|
||||||
PolicyId string `json:"policy_id"`
|
|
||||||
}
|
|
||||||
|
|
||||||
conf := &ClusterConf{}
|
|
||||||
err := convert.ToTyped(conf, v)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the policy id is set, we don't want to show the warning. This is because
|
|
||||||
// the user might have configured `spark_conf` and `custom_tags` correctly
|
|
||||||
// in their cluster policy.
|
|
||||||
if conf.PolicyId != "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
profile, ok := conf.SparkConf["spark.databricks.cluster.profile"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile not found in single-node cluster spec")
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if profile != "singleNode" {
|
|
||||||
log.Debugf(ctx, "spark_conf spark.databricks.cluster.profile is not singleNode in single-node cluster spec: %s", profile)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
master, ok := conf.SparkConf["spark.master"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf(ctx, "spark_conf spark.master not found in single-node cluster spec")
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(master, "local") {
|
|
||||||
log.Debugf(ctx, "spark_conf spark.master does not start with local in single-node cluster spec: %s", master)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
resourceClass, ok := conf.CustomTags["ResourceClass"]
|
|
||||||
if !ok {
|
|
||||||
log.Debugf(ctx, "custom_tag ResourceClass not found in single-node cluster spec")
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if resourceClass != "SingleNode" {
|
|
||||||
log.Debugf(ctx, "custom_tag ResourceClass is not SingleNode in single-node cluster spec: %s", resourceClass)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
|
||||||
diags := diag.Diagnostics{}
|
|
||||||
|
|
||||||
patterns := []dyn.Pattern{
|
|
||||||
// Interactive clusters
|
|
||||||
dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()),
|
|
||||||
// Job clusters
|
|
||||||
dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("job_clusters"), dyn.AnyIndex(), dyn.Key("new_cluster")),
|
|
||||||
// Job task clusters
|
|
||||||
dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("new_cluster")),
|
|
||||||
// Job for each task clusters
|
|
||||||
dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("task"), dyn.Key("new_cluster")),
|
|
||||||
// Pipeline clusters
|
|
||||||
dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("clusters"), dyn.AnyIndex()),
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range patterns {
|
|
||||||
_, err := dyn.MapByPattern(rb.Config().Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
|
||||||
warning := diag.Diagnostic{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: v.Locations(),
|
|
||||||
Paths: []dyn.Path{p},
|
|
||||||
}
|
|
||||||
|
|
||||||
if showSingleNodeClusterWarning(ctx, v) {
|
|
||||||
diags = append(diags, warning)
|
|
||||||
}
|
|
||||||
return v, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf(ctx, "Error while applying single node cluster validation: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return diags
|
|
||||||
}
|
|
|
@ -1,565 +0,0 @@
|
||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func failCases() []struct {
|
|
||||||
name string
|
|
||||||
sparkConf map[string]string
|
|
||||||
customTags map[string]string
|
|
||||||
} {
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
sparkConf map[string]string
|
|
||||||
customTags map[string]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "no tags or conf",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no tags",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no conf",
|
|
||||||
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid spark cluster profile",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "invalid",
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid spark.master",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
"spark.master": "invalid",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid tags",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"ResourceClass": "invalid"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "missing ResourceClass tag",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"what": "ever"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "missing spark.master",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "missing spark.databricks.cluster.profile",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range failCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Clusters: map[string]*resources.Cluster{
|
|
||||||
"foo": {
|
|
||||||
ClusterSpec: &compute.ClusterSpec{
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "resources.clusters.foo", []dyn.Location{{File: "a.yml", Line: 1, Column: 1}})
|
|
||||||
|
|
||||||
// We can't set num_workers to 0 explicitly in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0))
|
|
||||||
})
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Equal(t, diag.Diagnostics{
|
|
||||||
{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}},
|
|
||||||
Paths: []dyn.Path{dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))},
|
|
||||||
},
|
|
||||||
}, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range failCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
JobClusters: []jobs.JobCluster{
|
|
||||||
{
|
|
||||||
NewCluster: compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []dyn.Location{{File: "b.yml", Line: 1, Column: 1}})
|
|
||||||
|
|
||||||
// We can't set num_workers to 0 explicitly in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Equal(t, diag.Diagnostics{
|
|
||||||
{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}},
|
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.job_clusters[0].new_cluster")},
|
|
||||||
},
|
|
||||||
}, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range failCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
Tasks: []jobs.Task{
|
|
||||||
{
|
|
||||||
NewCluster: &compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []dyn.Location{{File: "c.yml", Line: 1, Column: 1}})
|
|
||||||
|
|
||||||
// We can't set num_workers to 0 explicitly in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Equal(t, diag.Diagnostics{
|
|
||||||
{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}},
|
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].new_cluster")},
|
|
||||||
},
|
|
||||||
}, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range failCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Pipelines: map[string]*resources.Pipeline{
|
|
||||||
"foo": {
|
|
||||||
PipelineSpec: &pipelines.PipelineSpec{
|
|
||||||
Clusters: []pipelines.PipelineCluster{
|
|
||||||
{
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []dyn.Location{{File: "d.yml", Line: 1, Column: 1}})
|
|
||||||
|
|
||||||
// We can't set num_workers to 0 explicitly in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Equal(t, diag.Diagnostics{
|
|
||||||
{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}},
|
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.foo.clusters[0]")},
|
|
||||||
},
|
|
||||||
}, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range failCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
Tasks: []jobs.Task{
|
|
||||||
{
|
|
||||||
ForEachTask: &jobs.ForEachTask{
|
|
||||||
Task: jobs.Task{
|
|
||||||
NewCluster: &compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster", []dyn.Location{{File: "e.yml", Line: 1, Column: 1}})
|
|
||||||
|
|
||||||
// We can't set num_workers to 0 explicitly in the typed configuration.
|
|
||||||
// Do it on the dyn.Value directly.
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Equal(t, diag.Diagnostics{
|
|
||||||
{
|
|
||||||
Severity: diag.Warning,
|
|
||||||
Summary: singleNodeWarningSummary,
|
|
||||||
Detail: singleNodeWarningDetail,
|
|
||||||
Locations: []dyn.Location{{File: "e.yml", Line: 1, Column: 1}},
|
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].for_each_task.task.new_cluster")},
|
|
||||||
},
|
|
||||||
}, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func passCases() []struct {
|
|
||||||
name string
|
|
||||||
numWorkers *int
|
|
||||||
sparkConf map[string]string
|
|
||||||
customTags map[string]string
|
|
||||||
policyId string
|
|
||||||
} {
|
|
||||||
zero := 0
|
|
||||||
one := 1
|
|
||||||
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
numWorkers *int
|
|
||||||
sparkConf map[string]string
|
|
||||||
customTags map[string]string
|
|
||||||
policyId string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "single node cluster",
|
|
||||||
sparkConf: map[string]string{
|
|
||||||
"spark.databricks.cluster.profile": "singleNode",
|
|
||||||
"spark.master": "local[*]",
|
|
||||||
},
|
|
||||||
customTags: map[string]string{
|
|
||||||
"ResourceClass": "SingleNode",
|
|
||||||
},
|
|
||||||
numWorkers: &zero,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "num workers is not zero",
|
|
||||||
numWorkers: &one,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "num workers is not set",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "policy id is not empty",
|
|
||||||
policyId: "policy-abc",
|
|
||||||
numWorkers: &zero,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range passCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Clusters: map[string]*resources.Cluster{
|
|
||||||
"foo": {
|
|
||||||
ClusterSpec: &compute.ClusterSpec{
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
PolicyId: tc.policyId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.numWorkers != nil {
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(*tc.numWorkers))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range passCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
JobClusters: []jobs.JobCluster{
|
|
||||||
{
|
|
||||||
NewCluster: compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
PolicyId: tc.policyId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.numWorkers != nil {
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range passCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
Tasks: []jobs.Task{
|
|
||||||
{
|
|
||||||
NewCluster: &compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
PolicyId: tc.policyId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.numWorkers != nil {
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range passCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Pipelines: map[string]*resources.Pipeline{
|
|
||||||
"foo": {
|
|
||||||
PipelineSpec: &pipelines.PipelineSpec{
|
|
||||||
Clusters: []pipelines.PipelineCluster{
|
|
||||||
{
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
PolicyId: tc.policyId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.numWorkers != nil {
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(*tc.numWorkers))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
for _, tc := range passCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
Config: config.Root{
|
|
||||||
Resources: config.Resources{
|
|
||||||
Jobs: map[string]*resources.Job{
|
|
||||||
"foo": {
|
|
||||||
JobSettings: &jobs.JobSettings{
|
|
||||||
Tasks: []jobs.Task{
|
|
||||||
{
|
|
||||||
ForEachTask: &jobs.ForEachTask{
|
|
||||||
Task: jobs.Task{
|
|
||||||
NewCluster: &compute.ClusterSpec{
|
|
||||||
ClusterName: "my_cluster",
|
|
||||||
SparkConf: tc.sparkConf,
|
|
||||||
CustomTags: tc.customTags,
|
|
||||||
PolicyId: tc.policyId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if tc.numWorkers != nil {
|
|
||||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
|
||||||
return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(*tc.numWorkers))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -8,7 +8,8 @@ import (
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
)
|
)
|
||||||
|
|
||||||
type validate struct{}
|
type validate struct {
|
||||||
|
}
|
||||||
|
|
||||||
type location struct {
|
type location struct {
|
||||||
path string
|
path string
|
||||||
|
@ -35,7 +36,6 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
|
||||||
ValidateSyncPatterns(),
|
ValidateSyncPatterns(),
|
||||||
JobTaskClusterSpec(),
|
JobTaskClusterSpec(),
|
||||||
ValidateFolderPermissions(),
|
ValidateFolderPermissions(),
|
||||||
SingleNodeCluster(),
|
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,7 +17,8 @@ func ValidateSyncPatterns() bundle.ReadOnlyMutator {
|
||||||
return &validateSyncPatterns{}
|
return &validateSyncPatterns{}
|
||||||
}
|
}
|
||||||
|
|
||||||
type validateSyncPatterns struct{}
|
type validateSyncPatterns struct {
|
||||||
|
}
|
||||||
|
|
||||||
func (v *validateSyncPatterns) Name() string {
|
func (v *validateSyncPatterns) Name() string {
|
||||||
return "validate:validate_sync_patterns"
|
return "validate:validate_sync_patterns"
|
||||||
|
|
|
@ -1,8 +1,11 @@
|
||||||
|
// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
|
||||||
|
|
||||||
package variable
|
package variable
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
"github.com/databricks/databricks-sdk-go"
|
||||||
)
|
)
|
||||||
|
@ -22,8 +25,6 @@ type Lookup struct {
|
||||||
|
|
||||||
Metastore string `json:"metastore,omitempty"`
|
Metastore string `json:"metastore,omitempty"`
|
||||||
|
|
||||||
NotificationDestination string `json:"notification_destination,omitempty"`
|
|
||||||
|
|
||||||
Pipeline string `json:"pipeline,omitempty"`
|
Pipeline string `json:"pipeline,omitempty"`
|
||||||
|
|
||||||
Query string `json:"query,omitempty"`
|
Query string `json:"query,omitempty"`
|
||||||
|
@ -33,78 +34,323 @@ type Lookup struct {
|
||||||
Warehouse string `json:"warehouse,omitempty"`
|
Warehouse string `json:"warehouse,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type resolver interface {
|
func LookupFromMap(m map[string]any) *Lookup {
|
||||||
// Resolve resolves the underlying entity's ID.
|
l := &Lookup{}
|
||||||
Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error)
|
if v, ok := m["alert"]; ok {
|
||||||
|
l.Alert = v.(string)
|
||||||
// String returns a human-readable representation of the resolver.
|
|
||||||
String() string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *Lookup) constructResolver() (resolver, error) {
|
|
||||||
var resolvers []resolver
|
|
||||||
|
|
||||||
if l.Alert != "" {
|
|
||||||
resolvers = append(resolvers, resolveAlert{name: l.Alert})
|
|
||||||
}
|
}
|
||||||
if l.ClusterPolicy != "" {
|
if v, ok := m["cluster_policy"]; ok {
|
||||||
resolvers = append(resolvers, resolveClusterPolicy{name: l.ClusterPolicy})
|
l.ClusterPolicy = v.(string)
|
||||||
}
|
}
|
||||||
if l.Cluster != "" {
|
if v, ok := m["cluster"]; ok {
|
||||||
resolvers = append(resolvers, resolveCluster{name: l.Cluster})
|
l.Cluster = v.(string)
|
||||||
}
|
}
|
||||||
if l.Dashboard != "" {
|
if v, ok := m["dashboard"]; ok {
|
||||||
resolvers = append(resolvers, resolveDashboard{name: l.Dashboard})
|
l.Dashboard = v.(string)
|
||||||
}
|
}
|
||||||
if l.InstancePool != "" {
|
if v, ok := m["instance_pool"]; ok {
|
||||||
resolvers = append(resolvers, resolveInstancePool{name: l.InstancePool})
|
l.InstancePool = v.(string)
|
||||||
}
|
}
|
||||||
if l.Job != "" {
|
if v, ok := m["job"]; ok {
|
||||||
resolvers = append(resolvers, resolveJob{name: l.Job})
|
l.Job = v.(string)
|
||||||
}
|
}
|
||||||
if l.Metastore != "" {
|
if v, ok := m["metastore"]; ok {
|
||||||
resolvers = append(resolvers, resolveMetastore{name: l.Metastore})
|
l.Metastore = v.(string)
|
||||||
}
|
}
|
||||||
if l.NotificationDestination != "" {
|
if v, ok := m["pipeline"]; ok {
|
||||||
resolvers = append(resolvers, resolveNotificationDestination{name: l.NotificationDestination})
|
l.Pipeline = v.(string)
|
||||||
}
|
}
|
||||||
if l.Pipeline != "" {
|
if v, ok := m["query"]; ok {
|
||||||
resolvers = append(resolvers, resolvePipeline{name: l.Pipeline})
|
l.Query = v.(string)
|
||||||
}
|
}
|
||||||
if l.Query != "" {
|
if v, ok := m["service_principal"]; ok {
|
||||||
resolvers = append(resolvers, resolveQuery{name: l.Query})
|
l.ServicePrincipal = v.(string)
|
||||||
}
|
}
|
||||||
if l.ServicePrincipal != "" {
|
if v, ok := m["warehouse"]; ok {
|
||||||
resolvers = append(resolvers, resolveServicePrincipal{name: l.ServicePrincipal})
|
l.Warehouse = v.(string)
|
||||||
}
|
|
||||||
if l.Warehouse != "" {
|
|
||||||
resolvers = append(resolvers, resolveWarehouse{name: l.Warehouse})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
switch len(resolvers) {
|
return l
|
||||||
case 0:
|
|
||||||
return nil, fmt.Errorf("no valid lookup fields provided")
|
|
||||||
case 1:
|
|
||||||
return resolvers[0], nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("exactly one lookup field must be provided")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
||||||
r, err := l.constructResolver()
|
if err := l.validate(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return r.Resolve(ctx, w)
|
r := allResolvers()
|
||||||
|
if l.Alert != "" {
|
||||||
|
return r.Alert(ctx, w, l.Alert)
|
||||||
|
}
|
||||||
|
if l.ClusterPolicy != "" {
|
||||||
|
return r.ClusterPolicy(ctx, w, l.ClusterPolicy)
|
||||||
|
}
|
||||||
|
if l.Cluster != "" {
|
||||||
|
return r.Cluster(ctx, w, l.Cluster)
|
||||||
|
}
|
||||||
|
if l.Dashboard != "" {
|
||||||
|
return r.Dashboard(ctx, w, l.Dashboard)
|
||||||
|
}
|
||||||
|
if l.InstancePool != "" {
|
||||||
|
return r.InstancePool(ctx, w, l.InstancePool)
|
||||||
|
}
|
||||||
|
if l.Job != "" {
|
||||||
|
return r.Job(ctx, w, l.Job)
|
||||||
|
}
|
||||||
|
if l.Metastore != "" {
|
||||||
|
return r.Metastore(ctx, w, l.Metastore)
|
||||||
|
}
|
||||||
|
if l.Pipeline != "" {
|
||||||
|
return r.Pipeline(ctx, w, l.Pipeline)
|
||||||
|
}
|
||||||
|
if l.Query != "" {
|
||||||
|
return r.Query(ctx, w, l.Query)
|
||||||
|
}
|
||||||
|
if l.ServicePrincipal != "" {
|
||||||
|
return r.ServicePrincipal(ctx, w, l.ServicePrincipal)
|
||||||
|
}
|
||||||
|
if l.Warehouse != "" {
|
||||||
|
return r.Warehouse(ctx, w, l.Warehouse)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("no valid lookup fields provided")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *Lookup) String() string {
|
func (l *Lookup) String() string {
|
||||||
r, _ := l.constructResolver()
|
if l.Alert != "" {
|
||||||
if r == nil {
|
return fmt.Sprintf("alert: %s", l.Alert)
|
||||||
return ""
|
}
|
||||||
|
if l.ClusterPolicy != "" {
|
||||||
|
return fmt.Sprintf("cluster-policy: %s", l.ClusterPolicy)
|
||||||
|
}
|
||||||
|
if l.Cluster != "" {
|
||||||
|
return fmt.Sprintf("cluster: %s", l.Cluster)
|
||||||
|
}
|
||||||
|
if l.Dashboard != "" {
|
||||||
|
return fmt.Sprintf("dashboard: %s", l.Dashboard)
|
||||||
|
}
|
||||||
|
if l.InstancePool != "" {
|
||||||
|
return fmt.Sprintf("instance-pool: %s", l.InstancePool)
|
||||||
|
}
|
||||||
|
if l.Job != "" {
|
||||||
|
return fmt.Sprintf("job: %s", l.Job)
|
||||||
|
}
|
||||||
|
if l.Metastore != "" {
|
||||||
|
return fmt.Sprintf("metastore: %s", l.Metastore)
|
||||||
|
}
|
||||||
|
if l.Pipeline != "" {
|
||||||
|
return fmt.Sprintf("pipeline: %s", l.Pipeline)
|
||||||
|
}
|
||||||
|
if l.Query != "" {
|
||||||
|
return fmt.Sprintf("query: %s", l.Query)
|
||||||
|
}
|
||||||
|
if l.ServicePrincipal != "" {
|
||||||
|
return fmt.Sprintf("service-principal: %s", l.ServicePrincipal)
|
||||||
|
}
|
||||||
|
if l.Warehouse != "" {
|
||||||
|
return fmt.Sprintf("warehouse: %s", l.Warehouse)
|
||||||
}
|
}
|
||||||
|
|
||||||
return r.String()
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Lookup) validate() error {
|
||||||
|
// Validate that only one field is set
|
||||||
|
count := 0
|
||||||
|
if l.Alert != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.ClusterPolicy != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Cluster != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Dashboard != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.InstancePool != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Job != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Metastore != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Pipeline != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Query != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.ServicePrincipal != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
if l.Warehouse != "" {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
|
||||||
|
if count != 1 {
|
||||||
|
return fmt.Errorf("exactly one lookup field must be provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(l.String(), "${var") {
|
||||||
|
return fmt.Errorf("lookup fields cannot contain variable references")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error)
|
||||||
|
type resolvers struct {
|
||||||
|
Alert resolverFunc
|
||||||
|
ClusterPolicy resolverFunc
|
||||||
|
Cluster resolverFunc
|
||||||
|
Dashboard resolverFunc
|
||||||
|
InstancePool resolverFunc
|
||||||
|
Job resolverFunc
|
||||||
|
Metastore resolverFunc
|
||||||
|
Pipeline resolverFunc
|
||||||
|
Query resolverFunc
|
||||||
|
ServicePrincipal resolverFunc
|
||||||
|
Warehouse resolverFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func allResolvers() *resolvers {
|
||||||
|
r := &resolvers{}
|
||||||
|
r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Alert"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Alerts.GetByDisplayName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.Id), nil
|
||||||
|
}
|
||||||
|
r.ClusterPolicy = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["ClusterPolicy"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.ClusterPolicies.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.PolicyId), nil
|
||||||
|
}
|
||||||
|
r.Cluster = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Cluster"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Clusters.GetByClusterName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.ClusterId), nil
|
||||||
|
}
|
||||||
|
r.Dashboard = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Dashboard"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Dashboards.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.Id), nil
|
||||||
|
}
|
||||||
|
r.InstancePool = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["InstancePool"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.InstancePools.GetByInstancePoolName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.InstancePoolId), nil
|
||||||
|
}
|
||||||
|
r.Job = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Job"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Jobs.GetBySettingsName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.JobId), nil
|
||||||
|
}
|
||||||
|
r.Metastore = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Metastore"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Metastores.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.MetastoreId), nil
|
||||||
|
}
|
||||||
|
r.Pipeline = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Pipeline"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Pipelines.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.PipelineId), nil
|
||||||
|
}
|
||||||
|
r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Query"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Queries.GetByDisplayName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.Id), nil
|
||||||
|
}
|
||||||
|
r.ServicePrincipal = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["ServicePrincipal"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.ApplicationId), nil
|
||||||
|
}
|
||||||
|
r.Warehouse = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
|
fn, ok := lookupOverrides["Warehouse"]
|
||||||
|
if ok {
|
||||||
|
return fn(ctx, w, name)
|
||||||
|
}
|
||||||
|
entity, err := w.Warehouses.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprint(entity.Id), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return r
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,18 +8,19 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
)
|
)
|
||||||
|
|
||||||
type resolveCluster struct {
|
var lookupOverrides = map[string]resolverFunc{
|
||||||
name string
|
"Cluster": resolveCluster,
|
||||||
}
|
}
|
||||||
|
|
||||||
// We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters.
|
// We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters.
|
||||||
// Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts.
|
// Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts.
|
||||||
func (l resolveCluster) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
|
||||||
result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{
|
result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{
|
||||||
FilterBy: &compute.ListClustersFilterBy{
|
FilterBy: &compute.ListClustersFilterBy{
|
||||||
ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
|
ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
@ -29,8 +30,6 @@ func (l resolveCluster) Resolve(ctx context.Context, w *databricks.WorkspaceClie
|
||||||
key := v.ClusterName
|
key := v.ClusterName
|
||||||
tmp[key] = append(tmp[key], v)
|
tmp[key] = append(tmp[key], v)
|
||||||
}
|
}
|
||||||
|
|
||||||
name := l.name
|
|
||||||
alternatives, ok := tmp[name]
|
alternatives, ok := tmp[name]
|
||||||
if !ok || len(alternatives) == 0 {
|
if !ok || len(alternatives) == 0 {
|
||||||
return "", fmt.Errorf("cluster named '%s' does not exist", name)
|
return "", fmt.Errorf("cluster named '%s' does not exist", name)
|
||||||
|
@ -40,7 +39,3 @@ func (l resolveCluster) Resolve(ctx context.Context, w *databricks.WorkspaceClie
|
||||||
}
|
}
|
||||||
return alternatives[0].ClusterId, nil
|
return alternatives[0].ClusterId, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l resolveCluster) String() string {
|
|
||||||
return fmt.Sprintf("cluster: %s", l.name)
|
|
||||||
}
|
|
|
@ -1,59 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLookup_Coverage(t *testing.T) {
|
|
||||||
var lookup Lookup
|
|
||||||
val := reflect.ValueOf(lookup)
|
|
||||||
typ := val.Type()
|
|
||||||
|
|
||||||
for i := 0; i < val.NumField(); i++ {
|
|
||||||
field := val.Field(i)
|
|
||||||
if field.Kind() != reflect.String {
|
|
||||||
t.Fatalf("Field %s is not a string", typ.Field(i).Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
fieldType := typ.Field(i)
|
|
||||||
t.Run(fieldType.Name, func(t *testing.T) {
|
|
||||||
// Use a fresh instance of the struct in each test
|
|
||||||
var lookup Lookup
|
|
||||||
|
|
||||||
// Set the field to a non-empty string
|
|
||||||
reflect.ValueOf(&lookup).Elem().Field(i).SetString("value")
|
|
||||||
|
|
||||||
// Test the [String] function
|
|
||||||
assert.NotEmpty(t, lookup.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookup_Empty(t *testing.T) {
|
|
||||||
var lookup Lookup
|
|
||||||
|
|
||||||
// Resolve returns an error when no fields are provided
|
|
||||||
_, err := lookup.Resolve(context.Background(), nil)
|
|
||||||
assert.ErrorContains(t, err, "no valid lookup fields provided")
|
|
||||||
|
|
||||||
// No string representation for an invalid lookup
|
|
||||||
assert.Empty(t, lookup.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookup_Multiple(t *testing.T) {
|
|
||||||
lookup := Lookup{
|
|
||||||
Alert: "alert",
|
|
||||||
Query: "query",
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve returns an error when multiple fields are provided
|
|
||||||
_, err := lookup.Resolve(context.Background(), nil)
|
|
||||||
assert.ErrorContains(t, err, "exactly one lookup field must be provided")
|
|
||||||
|
|
||||||
// No string representation for an invalid lookup
|
|
||||||
assert.Empty(t, lookup.String())
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type resolveAlert struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveAlert) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
|
||||||
entity, err := w.Alerts.GetByDisplayName(ctx, l.name)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprint(entity.Id), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveAlert) String() string {
|
|
||||||
return fmt.Sprintf("alert: %s", l.name)
|
|
||||||
}
|
|
|
@ -1,49 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/apierr"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/sql"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveAlert_ResolveSuccess(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockAlertsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByDisplayName(mock.Anything, "alert").
|
|
||||||
Return(&sql.ListAlertsResponseAlert{
|
|
||||||
Id: "1234",
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveAlert{name: "alert"}
|
|
||||||
result, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "1234", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveAlert_ResolveNotFound(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockAlertsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByDisplayName(mock.Anything, "alert").
|
|
||||||
Return(nil, &apierr.APIError{StatusCode: 404})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveAlert{name: "alert"}
|
|
||||||
_, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.ErrorIs(t, err, apierr.ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveAlert_String(t *testing.T) {
|
|
||||||
l := resolveAlert{name: "name"}
|
|
||||||
assert.Equal(t, "alert: name", l.String())
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type resolveClusterPolicy struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveClusterPolicy) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
|
||||||
entity, err := w.ClusterPolicies.GetByName(ctx, l.name)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprint(entity.PolicyId), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveClusterPolicy) String() string {
|
|
||||||
return fmt.Sprintf("cluster-policy: %s", l.name)
|
|
||||||
}
|
|
|
@ -1,49 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/apierr"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveClusterPolicy_ResolveSuccess(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockClusterPoliciesAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByName(mock.Anything, "policy").
|
|
||||||
Return(&compute.Policy{
|
|
||||||
PolicyId: "1234",
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveClusterPolicy{name: "policy"}
|
|
||||||
result, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "1234", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveClusterPolicy_ResolveNotFound(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockClusterPoliciesAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByName(mock.Anything, "policy").
|
|
||||||
Return(nil, &apierr.APIError{StatusCode: 404})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveClusterPolicy{name: "policy"}
|
|
||||||
_, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.ErrorIs(t, err, apierr.ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveClusterPolicy_String(t *testing.T) {
|
|
||||||
l := resolveClusterPolicy{name: "name"}
|
|
||||||
assert.Equal(t, "cluster-policy: name", l.String())
|
|
||||||
}
|
|
|
@ -1,50 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveCluster_ResolveSuccess(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockClustersAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
ListAll(mock.Anything, mock.Anything).
|
|
||||||
Return([]compute.ClusterDetails{
|
|
||||||
{ClusterId: "1234", ClusterName: "cluster1"},
|
|
||||||
{ClusterId: "2345", ClusterName: "cluster2"},
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveCluster{name: "cluster2"}
|
|
||||||
result, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "2345", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveCluster_ResolveNotFound(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockClustersAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
ListAll(mock.Anything, mock.Anything).
|
|
||||||
Return([]compute.ClusterDetails{}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveCluster{name: "cluster"}
|
|
||||||
_, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "cluster named 'cluster' does not exist")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveCluster_String(t *testing.T) {
|
|
||||||
l := resolveCluster{name: "name"}
|
|
||||||
assert.Equal(t, "cluster: name", l.String())
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type resolveDashboard struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveDashboard) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
|
||||||
entity, err := w.Dashboards.GetByName(ctx, l.name)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprint(entity.Id), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveDashboard) String() string {
|
|
||||||
return fmt.Sprintf("dashboard: %s", l.name)
|
|
||||||
}
|
|
|
@ -1,49 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/apierr"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/sql"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveDashboard_ResolveSuccess(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockDashboardsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByName(mock.Anything, "dashboard").
|
|
||||||
Return(&sql.Dashboard{
|
|
||||||
Id: "1234",
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveDashboard{name: "dashboard"}
|
|
||||||
result, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "1234", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveDashboard_ResolveNotFound(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockDashboardsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByName(mock.Anything, "dashboard").
|
|
||||||
Return(nil, &apierr.APIError{StatusCode: 404})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveDashboard{name: "dashboard"}
|
|
||||||
_, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.ErrorIs(t, err, apierr.ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveDashboard_String(t *testing.T) {
|
|
||||||
l := resolveDashboard{name: "name"}
|
|
||||||
assert.Equal(t, "dashboard: name", l.String())
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type resolveInstancePool struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveInstancePool) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
|
||||||
entity, err := w.InstancePools.GetByInstancePoolName(ctx, l.name)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprint(entity.InstancePoolId), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveInstancePool) String() string {
|
|
||||||
return fmt.Sprintf("instance-pool: %s", l.name)
|
|
||||||
}
|
|
|
@ -1,49 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go/apierr"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestResolveInstancePool_ResolveSuccess(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockInstancePoolsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByInstancePoolName(mock.Anything, "instance_pool").
|
|
||||||
Return(&compute.InstancePoolAndStats{
|
|
||||||
InstancePoolId: "5678",
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveInstancePool{name: "instance_pool"}
|
|
||||||
result, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "5678", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveInstancePool_ResolveNotFound(t *testing.T) {
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
|
||||||
|
|
||||||
api := m.GetMockInstancePoolsAPI()
|
|
||||||
api.EXPECT().
|
|
||||||
GetByInstancePoolName(mock.Anything, "instance_pool").
|
|
||||||
Return(nil, &apierr.APIError{StatusCode: 404})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
l := resolveInstancePool{name: "instance_pool"}
|
|
||||||
_, err := l.Resolve(ctx, m.WorkspaceClient)
|
|
||||||
require.ErrorIs(t, err, apierr.ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveInstancePool_String(t *testing.T) {
|
|
||||||
l := resolveInstancePool{name: "name"}
|
|
||||||
assert.Equal(t, "instance-pool: name", l.String())
|
|
||||||
}
|
|
|
@ -1,24 +0,0 @@
|
||||||
package variable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/databricks/databricks-sdk-go"
|
|
||||||
)
|
|
||||||
|
|
||||||
type resolveJob struct {
|
|
||||||
name string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveJob) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
|
|
||||||
entity, err := w.Jobs.GetBySettingsName(ctx, l.name)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprint(entity.JobId), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l resolveJob) String() string {
|
|
||||||
return fmt.Sprintf("job: %s", l.name)
|
|
||||||
}
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue