mirror of https://github.com/databricks/cli.git
merge
This commit is contained in:
commit
898b2c1cc3
|
@ -1,6 +1,10 @@
|
|||
## Changes
|
||||
<!-- Summary of your changes that are easy to understand -->
|
||||
<!-- Brief summary of your changes that is easy to understand -->
|
||||
|
||||
## Why
|
||||
<!-- Why are these changes needed? Provide the context that the reviewer might be missing.
|
||||
For example, were there any decisions behind the change that are not reflected in the code itself? -->
|
||||
|
||||
## Tests
|
||||
<!-- How is this tested? -->
|
||||
<!-- How have you tested the changes? -->
|
||||
|
||||
|
|
|
@ -53,7 +53,7 @@ jobs:
|
|||
go-version-file: go.mod
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
|
||||
uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
|
@ -95,7 +95,7 @@ jobs:
|
|||
# Exit with status code 1 if there are differences (i.e. unformatted files)
|
||||
git diff --exit-code
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
|
||||
uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0
|
||||
with:
|
||||
version: v1.63.4
|
||||
args: --timeout=15m
|
||||
|
|
|
@ -54,21 +54,21 @@ jobs:
|
|||
args: release --snapshot --skip docker
|
||||
|
||||
- name: Upload macOS binaries
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: cli_darwin_snapshot
|
||||
path: |
|
||||
dist/*_darwin_*/
|
||||
|
||||
- name: Upload Linux binaries
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: cli_linux_snapshot
|
||||
path: |
|
||||
dist/*_linux_*/
|
||||
|
||||
- name: Upload Windows binaries
|
||||
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: cli_windows_snapshot
|
||||
path: |
|
||||
|
|
|
@ -46,7 +46,7 @@ jobs:
|
|||
# QEMU is required to build cross platform docker images using buildx.
|
||||
# It allows virtualization of the CPU architecture at the application level.
|
||||
- name: Set up QEMU dependency
|
||||
uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
|
||||
|
||||
- name: Run GoReleaser
|
||||
id: releaser
|
||||
|
|
|
@ -390,6 +390,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
|
|||
if _, ok := Ignored[relPath]; ok {
|
||||
continue
|
||||
}
|
||||
if config.CompiledIgnoreObject.MatchesPath(relPath) {
|
||||
continue
|
||||
}
|
||||
unexpected = append(unexpected, relPath)
|
||||
if strings.HasPrefix(relPath, "out") {
|
||||
// We have a new file starting with "out"
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Usage: find.py <regex>
|
||||
Finds all files within current directory matching regex. The output is sorted and slashes are always forward.
|
||||
|
||||
If --expect N is provided, the number of matches must be N or error is printed.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("regex")
|
||||
parser.add_argument("--expect", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
regex = re.compile(args.regex)
|
||||
result = []
|
||||
|
||||
for root, dirs, files in os.walk("."):
|
||||
for filename in files:
|
||||
path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/")
|
||||
if regex.search(path):
|
||||
result.append(path)
|
||||
|
||||
result.sort()
|
||||
for item in result:
|
||||
print(item)
|
||||
sys.stdout.flush()
|
||||
|
||||
if args.expect is not None:
|
||||
if args.expect != len(result):
|
||||
sys.exit(f"Expected {args.expect}, got {len(result)}")
|
|
@ -0,0 +1,3 @@
|
|||
command:
|
||||
- python
|
||||
- app.py
|
|
@ -0,0 +1,8 @@
|
|||
bundle:
|
||||
name: apps_yaml
|
||||
|
||||
resources:
|
||||
apps:
|
||||
myapp:
|
||||
name: myapp
|
||||
source_code_path: ./app
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"method": "POST",
|
||||
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files/app/app.yml",
|
||||
"raw_body": "command:\n - python\n - app.py\n"
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
|
||||
>>> [CLI] bundle validate
|
||||
Name: apps_yaml
|
||||
Target: default
|
||||
Workspace:
|
||||
User: [USERNAME]
|
||||
Path: /Workspace/Users/[USERNAME]/.bundle/apps_yaml/default
|
||||
|
||||
Validation OK!
|
||||
|
||||
>>> [CLI] bundle deploy
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
|
@ -0,0 +1,4 @@
|
|||
trace $CLI bundle validate
|
||||
trace $CLI bundle deploy
|
||||
jq 'select(.path == "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files/app/app.yml")' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt
|
||||
rm out.requests.txt
|
|
@ -0,0 +1 @@
|
|||
print("Hello world!")
|
|
@ -0,0 +1,12 @@
|
|||
bundle:
|
||||
name: apps_config_section
|
||||
|
||||
resources:
|
||||
apps:
|
||||
myapp:
|
||||
name: myapp
|
||||
source_code_path: ./app
|
||||
config:
|
||||
command:
|
||||
- python
|
||||
- app.py
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"method": "POST",
|
||||
"path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files/app/app.yml",
|
||||
"raw_body": "command:\n - python\n - app.py\n"
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
|
||||
>>> [CLI] bundle validate
|
||||
Warning: App config section detected
|
||||
|
||||
remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead
|
||||
|
||||
Name: apps_config_section
|
||||
Target: default
|
||||
Workspace:
|
||||
User: [USERNAME]
|
||||
Path: /Workspace/Users/[USERNAME]/.bundle/apps_config_section/default
|
||||
|
||||
Found 1 warning
|
||||
|
||||
>>> [CLI] bundle deploy
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
Warning: App config section detected
|
||||
|
||||
remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
trace $CLI bundle validate
|
||||
trace $CLI bundle deploy
|
||||
jq 'select(.path == "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files/app/app.yml")' out.requests.txt > out.app.yml.txt
|
||||
rm out.requests.txt
|
|
@ -0,0 +1,26 @@
|
|||
Cloud = false
|
||||
RecordRequests = true
|
||||
|
||||
Ignore = [
|
||||
'.databricks',
|
||||
]
|
||||
|
||||
[[Server]]
|
||||
Pattern = "POST /api/2.0/apps"
|
||||
|
||||
[[Server]]
|
||||
Pattern = "GET /api/2.0/apps/myapp"
|
||||
Response.Body = '''
|
||||
{
|
||||
"name": "myapp",
|
||||
"description": "",
|
||||
"compute_status": {
|
||||
"state": "ACTIVE",
|
||||
"message": "App compute is active."
|
||||
},
|
||||
"app_status": {
|
||||
"state": "RUNNING",
|
||||
"message": "Application is running."
|
||||
}
|
||||
}
|
||||
'''
|
|
@ -1,3 +1,5 @@
|
|||
RecordRequests = false
|
||||
|
||||
[[Repls]]
|
||||
Old = '\\'
|
||||
New = '/'
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
export PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
uv venv -q --python 3.12 .venv
|
||||
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then
|
||||
source .venv/Scripts/activate
|
||||
else
|
||||
source .venv/bin/activate
|
||||
fi
|
||||
uv pip install -q setuptools
|
|
@ -0,0 +1,18 @@
|
|||
Cloud = false
|
||||
RecordRequests = true
|
||||
Ignore = [
|
||||
'.venv',
|
||||
'dist',
|
||||
'build',
|
||||
'*egg-info',
|
||||
'.databricks',
|
||||
]
|
||||
|
||||
[[Server]]
|
||||
Pattern = "GET /api/2.1/clusters/get"
|
||||
Response.Body = '''
|
||||
{
|
||||
"cluster_id": "0717-132531-5opeqon1",
|
||||
"spark_version": "13.3.x-scala2.12"
|
||||
}
|
||||
'''
|
|
@ -2,8 +2,8 @@
|
|||
>>> errcode [CLI] bundle deploy
|
||||
Building whl1...
|
||||
Building whl2...
|
||||
Uploading [package name]
|
||||
Uploading [package name]
|
||||
Uploading [package name]...
|
||||
Uploading [package name]...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/unique_name_libraries/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
Cloud = false
|
||||
RecordRequests = false
|
||||
|
||||
# The order in which files are uploaded can be different, so we just replace the name
|
||||
[[Repls]]
|
||||
Old="Uploading .*-0.0.1-py3-none-any.whl..."
|
||||
Old="Uploading (my_package|my_other_package)-0.0.1-py3-none-any.whl"
|
||||
New="Uploading [package name]"
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
=== Expecting to find no wheels
|
||||
>>> errcode find.py --expect 0 whl
|
||||
|
||||
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "dbfs:/path/to/dist/mywheel.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting no wheels to be uploaded
|
||||
>>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl
|
||||
|
||||
Exit code: 1
|
|
@ -0,0 +1,12 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
title "Expecting to find no wheels"
|
||||
trace errcode find.py --expect 0 whl
|
||||
|
||||
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting no wheels to be uploaded"
|
||||
trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl'
|
||||
|
||||
rm out.requests.txt
|
|
@ -5,7 +5,8 @@ artifacts:
|
|||
my_test_code:
|
||||
type: whl
|
||||
path: "./my_test_code"
|
||||
build: "python3 setup.py bdist_wheel"
|
||||
# using 'python' there because 'python3' does not exist in virtualenv on windows
|
||||
build: python setup.py bdist_wheel
|
||||
|
||||
resources:
|
||||
jobs:
|
|
@ -0,0 +1,34 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Building my_test_code...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
>>> find.py --expect 1 whl
|
||||
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting 1 wheel to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
trace find.py --expect 1 whl
|
||||
|
||||
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting 1 wheel to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -0,0 +1,34 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Building python_artifact...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
>>> find.py --expect 1 whl
|
||||
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 1 wheels in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting 1 wheels to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
trace find.py --expect 1 whl
|
||||
|
||||
title "Expecting 1 wheels in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting 1 wheels to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -0,0 +1,46 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
>>> find.py --expect 1 whl
|
||||
package/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt
|
||||
{
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"format": "MULTI_TASK",
|
||||
"max_concurrent_runs": 1,
|
||||
"name": "[default] My Wheel Job",
|
||||
"queue": {
|
||||
"enabled": true
|
||||
},
|
||||
"tasks": [
|
||||
{
|
||||
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
=== Expecting 1 wheel to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files/package/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
trace find.py --expect 1 whl
|
||||
|
||||
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt
|
||||
|
||||
title "Expecting 1 wheel to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -0,0 +1,33 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Building python_artifact...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
>>> find.py --expect 1 whl
|
||||
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"notebook_task": {
|
||||
"notebook_path": "/notebook.py"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting 1 wheel to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
trace find.py --expect 1 whl
|
||||
|
||||
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting 1 wheel to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -5,11 +5,11 @@ artifacts:
|
|||
my_test_code:
|
||||
type: whl
|
||||
path: "./my_test_code"
|
||||
build: "python3 setup.py bdist_wheel"
|
||||
build: "python setup.py bdist_wheel"
|
||||
my_test_code_2:
|
||||
type: whl
|
||||
path: "./my_test_code"
|
||||
build: "python3 setup2.py bdist_wheel"
|
||||
build: "python setup2.py bdist_wheel"
|
||||
|
||||
resources:
|
||||
jobs:
|
|
@ -0,0 +1,42 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Building my_test_code...
|
||||
Building my_test_code_2...
|
||||
Deploying resources...
|
||||
Deployment complete!
|
||||
Updating deployment state...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading my_test_code_2-0.0.1-py3-none-any.whl...
|
||||
|
||||
>>> find.py --expect 2 whl
|
||||
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 2 wheels in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
},
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting 2 wheels to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order
|
||||
|
||||
trace find.py --expect 2 whl
|
||||
|
||||
title "Expecting 2 wheels in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting 2 wheels to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm -fr out.requests.txt
|
|
@ -13,4 +13,4 @@ resources:
|
|||
entry_point: "run"
|
||||
libraries:
|
||||
- whl: ./dist/*.whl
|
||||
- whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl
|
||||
- whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
|
@ -0,0 +1,45 @@
|
|||
|
||||
>>> find.py --expect 2 whl
|
||||
dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
||||
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
>>> [CLI] bundle deploy
|
||||
Deploying resources...
|
||||
Deployment complete!
|
||||
Updating deployment state...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading other_test_code-0.0.1-py3-none-any.whl...
|
||||
|
||||
=== Expecting to find 2 wheels, same as initially provided
|
||||
>>> find.py --expect 2 whl
|
||||
dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
||||
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 2 wheels in libraries section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||
[
|
||||
{
|
||||
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||
"libraries": [
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
},
|
||||
{
|
||||
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl"
|
||||
}
|
||||
],
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
|
||||
=== Expecting 2 wheels to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,14 @@
|
|||
trace find.py --expect 2 whl
|
||||
|
||||
trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order
|
||||
|
||||
title "Expecting to find 2 wheels, same as initially provided"
|
||||
trace find.py --expect 2 whl
|
||||
|
||||
title "Expecting 2 wheels in libraries section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||
|
||||
title "Expecting 2 wheels to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -5,7 +5,7 @@ artifacts:
|
|||
my_test_code:
|
||||
type: whl
|
||||
path: "./my_test_code"
|
||||
build: "python3 setup.py bdist_wheel"
|
||||
build: python setup.py bdist_wheel
|
||||
|
||||
resources:
|
||||
jobs:
|
|
@ -0,0 +1,54 @@
|
|||
|
||||
>>> [CLI] bundle deploy
|
||||
Building my_test_code...
|
||||
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files...
|
||||
Deploying resources...
|
||||
Updating deployment state...
|
||||
Deployment complete!
|
||||
|
||||
>>> find.py --expect 1 whl
|
||||
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||
|
||||
=== Expecting 1 wheel in environments section in /jobs/create
|
||||
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt
|
||||
{
|
||||
"deployment": {
|
||||
"kind": "BUNDLE",
|
||||
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/state/metadata.json"
|
||||
},
|
||||
"edit_mode": "UI_LOCKED",
|
||||
"environments": [
|
||||
{
|
||||
"environment_key": "test_env",
|
||||
"spec": {
|
||||
"client": "1",
|
||||
"dependencies": [
|
||||
"/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"format": "MULTI_TASK",
|
||||
"max_concurrent_runs": 1,
|
||||
"name": "My Wheel Job",
|
||||
"queue": {
|
||||
"enabled": true
|
||||
},
|
||||
"tasks": [
|
||||
{
|
||||
"environment_key": "test_env",
|
||||
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||
"python_wheel_task": {
|
||||
"entry_point": "run",
|
||||
"package_name": "my_test_code"
|
||||
},
|
||||
"task_key": "TestTask"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
=== Expecting 1 wheel to be uploaded
|
||||
>>> jq .path
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
|||
trace $CLI bundle deploy
|
||||
|
||||
trace find.py --expect 1 whl
|
||||
|
||||
title "Expecting 1 wheel in environments section in /jobs/create"
|
||||
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt
|
||||
|
||||
title "Expecting 1 wheel to be uploaded"
|
||||
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||
|
||||
rm out.requests.txt
|
|
@ -1,15 +0,0 @@
|
|||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly)
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
|
||||
10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
|
||||
10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
|
||||
10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
|
||||
< HTTP/0.0 000 OK pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
|
||||
< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
|
||||
< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
|
|
@ -72,18 +72,30 @@
|
|||
10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=terraform.Initialize
|
||||
10:07:59 Debug: Apply pid=12345 mutator=scripts.postinit
|
||||
10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=scripts.postinit
|
||||
10:07:59 Debug: Apply pid=12345 mutator=validate
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly)
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:files_to_sync
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:folder_permissions
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:validate_sync_patterns
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_cluster_key_defined
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_task_cluster_spec
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:SingleNodeCluster
|
||||
10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:artifact_paths
|
||||
10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
|
||||
< HTTP/1.1 404 Not Found
|
||||
< {
|
||||
< "message": "Workspace path not found"
|
||||
< } pid=12345 mutator=validate:files_to_sync sdk=true
|
||||
10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate:files_to_sync sdk=true
|
||||
10:07:59 Debug: POST /api/2.0/workspace/mkdirs
|
||||
> {
|
||||
> "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
|
||||
> }
|
||||
< HTTP/1.1 200 OK pid=12345 mutator=validate:files_to_sync sdk=true
|
||||
10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
|
||||
< HTTP/1.1 200 OK
|
||||
< {
|
||||
< "object_type": "DIRECTORY",
|
||||
< "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
|
||||
< } pid=12345 mutator=validate:files_to_sync sdk=true
|
||||
10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate:files_to_sync
|
||||
10:07:59 Info: completed execution pid=12345 exit_code=0
|
||||
|
|
|
@ -1,4 +1 @@
|
|||
$CLI bundle validate --debug 2> full.stderr.txt
|
||||
grep -vw parallel full.stderr.txt > out.stderr.txt
|
||||
grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt
|
||||
rm full.stderr.txt
|
||||
$CLI bundle validate --debug 2> out.stderr.txt
|
||||
|
|
|
@ -2,10 +2,8 @@ Warning: Include section is defined outside root file
|
|||
at include
|
||||
in a.yml:2:3
|
||||
|
||||
The include section is defined in a file that is not the root file.
|
||||
These values will be ignored because only the includes defined in
|
||||
the bundle root file (that is databricks.yml or databricks.yaml)
|
||||
are loaded.
|
||||
An include section is defined in a file that is not databricks.yml.
|
||||
Only includes defined in databricks.yml are applied.
|
||||
|
||||
Name: include_outside_root
|
||||
Target: default
|
||||
|
|
|
@ -11,6 +11,7 @@ import (
|
|||
"github.com/BurntSushi/toml"
|
||||
"github.com/databricks/cli/libs/testdiff"
|
||||
"github.com/databricks/cli/libs/testserver"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -51,6 +52,11 @@ type TestConfig struct {
|
|||
|
||||
// List of request headers to include when recording requests.
|
||||
IncludeRequestHeaders []string
|
||||
|
||||
// List of gitignore patterns to ignore when checking output files
|
||||
Ignore []string
|
||||
|
||||
CompiledIgnoreObject *ignore.GitIgnore
|
||||
}
|
||||
|
||||
type ServerStub struct {
|
||||
|
@ -111,6 +117,8 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
|
|||
}
|
||||
}
|
||||
|
||||
result.CompiledIgnoreObject = ignore.CompileIgnoreLines(result.Ignore...)
|
||||
|
||||
return result, strings.Join(configs, ", ")
|
||||
}
|
||||
|
||||
|
|
|
@ -27,3 +27,7 @@ echo 123456
|
|||
|
||||
printf "\n=== Testing --version"
|
||||
trace $CLI --version
|
||||
|
||||
touch ignored_file.txt
|
||||
mkdir ignored_dir
|
||||
touch ignored_dir/hello.txt
|
||||
|
|
|
@ -1,5 +1,8 @@
|
|||
# Badness = "Brief description of what's wrong with the test output, if anything"
|
||||
|
||||
Ignore = ['ignore*']
|
||||
|
||||
|
||||
#[GOOS]
|
||||
# Disable on Windows
|
||||
#windows = false
|
||||
|
|
|
@ -3,8 +3,6 @@ package apps
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
|
@ -14,7 +12,6 @@ type validate struct{}
|
|||
|
||||
func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
var diags diag.Diagnostics
|
||||
possibleConfigFiles := []string{"app.yml", "app.yaml"}
|
||||
usedSourceCodePaths := make(map[string]string)
|
||||
|
||||
for key, app := range b.Config.Resources.Apps {
|
||||
|
@ -28,18 +25,14 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
|
|||
}
|
||||
usedSourceCodePaths[app.SourceCodePath] = key
|
||||
|
||||
for _, configFile := range possibleConfigFiles {
|
||||
appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
|
||||
cf := path.Join(appPath, configFile)
|
||||
if _, err := b.SyncRoot.Stat(cf); err == nil {
|
||||
if app.Config != nil {
|
||||
diags = append(diags, diag.Diagnostic{
|
||||
Severity: diag.Error,
|
||||
Summary: configFile + " detected",
|
||||
Detail: fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name),
|
||||
Severity: diag.Warning,
|
||||
Summary: "App config section detected",
|
||||
Detail: fmt.Sprintf("remove 'config' from app resource '%s' section and use app.yml file in the root of this app instead", key),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return diags
|
||||
}
|
||||
|
|
|
@ -17,46 +17,6 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAppsValidate(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testutil.Touch(t, tmpDir, "app1", "app.yml")
|
||||
testutil.Touch(t, tmpDir, "app2", "app.py")
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
SyncRootPath: tmpDir,
|
||||
SyncRoot: vfs.MustNew(tmpDir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/foo/bar/",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Apps: map[string]*resources.App{
|
||||
"app1": {
|
||||
App: &apps.App{
|
||||
Name: "app1",
|
||||
},
|
||||
SourceCodePath: "./app1",
|
||||
},
|
||||
"app2": {
|
||||
App: &apps.App{
|
||||
Name: "app2",
|
||||
},
|
||||
SourceCodePath: "./app2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||
|
||||
diags := bundle.ApplySeq(context.Background(), b, mutator.TranslatePaths(), Validate())
|
||||
require.Len(t, diags, 1)
|
||||
require.Equal(t, "app.yml detected", diags[0].Summary)
|
||||
require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource")
|
||||
}
|
||||
|
||||
func TestAppsValidateSameSourcePath(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testutil.Touch(t, tmpDir, "app1", "app.py")
|
||||
|
|
|
@ -1,49 +0,0 @@
|
|||
package bundle
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/libs/vfs"
|
||||
"github.com/databricks/databricks-sdk-go"
|
||||
)
|
||||
|
||||
type ReadOnlyBundle struct {
|
||||
b *Bundle
|
||||
}
|
||||
|
||||
func ReadOnly(b *Bundle) ReadOnlyBundle {
|
||||
return ReadOnlyBundle{b: b}
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) Config() config.Root {
|
||||
return r.b.Config
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) RootPath() string {
|
||||
return r.b.BundleRootPath
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) BundleRoot() vfs.Path {
|
||||
return r.b.BundleRoot
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) SyncRoot() vfs.Path {
|
||||
return r.b.SyncRoot
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) WorktreeRoot() vfs.Path {
|
||||
return r.b.WorktreeRoot
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
|
||||
return r.b.WorkspaceClient()
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) CacheDir(ctx context.Context, paths ...string) (string, error) {
|
||||
return r.b.CacheDir(ctx, paths...)
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
|
||||
return r.b.GetSyncIncludePatterns(ctx)
|
||||
}
|
|
@ -165,10 +165,8 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
|
|||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
Summary: "Include section is defined outside root file",
|
||||
Detail: `The include section is defined in a file that is not the root file.
|
||||
These values will be ignored because only the includes defined in
|
||||
the bundle root file (that is databricks.yml or databricks.yaml)
|
||||
are loaded.`,
|
||||
Detail: `An include section is defined in a file that is not databricks.yml.
|
||||
Only includes defined in databricks.yml are applied.`,
|
||||
Locations: this.GetLocations("include"),
|
||||
Paths: []dyn.Path{dyn.MustPathFromString("include")},
|
||||
})
|
||||
|
|
|
@ -14,18 +14,18 @@ import (
|
|||
// 2. The validation is blocking for bundle deployments.
|
||||
//
|
||||
// The full suite of validation mutators is available in the [Validate] mutator.
|
||||
type fastValidateReadonly struct{}
|
||||
type fastValidate struct{ bundle.RO }
|
||||
|
||||
func FastValidateReadonly() bundle.ReadOnlyMutator {
|
||||
return &fastValidateReadonly{}
|
||||
func FastValidate() bundle.ReadOnlyMutator {
|
||||
return &fastValidate{}
|
||||
}
|
||||
|
||||
func (f *fastValidateReadonly) Name() string {
|
||||
func (f *fastValidate) Name() string {
|
||||
return "fast_validate(readonly)"
|
||||
}
|
||||
|
||||
func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
return bundle.ApplyReadOnly(ctx, rb, bundle.Parallel(
|
||||
func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagnostics {
|
||||
return bundle.ApplyParallel(ctx, rb,
|
||||
// Fast mutators with only in-memory checks
|
||||
JobClusterKeyDefined(),
|
||||
JobTaskClusterSpec(),
|
||||
|
@ -33,19 +33,5 @@ func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBund
|
|||
|
||||
// Blocking mutators. Deployments will fail if these checks fail.
|
||||
ValidateArtifactPath(),
|
||||
))
|
||||
}
|
||||
|
||||
type fastValidate struct{}
|
||||
|
||||
func FastValidate() bundle.Mutator {
|
||||
return &fastValidate{}
|
||||
}
|
||||
|
||||
func (f *fastValidate) Name() string {
|
||||
return "fast_validate"
|
||||
}
|
||||
|
||||
func (f *fastValidate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), FastValidateReadonly())
|
||||
)
|
||||
}
|
||||
|
|
|
@ -13,20 +13,20 @@ func FilesToSync() bundle.ReadOnlyMutator {
|
|||
return &filesToSync{}
|
||||
}
|
||||
|
||||
type filesToSync struct{}
|
||||
type filesToSync struct{ bundle.RO }
|
||||
|
||||
func (v *filesToSync) Name() string {
|
||||
return "validate:files_to_sync"
|
||||
}
|
||||
|
||||
func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
// The user may be intentional about not synchronizing any files.
|
||||
// In this case, we should not show any warnings.
|
||||
if len(rb.Config().Sync.Paths) == 0 {
|
||||
if len(b.Config.Sync.Paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sync, err := files.GetSync(ctx, rb)
|
||||
sync, err := files.GetSync(ctx, b)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
@ -42,20 +42,20 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.
|
|||
}
|
||||
|
||||
diags := diag.Diagnostics{}
|
||||
if len(rb.Config().Sync.Exclude) == 0 {
|
||||
if len(b.Config.Sync.Exclude) == 0 {
|
||||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
Summary: "There are no files to sync, please check your .gitignore",
|
||||
})
|
||||
} else {
|
||||
loc := location{path: "sync.exclude", rb: rb}
|
||||
path := "sync.exclude"
|
||||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration",
|
||||
// Show all locations where sync.exclude is defined, since merging
|
||||
// sync.exclude is additive.
|
||||
Locations: loc.Locations(),
|
||||
Paths: []dyn.Path{loc.Path()},
|
||||
Locations: b.Config.GetLocations(path),
|
||||
Paths: []dyn.Path{dyn.MustPathFromString(path)},
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -29,8 +29,7 @@ func TestFilesToSync_NoPaths(t *testing.T) {
|
|||
}
|
||||
|
||||
ctx := context.Background()
|
||||
rb := bundle.ReadOnly(b)
|
||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
||||
diags := FilesToSync().Apply(ctx, b)
|
||||
assert.Empty(t, diags)
|
||||
}
|
||||
|
||||
|
@ -85,8 +84,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) {
|
|||
testutil.WriteFile(t, filepath.Join(b.BundleRootPath, ".gitignore"), "*\n.*\n")
|
||||
|
||||
ctx := context.Background()
|
||||
rb := bundle.ReadOnly(b)
|
||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
||||
diags := FilesToSync().Apply(ctx, b)
|
||||
require.Len(t, diags, 1)
|
||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
||||
assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary)
|
||||
|
@ -99,8 +97,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) {
|
|||
b.Config.Sync.Exclude = []string{"*"}
|
||||
|
||||
ctx := context.Background()
|
||||
rb := bundle.ReadOnly(b)
|
||||
diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
|
||||
diags := FilesToSync().Apply(ctx, b)
|
||||
require.Len(t, diags, 1)
|
||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
||||
assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary)
|
||||
|
|
|
@ -16,15 +16,14 @@ import (
|
|||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type folderPermissions struct{}
|
||||
type folderPermissions struct{ bundle.RO }
|
||||
|
||||
// Apply implements bundle.ReadOnlyMutator.
|
||||
func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
if len(b.Config().Permissions) == 0 {
|
||||
func (f *folderPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
if len(b.Config.Permissions) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config().Workspace)
|
||||
bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config.Workspace)
|
||||
|
||||
var diags diag.Diagnostics
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
|
@ -48,7 +47,7 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle)
|
|||
return diags
|
||||
}
|
||||
|
||||
func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderPath string) diag.Diagnostics {
|
||||
func checkFolderPermission(ctx context.Context, b *bundle.Bundle, folderPath string) diag.Diagnostics {
|
||||
// If the folder is shared, then we don't need to check permissions as it was already checked in the other mutator before.
|
||||
if libraries.IsWorkspaceSharedPath(folderPath) {
|
||||
return nil
|
||||
|
@ -69,7 +68,7 @@ func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderP
|
|||
}
|
||||
|
||||
p := permissions.ObjectAclToResourcePermissions(folderPath, objPermissions.AccessControlList)
|
||||
return p.Compare(b.Config().Permissions)
|
||||
return p.Compare(b.Config.Permissions)
|
||||
}
|
||||
|
||||
func getClosestExistingObject(ctx context.Context, w workspace.WorkspaceInterface, folderPath string) (*workspace.ObjectInfo, error) {
|
||||
|
|
|
@ -69,9 +69,7 @@ func TestFolderPermissionsInheritedWhenRootPathDoesNotExist(t *testing.T) {
|
|||
}, nil)
|
||||
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
rb := bundle.ReadOnly(b)
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions())
|
||||
diags := ValidateFolderPermissions().Apply(context.Background(), b)
|
||||
require.Empty(t, diags)
|
||||
}
|
||||
|
||||
|
@ -118,9 +116,7 @@ func TestValidateFolderPermissionsFailsOnMissingBundlePermission(t *testing.T) {
|
|||
}, nil)
|
||||
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
rb := bundle.ReadOnly(b)
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions())
|
||||
diags := ValidateFolderPermissions().Apply(context.Background(), b)
|
||||
require.Len(t, diags, 1)
|
||||
require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary)
|
||||
require.Equal(t, diag.Warning, diags[0].Severity)
|
||||
|
@ -164,9 +160,7 @@ func TestValidateFolderPermissionsFailsOnPermissionMismatch(t *testing.T) {
|
|||
}, nil)
|
||||
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
rb := bundle.ReadOnly(b)
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions())
|
||||
diags := ValidateFolderPermissions().Apply(context.Background(), b)
|
||||
require.Len(t, diags, 1)
|
||||
require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary)
|
||||
require.Equal(t, diag.Warning, diags[0].Severity)
|
||||
|
@ -199,9 +193,7 @@ func TestValidateFolderPermissionsFailsOnNoRootFolder(t *testing.T) {
|
|||
})
|
||||
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
rb := bundle.ReadOnly(b)
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions())
|
||||
diags := ValidateFolderPermissions().Apply(context.Background(), b)
|
||||
require.Len(t, diags, 1)
|
||||
require.Equal(t, "folder / and its parent folders do not exist", diags[0].Summary)
|
||||
require.Equal(t, diag.Error, diags[0].Severity)
|
||||
|
|
|
@ -24,7 +24,10 @@ func (f *noInterpolationInAuthConfig) Name() string {
|
|||
func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
authFields := []string{
|
||||
// Generic attributes.
|
||||
"host", "profile", "auth_type", "metadata_service_url",
|
||||
"host",
|
||||
"profile",
|
||||
"auth_type",
|
||||
"metadata_service_url",
|
||||
|
||||
// OAuth specific attributes.
|
||||
"client_id",
|
||||
|
@ -33,8 +36,12 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl
|
|||
"google_service_account",
|
||||
|
||||
// Azure specific attributes.
|
||||
"azure_resource_id", "azure_use_msi", "azure_client_id", "azure_tenant_id",
|
||||
"azure_environment", "azure_login_app_id",
|
||||
"azure_resource_id",
|
||||
"azure_use_msi",
|
||||
"azure_client_id",
|
||||
"azure_tenant_id",
|
||||
"azure_environment",
|
||||
"azure_login_app_id",
|
||||
}
|
||||
|
||||
diags := diag.Diagnostics{}
|
||||
|
@ -49,12 +56,11 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl
|
|||
return diag.FromErr(err)
|
||||
}
|
||||
|
||||
if v.Kind() == dyn.KindInvalid || v.Kind() == dyn.KindNil {
|
||||
vv, ok := v.AsString()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
vv := v.MustString()
|
||||
|
||||
// Check if the field contains interpolation.
|
||||
if dynvar.ContainsVariableReference(vv) {
|
||||
envVar, ok := auth.GetEnvFor(fieldName)
|
||||
|
|
|
@ -13,16 +13,16 @@ func JobClusterKeyDefined() bundle.ReadOnlyMutator {
|
|||
return &jobClusterKeyDefined{}
|
||||
}
|
||||
|
||||
type jobClusterKeyDefined struct{}
|
||||
type jobClusterKeyDefined struct{ bundle.RO }
|
||||
|
||||
func (v *jobClusterKeyDefined) Name() string {
|
||||
return "validate:job_cluster_key_defined"
|
||||
}
|
||||
|
||||
func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
for k, job := range rb.Config().Resources.Jobs {
|
||||
for k, job := range b.Config.Resources.Jobs {
|
||||
jobClusterKeys := make(map[string]bool)
|
||||
for _, cluster := range job.JobClusters {
|
||||
if cluster.JobClusterKey != "" {
|
||||
|
@ -33,10 +33,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund
|
|||
for index, task := range job.Tasks {
|
||||
if task.JobClusterKey != "" {
|
||||
if _, ok := jobClusterKeys[task.JobClusterKey]; !ok {
|
||||
loc := location{
|
||||
path: fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index),
|
||||
rb: rb,
|
||||
}
|
||||
path := fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index)
|
||||
|
||||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
|
@ -44,8 +41,8 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund
|
|||
// Show only the location where the job_cluster_key is defined.
|
||||
// Other associated locations are not relevant since they are
|
||||
// overridden during merging.
|
||||
Locations: []dyn.Location{loc.Location()},
|
||||
Paths: []dyn.Path{loc.Path()},
|
||||
Locations: b.Config.GetLocations(path),
|
||||
Paths: []dyn.Path{dyn.MustPathFromString(path)},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -33,7 +33,7 @@ func TestJobClusterKeyDefined(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
|
||||
diags := JobClusterKeyDefined().Apply(context.Background(), b)
|
||||
require.Empty(t, diags)
|
||||
require.NoError(t, diags.Error())
|
||||
}
|
||||
|
@ -56,7 +56,7 @@ func TestJobClusterKeyNotDefined(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
|
||||
diags := JobClusterKeyDefined().Apply(context.Background(), b)
|
||||
require.Len(t, diags, 1)
|
||||
require.NoError(t, diags.Error())
|
||||
require.Equal(t, diag.Warning, diags[0].Severity)
|
||||
|
@ -89,7 +89,7 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined())
|
||||
diags := JobClusterKeyDefined().Apply(context.Background(), b)
|
||||
require.Len(t, diags, 1)
|
||||
require.NoError(t, diags.Error())
|
||||
require.Equal(t, diag.Warning, diags[0].Severity)
|
||||
|
|
|
@ -17,31 +17,31 @@ func JobTaskClusterSpec() bundle.ReadOnlyMutator {
|
|||
return &jobTaskClusterSpec{}
|
||||
}
|
||||
|
||||
type jobTaskClusterSpec struct{}
|
||||
type jobTaskClusterSpec struct{ bundle.RO }
|
||||
|
||||
func (v *jobTaskClusterSpec) Name() string {
|
||||
return "validate:job_task_cluster_spec"
|
||||
}
|
||||
|
||||
func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
|
||||
|
||||
for resourceName, job := range rb.Config().Resources.Jobs {
|
||||
for resourceName, job := range b.Config.Resources.Jobs {
|
||||
resourcePath := jobsPath.Append(dyn.Key(resourceName))
|
||||
|
||||
for taskIndex, task := range job.Tasks {
|
||||
taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex))
|
||||
|
||||
diags = diags.Extend(validateJobTask(rb, task, taskPath))
|
||||
diags = diags.Extend(validateJobTask(b, task, taskPath))
|
||||
}
|
||||
}
|
||||
|
||||
return diags
|
||||
}
|
||||
|
||||
func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics {
|
||||
func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
var specified []string
|
||||
|
@ -74,7 +74,7 @@ func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path
|
|||
if task.ForEachTask != nil {
|
||||
forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task"))
|
||||
|
||||
diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath))
|
||||
diags = diags.Extend(validateJobTask(b, task.ForEachTask.Task, forEachTaskPath))
|
||||
}
|
||||
|
||||
if isComputeTask(task) && len(specified) == 0 {
|
||||
|
@ -92,7 +92,7 @@ func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path
|
|||
Severity: diag.Error,
|
||||
Summary: "Missing required cluster or environment settings",
|
||||
Detail: detail,
|
||||
Locations: rb.Config().GetLocations(taskPath.String()),
|
||||
Locations: b.Config.GetLocations(taskPath.String()),
|
||||
Paths: []dyn.Path{taskPath},
|
||||
})
|
||||
}
|
||||
|
|
|
@ -174,7 +174,7 @@ Specify one of the following fields: job_cluster_key, environment_key, existing_
|
|||
}
|
||||
|
||||
b := createBundle(map[string]*resources.Job{"job1": job})
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec())
|
||||
diags := JobTaskClusterSpec().Apply(context.Background(), b)
|
||||
|
||||
if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" {
|
||||
assert.Len(t, diags, 1)
|
||||
|
|
|
@ -16,7 +16,7 @@ func SingleNodeCluster() bundle.ReadOnlyMutator {
|
|||
return &singleNodeCluster{}
|
||||
}
|
||||
|
||||
type singleNodeCluster struct{}
|
||||
type singleNodeCluster struct{ bundle.RO }
|
||||
|
||||
func (m *singleNodeCluster) Name() string {
|
||||
return "validate:SingleNodeCluster"
|
||||
|
@ -98,7 +98,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
patterns := []dyn.Pattern{
|
||||
|
@ -115,7 +115,7 @@ func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle)
|
|||
}
|
||||
|
||||
for _, p := range patterns {
|
||||
_, err := dyn.MapByPattern(rb.Config().Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
_, err := dyn.MapByPattern(b.Config.Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
warning := diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
Summary: singleNodeWarningSummary,
|
||||
|
|
|
@ -116,7 +116,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) {
|
|||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||
return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0))
|
||||
})
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := SingleNodeCluster().Apply(ctx, b)
|
||||
assert.Equal(t, diag.Diagnostics{
|
||||
{
|
||||
Severity: diag.Warning,
|
||||
|
@ -165,7 +165,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) {
|
|||
return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0))
|
||||
})
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := SingleNodeCluster().Apply(ctx, b)
|
||||
assert.Equal(t, diag.Diagnostics{
|
||||
{
|
||||
Severity: diag.Warning,
|
||||
|
@ -214,7 +214,7 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) {
|
|||
return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0))
|
||||
})
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Equal(t, diag.Diagnostics{
|
||||
{
|
||||
Severity: diag.Warning,
|
||||
|
@ -260,7 +260,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) {
|
|||
return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0))
|
||||
})
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Equal(t, diag.Diagnostics{
|
||||
{
|
||||
Severity: diag.Warning,
|
||||
|
@ -313,7 +313,7 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) {
|
|||
return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0))
|
||||
})
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Equal(t, diag.Diagnostics{
|
||||
{
|
||||
Severity: diag.Warning,
|
||||
|
@ -397,7 +397,7 @@ func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Empty(t, diags)
|
||||
})
|
||||
}
|
||||
|
@ -437,7 +437,7 @@ func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Empty(t, diags)
|
||||
})
|
||||
}
|
||||
|
@ -477,7 +477,7 @@ func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Empty(t, diags)
|
||||
})
|
||||
}
|
||||
|
@ -514,7 +514,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Empty(t, diags)
|
||||
})
|
||||
}
|
||||
|
@ -558,7 +558,7 @@ func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||
diags := bundle.Apply(ctx, b, SingleNodeCluster())
|
||||
assert.Empty(t, diags)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -5,46 +5,16 @@ import (
|
|||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type validate struct{}
|
||||
|
||||
type location struct {
|
||||
path string
|
||||
rb bundle.ReadOnlyBundle
|
||||
}
|
||||
|
||||
func (l location) Location() dyn.Location {
|
||||
return l.rb.Config().GetLocation(l.path)
|
||||
}
|
||||
|
||||
func (l location) Locations() []dyn.Location {
|
||||
return l.rb.Config().GetLocations(l.path)
|
||||
}
|
||||
|
||||
func (l location) Path() dyn.Path {
|
||||
return dyn.MustPathFromString(l.path)
|
||||
}
|
||||
|
||||
// Apply implements bundle.Mutator.
|
||||
func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), bundle.Parallel(
|
||||
FastValidateReadonly(),
|
||||
func Validate(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
return bundle.ApplyParallel(ctx, b,
|
||||
FastValidate(),
|
||||
|
||||
// Slow mutators that require network or file i/o. These are only
|
||||
// run in the `bundle validate` command.
|
||||
FilesToSync(),
|
||||
ValidateFolderPermissions(),
|
||||
ValidateSyncPatterns(),
|
||||
))
|
||||
}
|
||||
|
||||
// Name implements bundle.Mutator.
|
||||
func (v *validate) Name() string {
|
||||
return "validate"
|
||||
}
|
||||
|
||||
func Validate() bundle.Mutator {
|
||||
return &validate{}
|
||||
)
|
||||
}
|
||||
|
|
|
@ -16,7 +16,7 @@ import (
|
|||
"github.com/databricks/databricks-sdk-go/apierr"
|
||||
)
|
||||
|
||||
type validateArtifactPath struct{}
|
||||
type validateArtifactPath struct{ bundle.RO }
|
||||
|
||||
func ValidateArtifactPath() bundle.ReadOnlyMutator {
|
||||
return &validateArtifactPath{}
|
||||
|
@ -74,9 +74,9 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin
|
|||
return nil, nil, false
|
||||
}
|
||||
|
||||
func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
// We only validate UC Volumes paths right now.
|
||||
if !libraries.IsVolumesPath(rb.Config().Workspace.ArtifactPath) {
|
||||
if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -85,25 +85,25 @@ func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBund
|
|||
{
|
||||
Summary: s,
|
||||
Severity: diag.Error,
|
||||
Locations: rb.Config().GetLocations("workspace.artifact_path"),
|
||||
Locations: b.Config.GetLocations("workspace.artifact_path"),
|
||||
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
catalogName, schemaName, volumeName, err := extractVolumeFromPath(rb.Config().Workspace.ArtifactPath)
|
||||
catalogName, schemaName, volumeName, err := extractVolumeFromPath(b.Config.Workspace.ArtifactPath)
|
||||
if err != nil {
|
||||
return wrapErrorMsg(err.Error())
|
||||
}
|
||||
volumeFullName := fmt.Sprintf("%s.%s.%s", catalogName, schemaName, volumeName)
|
||||
w := rb.WorkspaceClient()
|
||||
w := b.WorkspaceClient()
|
||||
_, err = w.Volumes.ReadByName(ctx, volumeFullName)
|
||||
|
||||
if errors.Is(err, apierr.ErrPermissionDenied) {
|
||||
return wrapErrorMsg(fmt.Sprintf("cannot access volume %s: %s", volumeFullName, err))
|
||||
}
|
||||
if errors.Is(err, apierr.ErrNotFound) {
|
||||
path, locations, ok := findVolumeInBundle(rb.Config(), catalogName, schemaName, volumeName)
|
||||
path, locations, ok := findVolumeInBundle(b.Config, catalogName, schemaName, volumeName)
|
||||
if !ok {
|
||||
return wrapErrorMsg(fmt.Sprintf("volume %s does not exist", volumeFullName))
|
||||
}
|
||||
|
@ -117,7 +117,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBund
|
|||
this bundle but which has not been deployed yet. Please first deploy
|
||||
the volume using 'bundle deploy' and then switch over to using it in
|
||||
the artifact_path.`,
|
||||
Locations: slices.Concat(rb.Config().GetLocations("workspace.artifact_path"), locations),
|
||||
Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations),
|
||||
Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path),
|
||||
}}
|
||||
|
||||
|
|
|
@ -49,7 +49,7 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) {
|
|||
})
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), ValidateArtifactPath())
|
||||
diags := ValidateArtifactPath().Apply(ctx, b)
|
||||
assert.Equal(t, diag.Diagnostics{{
|
||||
Severity: diag.Error,
|
||||
Summary: "volume catalogN.schemaN.volumeN does not exist",
|
||||
|
@ -88,7 +88,6 @@ func TestValidateArtifactPath(t *testing.T) {
|
|||
}}, diags)
|
||||
}
|
||||
|
||||
rb := bundle.ReadOnly(b)
|
||||
ctx := context.Background()
|
||||
|
||||
tcases := []struct {
|
||||
|
@ -123,7 +122,7 @@ func TestValidateArtifactPath(t *testing.T) {
|
|||
api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err)
|
||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||
|
||||
diags := bundle.ApplyReadOnly(ctx, rb, ValidateArtifactPath())
|
||||
diags := ValidateArtifactPath().Apply(ctx, b)
|
||||
assertDiags(t, diags, tc.expectedSummary)
|
||||
}
|
||||
}
|
||||
|
@ -167,7 +166,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) {
|
|||
|
||||
bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
|
||||
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath())
|
||||
diags := ValidateArtifactPath().Apply(context.Background(), b)
|
||||
require.Equal(t, diag.Diagnostics{{
|
||||
Severity: diag.Error,
|
||||
Summary: "expected UC volume path to be in the format /Volumes/<catalog>/<schema>/<volume>/..., got " + p,
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue