mirror of https://github.com/databricks/cli.git
Convert python wheel tests to acceptance (#2396)
## Changes Rewrite bundle/tests/python_wheel_test.go into acceptance tests. The same configs are used, but the test now runs 'bundle deploy' and in addition to checking the files on the file system, also checks that the files were uploaded and records jobs/create request. There is a new test helper bin/find.py which filters out paths based on regex, asserts on number of expected results. I've added it because 'find' on Windows behaves differently, so this helps avoid cross-platform differences.
This commit is contained in:
parent
03e4bb2575
commit
8f8f24c3a9
|
@ -0,0 +1,36 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Usage: find.py <regex>
|
||||||
|
Finds all files within current directory matching regex. The output is sorted and slashes are always forward.
|
||||||
|
|
||||||
|
If --expect N is provided, the number of matches must be N or error is printed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("regex")
|
||||||
|
parser.add_argument("--expect", type=int)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
regex = re.compile(args.regex)
|
||||||
|
result = []
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk("."):
|
||||||
|
for filename in files:
|
||||||
|
path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/")
|
||||||
|
if regex.search(path):
|
||||||
|
result.append(path)
|
||||||
|
|
||||||
|
result.sort()
|
||||||
|
for item in result:
|
||||||
|
print(item)
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
if args.expect is not None:
|
||||||
|
if args.expect != len(result):
|
||||||
|
sys.exit(f"Expected {args.expect}, got {len(result)}")
|
|
@ -1,3 +1,5 @@
|
||||||
|
RecordRequests = false
|
||||||
|
|
||||||
[[Repls]]
|
[[Repls]]
|
||||||
Old = '\\'
|
Old = '\\'
|
||||||
New = '/'
|
New = '/'
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
export PYTHONDONTWRITEBYTECODE=1
|
||||||
|
|
||||||
|
uv venv -q --python 3.12 .venv
|
||||||
|
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then
|
||||||
|
source .venv/Scripts/activate
|
||||||
|
else
|
||||||
|
source .venv/bin/activate
|
||||||
|
fi
|
||||||
|
uv pip install -q setuptools
|
|
@ -0,0 +1,18 @@
|
||||||
|
Cloud = false
|
||||||
|
RecordRequests = true
|
||||||
|
Ignore = [
|
||||||
|
'.venv',
|
||||||
|
'dist',
|
||||||
|
'build',
|
||||||
|
'*egg-info',
|
||||||
|
'.databricks',
|
||||||
|
]
|
||||||
|
|
||||||
|
[[Server]]
|
||||||
|
Pattern = "GET /api/2.1/clusters/get"
|
||||||
|
Response.Body = '''
|
||||||
|
{
|
||||||
|
"cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"spark_version": "13.3.x-scala2.12"
|
||||||
|
}
|
||||||
|
'''
|
|
@ -1,4 +1,4 @@
|
||||||
Cloud = false
|
RecordRequests = false
|
||||||
|
|
||||||
# The order in which files are uploaded can be different, so we just replace the name
|
# The order in which files are uploaded can be different, so we just replace the name
|
||||||
[[Repls]]
|
[[Repls]]
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
=== Expecting to find no wheels
|
||||||
|
>>> errcode find.py --expect 0 whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "dbfs:/path/to/dist/mywheel.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting no wheels to be uploaded
|
||||||
|
>>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl
|
||||||
|
|
||||||
|
Exit code: 1
|
|
@ -0,0 +1,12 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
title "Expecting to find no wheels"
|
||||||
|
trace errcode find.py --expect 0 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting no wheels to be uploaded"
|
||||||
|
trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl'
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -5,7 +5,8 @@ artifacts:
|
||||||
my_test_code:
|
my_test_code:
|
||||||
type: whl
|
type: whl
|
||||||
path: "./my_test_code"
|
path: "./my_test_code"
|
||||||
build: "python3 setup.py bdist_wheel"
|
# using 'python' there because 'python3' does not exist in virtualenv on windows
|
||||||
|
build: python setup.py bdist_wheel
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
jobs:
|
jobs:
|
|
@ -0,0 +1,34 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Building my_test_code...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
>>> find.py --expect 1 whl
|
||||||
|
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting 1 wheel to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
trace find.py --expect 1 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 1 wheel to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -0,0 +1,34 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Building python_artifact...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
>>> find.py --expect 1 whl
|
||||||
|
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheels in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting 1 wheels to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
trace find.py --expect 1 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheels in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 1 wheels to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -0,0 +1,46 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
>>> find.py --expect 1 whl
|
||||||
|
package/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt
|
||||||
|
{
|
||||||
|
"deployment": {
|
||||||
|
"kind": "BUNDLE",
|
||||||
|
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/state/metadata.json"
|
||||||
|
},
|
||||||
|
"edit_mode": "UI_LOCKED",
|
||||||
|
"format": "MULTI_TASK",
|
||||||
|
"max_concurrent_runs": 1,
|
||||||
|
"name": "[default] My Wheel Job",
|
||||||
|
"queue": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
=== Expecting 1 wheel to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files/package/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
trace find.py --expect 1 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 1 wheel to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -0,0 +1,33 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Building python_artifact...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
>>> find.py --expect 1 whl
|
||||||
|
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheel in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-aaaaa-bbbbbb",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"notebook_task": {
|
||||||
|
"notebook_path": "/notebook.py"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting 1 wheel to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
trace find.py --expect 1 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheel in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 1 wheel to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -5,11 +5,11 @@ artifacts:
|
||||||
my_test_code:
|
my_test_code:
|
||||||
type: whl
|
type: whl
|
||||||
path: "./my_test_code"
|
path: "./my_test_code"
|
||||||
build: "python3 setup.py bdist_wheel"
|
build: "python setup.py bdist_wheel"
|
||||||
my_test_code_2:
|
my_test_code_2:
|
||||||
type: whl
|
type: whl
|
||||||
path: "./my_test_code"
|
path: "./my_test_code"
|
||||||
build: "python3 setup2.py bdist_wheel"
|
build: "python setup2.py bdist_wheel"
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
jobs:
|
jobs:
|
|
@ -0,0 +1,42 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Building my_test_code...
|
||||||
|
Building my_test_code_2...
|
||||||
|
Deploying resources...
|
||||||
|
Deployment complete!
|
||||||
|
Updating deployment state...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading my_test_code_2-0.0.1-py3-none-any.whl...
|
||||||
|
|
||||||
|
>>> find.py --expect 2 whl
|
||||||
|
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 2 wheels in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting 2 wheels to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order
|
||||||
|
|
||||||
|
trace find.py --expect 2 whl
|
||||||
|
|
||||||
|
title "Expecting 2 wheels in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 2 wheels to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm -fr out.requests.txt
|
|
@ -13,4 +13,4 @@ resources:
|
||||||
entry_point: "run"
|
entry_point: "run"
|
||||||
libraries:
|
libraries:
|
||||||
- whl: ./dist/*.whl
|
- whl: ./dist/*.whl
|
||||||
- whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl
|
- whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
|
@ -0,0 +1,45 @@
|
||||||
|
|
||||||
|
>>> find.py --expect 2 whl
|
||||||
|
dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
||||||
|
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Deploying resources...
|
||||||
|
Deployment complete!
|
||||||
|
Updating deployment state...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading other_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
|
||||||
|
=== Expecting to find 2 wheels, same as initially provided
|
||||||
|
>>> find.py --expect 2 whl
|
||||||
|
dist/lib/other_test_code-0.0.1-py3-none-any.whl
|
||||||
|
dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 2 wheels in libraries section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"libraries": [
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
=== Expecting 2 wheels to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,14 @@
|
||||||
|
trace find.py --expect 2 whl
|
||||||
|
|
||||||
|
trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order
|
||||||
|
|
||||||
|
title "Expecting to find 2 wheels, same as initially provided"
|
||||||
|
trace find.py --expect 2 whl
|
||||||
|
|
||||||
|
title "Expecting 2 wheels in libraries section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 2 wheels to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -5,7 +5,7 @@ artifacts:
|
||||||
my_test_code:
|
my_test_code:
|
||||||
type: whl
|
type: whl
|
||||||
path: "./my_test_code"
|
path: "./my_test_code"
|
||||||
build: "python3 setup.py bdist_wheel"
|
build: python setup.py bdist_wheel
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
jobs:
|
jobs:
|
|
@ -0,0 +1,54 @@
|
||||||
|
|
||||||
|
>>> [CLI] bundle deploy
|
||||||
|
Building my_test_code...
|
||||||
|
Uploading my_test_code-0.0.1-py3-none-any.whl...
|
||||||
|
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files...
|
||||||
|
Deploying resources...
|
||||||
|
Updating deployment state...
|
||||||
|
Deployment complete!
|
||||||
|
|
||||||
|
>>> find.py --expect 1 whl
|
||||||
|
my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
|
||||||
|
|
||||||
|
=== Expecting 1 wheel in environments section in /jobs/create
|
||||||
|
>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt
|
||||||
|
{
|
||||||
|
"deployment": {
|
||||||
|
"kind": "BUNDLE",
|
||||||
|
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/state/metadata.json"
|
||||||
|
},
|
||||||
|
"edit_mode": "UI_LOCKED",
|
||||||
|
"environments": [
|
||||||
|
{
|
||||||
|
"environment_key": "test_env",
|
||||||
|
"spec": {
|
||||||
|
"client": "1",
|
||||||
|
"dependencies": [
|
||||||
|
"/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"format": "MULTI_TASK",
|
||||||
|
"max_concurrent_runs": 1,
|
||||||
|
"name": "My Wheel Job",
|
||||||
|
"queue": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"tasks": [
|
||||||
|
{
|
||||||
|
"environment_key": "test_env",
|
||||||
|
"existing_cluster_id": "0717-132531-5opeqon1",
|
||||||
|
"python_wheel_task": {
|
||||||
|
"entry_point": "run",
|
||||||
|
"package_name": "my_test_code"
|
||||||
|
},
|
||||||
|
"task_key": "TestTask"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
=== Expecting 1 wheel to be uploaded
|
||||||
|
>>> jq .path
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
|
||||||
|
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
|
|
@ -0,0 +1,11 @@
|
||||||
|
trace $CLI bundle deploy
|
||||||
|
|
||||||
|
trace find.py --expect 1 whl
|
||||||
|
|
||||||
|
title "Expecting 1 wheel in environments section in /jobs/create"
|
||||||
|
trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt
|
||||||
|
|
||||||
|
title "Expecting 1 wheel to be uploaded"
|
||||||
|
trace jq .path < out.requests.txt | grep import | grep whl | sort
|
||||||
|
|
||||||
|
rm out.requests.txt
|
|
@ -9,11 +9,6 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEnvironmentKeySupported(t *testing.T) {
|
|
||||||
_, diags := loadTargetWithDiags("./python_wheel/environment_key", "default")
|
|
||||||
require.Empty(t, diags)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) {
|
func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) {
|
||||||
b, diags := loadTargetWithDiags("./environment_key_only", "default")
|
b, diags := loadTargetWithDiags("./environment_key_only", "default")
|
||||||
require.Empty(t, diags)
|
require.Empty(t, diags)
|
||||||
|
|
|
@ -1,144 +0,0 @@
|
||||||
package config_tests
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/libraries"
|
|
||||||
"github.com/databricks/cli/bundle/phases"
|
|
||||||
mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
|
|
||||||
"github.com/databricks/cli/libs/filer"
|
|
||||||
"github.com/stretchr/testify/mock"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPythonWheelBuild(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, matches, 1)
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelBuildAutoDetect(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, matches, 1)
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, matches, 1)
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelWithDBFSLib(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
mockFiler := mockfiler.NewMockFiler(t)
|
|
||||||
mockFiler.EXPECT().Write(
|
|
||||||
mock.Anything,
|
|
||||||
filepath.Join("my_test_code-0.0.1-py3-none-any.whl"),
|
|
||||||
mock.AnythingOfType("*os.File"),
|
|
||||||
filer.OverwriteIfExists,
|
|
||||||
filer.CreateParentDirectories,
|
|
||||||
).Return(nil)
|
|
||||||
|
|
||||||
diags = bundle.ApplySeq(ctx, b,
|
|
||||||
libraries.ExpandGlobReferences(),
|
|
||||||
libraries.UploadWithClient(mockFiler),
|
|
||||||
)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
require.Empty(t, diags)
|
|
||||||
require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/environment_key", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, matches, 1)
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelBuildMultiple(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, matches, 2)
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPythonWheelNoBuild(t *testing.T) {
|
|
||||||
b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default")
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := phases.Build(ctx, b)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
|
|
||||||
match := libraries.ExpandGlobReferences()
|
|
||||||
diags = bundle.Apply(ctx, b, match)
|
|
||||||
require.NoError(t, diags.Error())
|
|
||||||
}
|
|
Loading…
Reference in New Issue