From 8f8f24c3a9fa1cdd50613893e56efe65d524a047 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 3 Mar 2025 12:09:25 +0100 Subject: [PATCH] Convert python wheel tests to acceptance (#2396) ## Changes Rewrite bundle/tests/python_wheel_test.go into acceptance tests. The same configs are used, but the test now runs 'bundle deploy' and in addition to checking the files on the file system, also checks that the files were uploaded and records jobs/create request. There is a new test helper bin/find.py which filters out paths based on regex, asserts on number of expected results. I've added it because 'find' on Windows behaves differently, so this helps avoid cross-platform differences. --- acceptance/bin/find.py | 36 +++++ .../artifacts/same_name_libraries/test.toml | 2 + acceptance/bundle/artifacts/script.prepare | 9 ++ acceptance/bundle/artifacts/test.toml | 18 +++ .../artifacts/unique_name_libraries/test.toml | 2 +- .../bundle/artifacts/whl_dbfs/databricks.yml | 0 .../bundle/artifacts/whl_dbfs/output.txt | 32 ++++ acceptance/bundle/artifacts/whl_dbfs/script | 12 ++ .../bundle/artifacts/whl_explicit}/.gitignore | 0 .../artifacts/whl_explicit/databricks.yml | 3 +- .../whl_explicit}/my_test_code/setup.py | 0 .../my_test_code/src/__init__.py | 0 .../my_test_code/src/__main__.py | 0 .../bundle/artifacts/whl_explicit/output.txt | 34 +++++ .../bundle/artifacts/whl_explicit/script | 11 ++ .../bundle/artifacts/whl_implicit}/.gitignore | 0 .../artifacts/whl_implicit/databricks.yml | 0 .../whl_implicit/my_test_code}/__init__.py | 0 .../whl_implicit/my_test_code}/__main__.py | 0 .../bundle/artifacts/whl_implicit/output.txt | 34 +++++ .../bundle/artifacts/whl_implicit/script | 11 ++ .../bundle/artifacts/whl_implicit}/setup.py | 0 .../whl_implicit_custom_path}/.gitignore | 0 .../whl_implicit_custom_path/databricks.yml | 0 .../whl_implicit_custom_path/output.txt | 46 ++++++ .../my_test_code-0.0.1-py3-none-any.whl | Bin .../artifacts/whl_implicit_custom_path/script | 11 ++ .../whl_implicit_notebook}/.gitignore | 0 .../whl_implicit_notebook/databricks.yml | 0 .../my_test_code}/__init__.py | 0 .../my_test_code}/__main__.py | 0 .../whl_implicit_notebook}/notebook.py | 0 .../whl_implicit_notebook/output.txt | 33 ++++ .../artifacts/whl_implicit_notebook/script | 11 ++ .../artifacts/whl_implicit_notebook}/setup.py | 0 .../bundle/artifacts/whl_multiple}/.gitignore | 0 .../artifacts/whl_multiple/databricks.yml | 4 +- .../whl_multiple}/my_test_code/setup.py | 0 .../whl_multiple}/my_test_code/setup2.py | 0 .../my_test_code/src}/__init__.py | 0 .../my_test_code/src}/__main__.py | 0 .../bundle/artifacts/whl_multiple/output.txt | 42 +++++ .../bundle/artifacts/whl_multiple/script | 11 ++ .../whl_prebuilt_multiple}/.gitignore | 0 .../whl_prebuilt_multiple/databricks.yml | 2 +- .../other_test_code-0.0.1-py3-none-any.whl | Bin .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin .../whl_prebuilt_multiple/output.txt | 45 ++++++ .../artifacts/whl_prebuilt_multiple/script | 14 ++ .../whl_via_environment_key}/.gitignore | 0 .../whl_via_environment_key}/databricks.yml | 2 +- .../my_test_code/setup.py | 0 .../my_test_code/src}/__init__.py | 0 .../my_test_code/src}/__main__.py | 0 .../whl_via_environment_key/output.txt | 54 +++++++ .../artifacts/whl_via_environment_key/script | 11 ++ bundle/tests/enviroment_key_test.go | 5 - bundle/tests/python_wheel_test.go | 144 ------------------ 58 files changed, 484 insertions(+), 155 deletions(-) create mode 100755 acceptance/bin/find.py create mode 100644 acceptance/bundle/artifacts/script.prepare create mode 100644 acceptance/bundle/artifacts/test.toml rename bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml => acceptance/bundle/artifacts/whl_dbfs/databricks.yml (100%) create mode 100644 acceptance/bundle/artifacts/whl_dbfs/output.txt create mode 100644 acceptance/bundle/artifacts/whl_dbfs/script rename {bundle/tests/python_wheel/environment_key => acceptance/bundle/artifacts/whl_explicit}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel/bundle.yml => acceptance/bundle/artifacts/whl_explicit/databricks.yml (77%) rename {bundle/tests/python_wheel/environment_key => acceptance/bundle/artifacts/whl_explicit}/my_test_code/setup.py (100%) rename {bundle/tests/python_wheel/environment_key => acceptance/bundle/artifacts/whl_explicit}/my_test_code/src/__init__.py (100%) rename {bundle/tests/python_wheel/environment_key => acceptance/bundle/artifacts/whl_explicit}/my_test_code/src/__main__.py (100%) create mode 100644 acceptance/bundle/artifacts/whl_explicit/output.txt create mode 100644 acceptance/bundle/artifacts/whl_explicit/script rename {bundle/tests/python_wheel/python_wheel => acceptance/bundle/artifacts/whl_implicit}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml => acceptance/bundle/artifacts/whl_implicit/databricks.yml (100%) rename {bundle/tests/python_wheel/python_wheel/my_test_code/src => acceptance/bundle/artifacts/whl_implicit/my_test_code}/__init__.py (100%) rename {bundle/tests/python_wheel/python_wheel/my_test_code/src => acceptance/bundle/artifacts/whl_implicit/my_test_code}/__main__.py (100%) create mode 100644 acceptance/bundle/artifacts/whl_implicit/output.txt create mode 100644 acceptance/bundle/artifacts/whl_implicit/script rename {bundle/tests/python_wheel/python_wheel_no_artifact => acceptance/bundle/artifacts/whl_implicit}/setup.py (100%) rename {bundle/tests/python_wheel/python_wheel_multiple => acceptance/bundle/artifacts/whl_implicit_custom_path}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml => acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml (100%) create mode 100644 acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt rename {bundle/tests/python_wheel/python_wheel_no_artifact_no_setup => acceptance/bundle/artifacts/whl_implicit_custom_path}/package/my_test_code-0.0.1-py3-none-any.whl (100%) create mode 100644 acceptance/bundle/artifacts/whl_implicit_custom_path/script rename {bundle/tests/python_wheel/python_wheel_no_artifact => acceptance/bundle/artifacts/whl_implicit_notebook}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml => acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml (100%) rename {bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src => acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code}/__init__.py (100%) rename {bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src => acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code}/__main__.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact_notebook => acceptance/bundle/artifacts/whl_implicit_notebook}/notebook.py (100%) create mode 100644 acceptance/bundle/artifacts/whl_implicit_notebook/output.txt create mode 100644 acceptance/bundle/artifacts/whl_implicit_notebook/script rename {bundle/tests/python_wheel/python_wheel_no_artifact_notebook => acceptance/bundle/artifacts/whl_implicit_notebook}/setup.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact_no_setup => acceptance/bundle/artifacts/whl_multiple}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel_multiple/bundle.yml => acceptance/bundle/artifacts/whl_multiple/databricks.yml (85%) rename {bundle/tests/python_wheel/python_wheel => acceptance/bundle/artifacts/whl_multiple}/my_test_code/setup.py (100%) rename {bundle/tests/python_wheel/python_wheel_multiple => acceptance/bundle/artifacts/whl_multiple}/my_test_code/setup2.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code => acceptance/bundle/artifacts/whl_multiple/my_test_code/src}/__init__.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code => acceptance/bundle/artifacts/whl_multiple/my_test_code/src}/__main__.py (100%) create mode 100644 acceptance/bundle/artifacts/whl_multiple/output.txt create mode 100644 acceptance/bundle/artifacts/whl_multiple/script rename {bundle/tests/python_wheel/python_wheel_no_artifact_notebook => acceptance/bundle/artifacts/whl_prebuilt_multiple}/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel_no_build/bundle.yml => acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml (84%) rename bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl => acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl (100%) rename {bundle/tests/python_wheel/python_wheel_no_build => acceptance/bundle/artifacts/whl_prebuilt_multiple}/dist/my_test_code-0.0.1-py3-none-any.whl (100%) create mode 100644 acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt create mode 100644 acceptance/bundle/artifacts/whl_prebuilt_multiple/script rename {bundle/tests/python_wheel/python_wheel_no_build => acceptance/bundle/artifacts/whl_via_environment_key}/.gitignore (100%) rename {bundle/tests/python_wheel/environment_key => acceptance/bundle/artifacts/whl_via_environment_key}/databricks.yml (92%) rename {bundle/tests/python_wheel/python_wheel_multiple => acceptance/bundle/artifacts/whl_via_environment_key}/my_test_code/setup.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code => acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src}/__init__.py (100%) rename {bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code => acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src}/__main__.py (100%) create mode 100644 acceptance/bundle/artifacts/whl_via_environment_key/output.txt create mode 100644 acceptance/bundle/artifacts/whl_via_environment_key/script delete mode 100644 bundle/tests/python_wheel_test.go diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py new file mode 100755 index 000000000..d122404b2 --- /dev/null +++ b/acceptance/bin/find.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +Usage: find.py +Finds all files within current directory matching regex. The output is sorted and slashes are always forward. + +If --expect N is provided, the number of matches must be N or error is printed. +""" + +import sys +import os +import re +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument("regex") +parser.add_argument("--expect", type=int) +args = parser.parse_args() + +regex = re.compile(args.regex) +result = [] + +for root, dirs, files in os.walk("."): + for filename in files: + path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/") + if regex.search(path): + result.append(path) + +result.sort() +for item in result: + print(item) +sys.stdout.flush() + +if args.expect is not None: + if args.expect != len(result): + sys.exit(f"Expected {args.expect}, got {len(result)}") diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml index a298217f2..a17f2659f 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/test.toml @@ -1,3 +1,5 @@ +RecordRequests = false + [[Repls]] Old = '\\' New = '/' diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare new file mode 100644 index 000000000..673b20af9 --- /dev/null +++ b/acceptance/bundle/artifacts/script.prepare @@ -0,0 +1,9 @@ +export PYTHONDONTWRITEBYTECODE=1 + +uv venv -q --python 3.12 .venv +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + source .venv/Scripts/activate +else + source .venv/bin/activate +fi +uv pip install -q setuptools diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml new file mode 100644 index 000000000..42432706e --- /dev/null +++ b/acceptance/bundle/artifacts/test.toml @@ -0,0 +1,18 @@ +Cloud = false +RecordRequests = true +Ignore = [ + '.venv', + 'dist', + 'build', + '*egg-info', + '.databricks', +] + +[[Server]] +Pattern = "GET /api/2.1/clusters/get" +Response.Body = ''' +{ + "cluster_id": "0717-132531-5opeqon1", + "spark_version": "13.3.x-scala2.12" +} +''' diff --git a/acceptance/bundle/artifacts/unique_name_libraries/test.toml b/acceptance/bundle/artifacts/unique_name_libraries/test.toml index 280338bd6..f548ef4fc 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/test.toml @@ -1,4 +1,4 @@ -Cloud = false +RecordRequests = false # The order in which files are uploaded can be different, so we just replace the name [[Repls]] diff --git a/bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml b/acceptance/bundle/artifacts/whl_dbfs/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml rename to acceptance/bundle/artifacts/whl_dbfs/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt new file mode 100644 index 000000000..f0615c558 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -0,0 +1,32 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Expecting to find no wheels +>>> errcode find.py --expect 0 whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "dbfs:/path/to/dist/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting no wheels to be uploaded +>>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script new file mode 100644 index 000000000..d3b12cb8e --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -0,0 +1,12 @@ +trace $CLI bundle deploy + +title "Expecting to find no wheels" +trace errcode find.py --expect 0 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting no wheels to be uploaded" +trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/environment_key/.gitignore b/acceptance/bundle/artifacts/whl_explicit/.gitignore similarity index 100% rename from bundle/tests/python_wheel/environment_key/.gitignore rename to acceptance/bundle/artifacts/whl_explicit/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel/bundle.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml similarity index 77% rename from bundle/tests/python_wheel/python_wheel/bundle.yml rename to acceptance/bundle/artifacts/whl_explicit/databricks.yml index 017fe1c43..45106c0b0 100644 --- a/bundle/tests/python_wheel/python_wheel/bundle.yml +++ b/acceptance/bundle/artifacts/whl_explicit/databricks.yml @@ -5,7 +5,8 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + # using 'python' there because 'python3' does not exist in virtualenv on windows + build: python setup.py bdist_wheel resources: jobs: diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt new file mode 100644 index 000000000..1018501db --- /dev/null +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script new file mode 100644 index 000000000..bb7e26ae1 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel/.gitignore b/acceptance/bundle/artifacts/whl_implicit/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel/.gitignore rename to acceptance/bundle/artifacts/whl_implicit/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml b/acceptance/bundle/artifacts/whl_implicit/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit/databricks.yml diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt new file mode 100644 index 000000000..69ff56c42 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script new file mode 100644 index 000000000..78c4d75e0 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/setup.py b/acceptance/bundle/artifacts/whl_implicit/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/setup.py rename to acceptance/bundle/artifacts/whl_implicit/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt new file mode 100644 index 000000000..0658dce3a --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -0,0 +1,46 @@ + +>>> [CLI] bundle deploy +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +package/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "[default] My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files/package/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script new file mode 100644 index 000000000..d4c2438db --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore b/acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt new file mode 100644 index 000000000..9c7296b9a --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/notebook.py" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script new file mode 100644 index 000000000..bb7e26ae1 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/acceptance/bundle/artifacts/whl_implicit_notebook/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore b/acceptance/bundle/artifacts/whl_multiple/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore rename to acceptance/bundle/artifacts/whl_multiple/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/acceptance/bundle/artifacts/whl_multiple/databricks.yml similarity index 85% rename from bundle/tests/python_wheel/python_wheel_multiple/bundle.yml rename to acceptance/bundle/artifacts/whl_multiple/databricks.yml index 770110416..2bfd85abd 100644 --- a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml +++ b/acceptance/bundle/artifacts/whl_multiple/databricks.yml @@ -5,11 +5,11 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: "python setup.py bdist_wheel" my_test_code_2: type: whl path: "./my_test_code" - build: "python3 setup2.py bdist_wheel" + build: "python setup2.py bdist_wheel" resources: jobs: diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt new file mode 100644 index 000000000..9335b9cc5 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -0,0 +1,42 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Building my_test_code_2... +Deploying resources... +Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code_2-0.0.1-py3-none-any.whl... + +>>> find.py --expect 2 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script new file mode 100644 index 000000000..a475e9f73 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order + +trace find.py --expect 2 whl + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm -fr out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml similarity index 84% rename from bundle/tests/python_wheel/python_wheel_no_build/bundle.yml rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml index e10e3993d..4ad0c6afa 100644 --- a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml @@ -13,4 +13,4 @@ resources: entry_point: "run" libraries: - whl: ./dist/*.whl - - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl + - whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt new file mode 100644 index 000000000..c9cd82fb8 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -0,0 +1,45 @@ + +>>> find.py --expect 2 whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl + +>>> [CLI] bundle deploy +Deploying resources... +Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading other_test_code-0.0.1-py3-none-any.whl... + +=== Expecting to find 2 wheels, same as initially provided +>>> find.py --expect 2 whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script new file mode 100644 index 000000000..5265151e2 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -0,0 +1,14 @@ +trace find.py --expect 2 whl + +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order + +title "Expecting to find 2 wheels, same as initially provided" +trace find.py --expect 2 whl + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/acceptance/bundle/artifacts/whl_via_environment_key/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/.gitignore rename to acceptance/bundle/artifacts/whl_via_environment_key/.gitignore diff --git a/bundle/tests/python_wheel/environment_key/databricks.yml b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml similarity index 92% rename from bundle/tests/python_wheel/environment_key/databricks.yml rename to acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml index 198f8c0d2..4ca3f9113 100644 --- a/bundle/tests/python_wheel/environment_key/databricks.yml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: python setup.py bdist_wheel resources: jobs: diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt new file mode 100644 index 000000000..8b6b781aa --- /dev/null +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -0,0 +1,54 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in environments section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "test_env", + "spec": { + "client": "1", + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + ] + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "environment_key": "test_env", + "existing_cluster_id": "0717-132531-5opeqon1", + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script new file mode 100644 index 000000000..1914aeb8c --- /dev/null +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in environments section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index 135ef1917..c22059313 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -9,11 +9,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestEnvironmentKeySupported(t *testing.T) { - _, diags := loadTargetWithDiags("./python_wheel/environment_key", "default") - require.Empty(t, diags) -} - func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go deleted file mode 100644 index 22702ec44..000000000 --- a/bundle/tests/python_wheel_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package config_tests - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/phases" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/libs/filer" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestPythonWheelBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildAutoDetect(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelWithDBFSLib(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), - mock.AnythingOfType("*os.File"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - diags = bundle.ApplySeq(ctx, b, - libraries.ExpandGlobReferences(), - libraries.UploadWithClient(mockFiler), - ) - require.NoError(t, diags.Error()) - require.Empty(t, diags) - require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) -} - -func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { - b := loadTarget(t, "./python_wheel/environment_key", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildMultiple(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") - require.NoError(t, err) - require.Len(t, matches, 2) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelNoBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -}