diff --git a/acceptance/bin/update_file.py b/acceptance/bin/update_file.py new file mode 100755 index 000000000..24e340ed0 --- /dev/null +++ b/acceptance/bin/update_file.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Usage: update_file.py FILENAME OLD NEW + +Replace all strings OLD with NEW in FILENAME. + +If OLD is not found in FILENAME, the script reports error. +""" + +import sys + +filename, old, new = sys.argv[1:] +data = open(filename).read() +newdata = data.replace(old, new) +if newdata == data: + sys.exit(f"{old=} not found in {filename=}\n{data}") +with open(filename, "w") as fobj: + fobj.write(newdata) diff --git a/acceptance/bundle/integration_whl/base/output.txt b/acceptance/bundle/integration_whl/base/output.txt index 34ae35e40..510ef79a6 100644 --- a/acceptance/bundle/integration_whl/base/output.txt +++ b/acceptance/bundle/integration_whl/base/output.txt @@ -47,6 +47,24 @@ Hello from my func Got arguments: ['my_test_code', 'one', 'two'] +=== Make a change to code without version change and run the job again +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +UPDATED MY FUNC +Got arguments: +['my_test_code', 'one', 'two'] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete job some_other_job diff --git a/acceptance/bundle/integration_whl/base/script b/acceptance/bundle/integration_whl/base/script index 4e5e5f360..7db0843b7 100644 --- a/acceptance/bundle/integration_whl/base/script +++ b/acceptance/bundle/integration_whl/base/script @@ -4,3 +4,8 @@ trace cat databricks.yml trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle deploy trace $CLI bundle run some_other_job + +title "Make a change to code without version change and run the job again" +update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC' +trace $CLI bundle deploy +trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl index 85e44eb6a..d59e9a432 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl +++ b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl @@ -11,7 +11,7 @@ resources: spark_version: "$DEFAULT_SPARK_VERSION" node_type_id: "$NODE_TYPE_ID" num_workers: 1 - data_security_mode: USER_ISOLATION + data_security_mode: $DATA_SECURITY_MODE jobs: some_other_job: diff --git a/acceptance/bundle/integration_whl/interactive_cluster/output.txt b/acceptance/bundle/integration_whl/interactive_cluster/output.txt index be1d9e375..f8cabd89d 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster/output.txt @@ -47,6 +47,24 @@ Hello from my func Got arguments: ['my_test_code', 'one', 'two'] +=== Make a change to code without version change and run the job again +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'one', 'two'] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete cluster test_cluster diff --git a/acceptance/bundle/integration_whl/interactive_cluster/script b/acceptance/bundle/integration_whl/interactive_cluster/script index 84498a7c9..6224f1d24 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/script +++ b/acceptance/bundle/integration_whl/interactive_cluster/script @@ -1,5 +1,11 @@ +export DATA_SECURITY_MODE=USER_ISOLATION envsubst < databricks.yml.tmpl > databricks.yml trace cat databricks.yml trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle deploy trace $CLI bundle run some_other_job + +title "Make a change to code without version change and run the job again" +update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC' +trace $CLI bundle deploy +trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/interactive_single_user/output.txt b/acceptance/bundle/integration_whl/interactive_single_user/output.txt new file mode 100644 index 000000000..9e39998cc --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_single_user/output.txt @@ -0,0 +1,76 @@ + +>>> cat databricks.yml +bundle: + name: wheel-task + +workspace: + root_path: "~/.bundle/[UNIQUE_NAME]" + +resources: + clusters: + test_cluster: + cluster_name: "test-cluster-[UNIQUE_NAME]" + spark_version: "13.3.x-snapshot-scala2.12" + node_type_id: "[NODE_TYPE_ID]" + num_workers: 1 + data_security_mode: SINGLE_USER + + jobs: + some_other_job: + name: "[${bundle.target}] Test Wheel Job [UNIQUE_NAME]" + tasks: + - task_key: TestTask + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + python_wheel_task: + package_name: my_test_code + entry_point: run + parameters: + - "one" + - "two" + libraries: + - whl: ./dist/*.whl + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'one', 'two'] + +=== Make a change to code without version change and run the job again +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +UPDATED MY FUNC +Got arguments: +['my_test_code', 'one', 'two'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/interactive_single_user/script b/acceptance/bundle/integration_whl/interactive_single_user/script new file mode 100644 index 000000000..b7803c293 --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_single_user/script @@ -0,0 +1,12 @@ +export DATA_SECURITY_MODE=SINGLE_USER +envsubst < $TESTDIR/../interactive_cluster/databricks.yml.tmpl > databricks.yml +trace cat databricks.yml +cp -r $TESTDIR/../interactive_cluster/{setup.py,my_test_code} . +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy +trace $CLI bundle run some_other_job + +title "Make a change to code without version change and run the job again" +update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC' +trace $CLI bundle deploy +trace $CLI bundle run some_other_job