acc: Add test for stale wheel problem (#2477)

This tests record current situation with wheel refresh on databricks
clusters.

See #2427 for background on "stale wheel" problem.

acceptance/bin/update_file.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""
+Usage: update_file.py FILENAME OLD NEW
+
+Replace all strings OLD with NEW in FILENAME.
+
+If OLD is not found in FILENAME, the script reports error.
+"""
+
+import sys
+
+filename, old, new = sys.argv[1:]
+data = open(filename).read()
+newdata = data.replace(old, new)
+if newdata == data:
+    sys.exit(f"{old=} not found in {filename=}\n{data}")
+with open(filename, "w") as fobj:
+    fobj.write(newdata)

acceptance/bundle/integration_whl/base/output.txt

@@ -47,6 +47,24 @@ Hello from my func
 Got arguments:
 ['my_test_code', 'one', 'two']
 
+=== Make a change to code without version change and run the job again
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading my_test_code-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+UPDATED MY FUNC
+Got arguments:
+['my_test_code', 'one', 'two']
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete job some_other_job

acceptance/bundle/integration_whl/base/script

@@ -4,3 +4,8 @@ trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy
 trace $CLI bundle run some_other_job
+
+title "Make a change to code without version change and run the job again"
+update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC'
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job

acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl

@@ -11,7 +11,7 @@ resources:
       spark_version: "$DEFAULT_SPARK_VERSION"
       node_type_id: "$NODE_TYPE_ID"
       num_workers: 1
-      data_security_mode: USER_ISOLATION
+      data_security_mode: $DATA_SECURITY_MODE
 
   jobs:
     some_other_job:

acceptance/bundle/integration_whl/interactive_cluster/output.txt

@@ -47,6 +47,24 @@ Hello from my func
 Got arguments:
 ['my_test_code', 'one', 'two']
 
+=== Make a change to code without version change and run the job again
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading my_test_code-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+Hello from my func
+Got arguments:
+['my_test_code', 'one', 'two']
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete cluster test_cluster

acceptance/bundle/integration_whl/interactive_cluster/script

@@ -1,5 +1,11 @@
+export DATA_SECURITY_MODE=USER_ISOLATION
 envsubst < databricks.yml.tmpl > databricks.yml
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy
 trace $CLI bundle run some_other_job
+
+title "Make a change to code without version change and run the job again"
+update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC'
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job

acceptance/bundle/integration_whl/interactive_single_user/output.txt

@@ -0,0 +1,76 @@
+
+>>> cat databricks.yml
+bundle:
+  name: wheel-task
+
+workspace:
+  root_path: "~/.bundle/[UNIQUE_NAME]"
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: "test-cluster-[UNIQUE_NAME]"
+      spark_version: "13.3.x-snapshot-scala2.12"
+      node_type_id: "[NODE_TYPE_ID]"
+      num_workers: 1
+      data_security_mode: SINGLE_USER
+
+  jobs:
+    some_other_job:
+      name: "[${bundle.target}] Test Wheel Job [UNIQUE_NAME]"
+      tasks:
+        - task_key: TestTask
+          existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}"
+          python_wheel_task:
+            package_name: my_test_code
+            entry_point: run
+            parameters:
+              - "one"
+              - "two"
+          libraries:
+            - whl: ./dist/*.whl
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading my_test_code-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+Hello from my func
+Got arguments:
+['my_test_code', 'one', 'two']
+
+=== Make a change to code without version change and run the job again
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading my_test_code-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+UPDATED MY FUNC
+Got arguments:
+['my_test_code', 'one', 'two']
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete cluster test_cluster
+  delete job some_other_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!

acceptance/bundle/integration_whl/interactive_single_user/script

@@ -0,0 +1,12 @@
+export DATA_SECURITY_MODE=SINGLE_USER
+envsubst < $TESTDIR/../interactive_cluster/databricks.yml.tmpl > databricks.yml
+trace cat databricks.yml
+cp -r $TESTDIR/../interactive_cluster/{setup.py,my_test_code} .
+trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job
+
+title "Make a change to code without version change and run the job again"
+update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC'
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job