acc: Simplify integration_whl tests (#2479)

## Changes
Instead of full templates, only template databricks.yml and use envsubst
for that.

acceptance/bundle/integration_whl/base/databricks.yml.tmpl

@@ -2,25 +2,23 @@ bundle:
   name: wheel-task
 
 workspace:
-  root_path: "~/.bundle/{{.unique_id}}"
+  root_path: "~/.bundle/$UNIQUE_NAME"
 
-{{if .python_wheel_wrapper}}
+include:
-experimental:
+  - $EXTRA_CONFIG
-  python_wheel_wrapper: true
-{{end}}
 
 resources:
   jobs:
     some_other_job:
-      name: "[${bundle.target}] Test Wheel Job {{.unique_id}}"
+      name: "[${bundle.target}] Test Wheel Job $UNIQUE_NAME"
       tasks:
         - task_key: TestTask
           new_cluster:
             num_workers: 1
-            spark_version: "{{.spark_version}}"
+            spark_version: $DEFAULT_SPARK_VERSION
-            node_type_id: "{{.node_type_id}}"
+            node_type_id: $NODE_TYPE_ID
             data_security_mode: USER_ISOLATION
-            instance_pool_id: "{{.instance_pool_id}}"
+            instance_pool_id: $TEST_INSTANCE_POOL_ID
           python_wheel_task:
             package_name: my_test_code
             entry_point: run

acceptance/bundle/integration_whl/base/databricks_template_schema.json

@@ -1,29 +0,0 @@
-{
-    "properties": {
-        "project_name": {
-            "type": "string",
-            "default": "my_test_code",
-            "description": "Unique name for this project"
-        },
-        "spark_version": {
-            "type": "string",
-            "description": "Spark version used for job cluster"
-        },
-        "node_type_id": {
-            "type": "string",
-            "description": "Node type id for job cluster"
-        },
-        "unique_id": {
-            "type": "string",
-            "description": "Unique ID for job name"
-        },
-        "python_wheel_wrapper": {
-            "type": "boolean",
-            "description": "Whether or not to enable python wheel wrapper"
-        },
-        "instance_pool_id": {
-            "type": "string",
-            "description": "Instance pool id for job cluster"
-        }
-    }
-}

acceptance/bundle/integration_whl/base/output.txt

@@ -1,15 +1,4 @@
 
->>> cat input.json
-{
-    "project_name": "my_test_code",
-    "spark_version": "13.3.x-snapshot-scala2.12",
-    "node_type_id": "[NODE_TYPE_ID]",
-    "unique_id": "[UNIQUE_NAME]",
-    "python_wheel_wrapper": false,
-    "instance_pool_id": "[TEST_INSTANCE_POOL_ID]"
-}
-✨ Successfully initialized template
-
 >>> cat databricks.yml
 bundle:
   name: wheel-task
@@ -17,7 +6,8 @@ bundle:
 workspace:
   root_path: "~/.bundle/[UNIQUE_NAME]"
 
-
+include:
+  - empty.yml
 
 resources:
   jobs:
@@ -27,10 +17,10 @@ resources:
         - task_key: TestTask
           new_cluster:
             num_workers: 1
-            spark_version: "13.3.x-snapshot-scala2.12"
+            spark_version: 13.3.x-snapshot-scala2.12
-            node_type_id: "[NODE_TYPE_ID]"
+            node_type_id: [NODE_TYPE_ID]
             data_security_mode: USER_ISOLATION
-            instance_pool_id: "[TEST_INSTANCE_POOL_ID]"
+            instance_pool_id: [TEST_INSTANCE_POOL_ID]
           python_wheel_task:
             package_name: my_test_code
             entry_point: run

acceptance/bundle/integration_whl/base/python_wheel_wrapper.yml

@@ -0,0 +1,2 @@
+experimental:
+  python_wheel_wrapper: true

acceptance/bundle/integration_whl/base/script

@@ -1,8 +1,5 @@
-export SPARK_VERSION=$DEFAULT_SPARK_VERSION
+export EXTRA_CONFIG=empty.yml
-export PYTHON_WHEEL_WRAPPER=false
+envsubst < databricks.yml.tmpl > databricks.yml
-envsubst < input.json.tmpl > input.json
-trace cat input.json
-$CLI bundle init . --config-file input.json
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy

acceptance/bundle/integration_whl/base/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import my_test_code
+
+setup(
+    name="my_test_code",
+    version=my_test_code.__version__,
+    author=my_test_code.__author__,
+    url="https://databricks.com",
+    author_email="john.doe@databricks.com",
+    description="my example wheel",
+    packages=find_packages(include=["my_test_code"]),
+    entry_points={"group1": "run=my_test_code.__main__:main"},
+    install_requires=["setuptools"],
+)

acceptance/bundle/integration_whl/base/template/setup.py.tmpl

@@ -1,15 +0,0 @@
-from setuptools import setup, find_packages
-
-import {{.project_name}}
-
-setup(
-    name="{{.project_name}}",
-    version={{.project_name}}.__version__,
-    author={{.project_name}}.__author__,
-    url="https://databricks.com",
-    author_email="john.doe@databricks.com",
-    description="my example wheel",
-    packages=find_packages(include=["{{.project_name}}"]),
-    entry_points={"group1": "run={{.project_name}}.__main__:main"},
-    install_requires=["setuptools"],
-)

acceptance/bundle/integration_whl/custom_params/output.txt

@@ -1,15 +1,4 @@
 
->>> cat input.json
-{
-    "project_name": "my_test_code",
-    "spark_version": "13.3.x-snapshot-scala2.12",
-    "node_type_id": "[NODE_TYPE_ID]",
-    "unique_id": "[UNIQUE_NAME]",
-    "python_wheel_wrapper": false,
-    "instance_pool_id": "[TEST_INSTANCE_POOL_ID]"
-}
-✨ Successfully initialized template
-
 >>> cat databricks.yml
 bundle:
   name: wheel-task
@@ -17,7 +6,8 @@ bundle:
 workspace:
   root_path: "~/.bundle/[UNIQUE_NAME]"
 
-
+include:
+  - empty.yml
 
 resources:
   jobs:
@@ -27,10 +17,10 @@ resources:
         - task_key: TestTask
           new_cluster:
             num_workers: 1
-            spark_version: "13.3.x-snapshot-scala2.12"
+            spark_version: 13.3.x-snapshot-scala2.12
-            node_type_id: "[NODE_TYPE_ID]"
+            node_type_id: [NODE_TYPE_ID]
             data_security_mode: USER_ISOLATION
-            instance_pool_id: "[TEST_INSTANCE_POOL_ID]"
+            instance_pool_id: [TEST_INSTANCE_POOL_ID]
           python_wheel_task:
             package_name: my_test_code
             entry_point: run

acceptance/bundle/integration_whl/custom_params/script

@@ -1,8 +1,6 @@
-export SPARK_VERSION=$DEFAULT_SPARK_VERSION
+export EXTRA_CONFIG=empty.yml
-export PYTHON_WHEEL_WRAPPER=false
+envsubst < $TESTDIR/../base/databricks.yml.tmpl > databricks.yml
-envsubst < $TESTDIR/../base/input.json.tmpl > input.json
+cp -r $TESTDIR/../base/{$EXTRA_CONFIG,setup.py,my_test_code} .
-trace cat input.json
-$CLI bundle init $TESTDIR/../base --config-file input.json
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy

acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl

@@ -2,20 +2,20 @@ bundle:
   name: wheel-task
 
 workspace:
-  root_path: "~/.bundle/{{.unique_id}}"
+  root_path: "~/.bundle/$UNIQUE_NAME"
 
 resources:
   clusters:
     test_cluster:
-      cluster_name: "test-cluster-{{.unique_id}}"
+      cluster_name: "test-cluster-$UNIQUE_NAME"
-      spark_version: "{{.spark_version}}"
+      spark_version: "$DEFAULT_SPARK_VERSION"
-      node_type_id: "{{.node_type_id}}"
+      node_type_id: "$NODE_TYPE_ID"
       num_workers: 1
       data_security_mode: USER_ISOLATION
 
   jobs:
     some_other_job:
-      name: "[${bundle.target}] Test Wheel Job {{.unique_id}}"
+      name: "[${bundle.target}] Test Wheel Job $UNIQUE_NAME"
       tasks:
         - task_key: TestTask
           existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}"

acceptance/bundle/integration_whl/interactive_cluster/databricks_template_schema.json

@@ -1,25 +0,0 @@
-{
-    "properties": {
-        "project_name": {
-            "type": "string",
-            "default": "my_test_code",
-            "description": "Unique name for this project"
-        },
-        "spark_version": {
-            "type": "string",
-            "description": "Spark version used for job cluster"
-        },
-        "node_type_id": {
-            "type": "string",
-            "description": "Node type id for job cluster"
-        },
-        "unique_id": {
-            "type": "string",
-            "description": "Unique ID for job name"
-        },
-        "instance_pool_id": {
-            "type": "string",
-            "description": "Instance pool id for job cluster"
-        }
-    }
-}

acceptance/bundle/integration_whl/interactive_cluster/output.txt

@@ -1,15 +1,4 @@
 
->>> cat input.json
-{
-    "project_name": "my_test_code",
-    "spark_version": "13.3.x-snapshot-scala2.12",
-    "node_type_id": "[NODE_TYPE_ID]",
-    "unique_id": "[UNIQUE_NAME]",
-    "python_wheel_wrapper": false,
-    "instance_pool_id": "[TEST_INSTANCE_POOL_ID]"
-}
-✨ Successfully initialized template
-
 >>> cat databricks.yml
 bundle:
   name: wheel-task

acceptance/bundle/integration_whl/interactive_cluster/script

@@ -1,8 +1,4 @@
-export SPARK_VERSION=$DEFAULT_SPARK_VERSION
+envsubst < databricks.yml.tmpl > databricks.yml
-export PYTHON_WHEEL_WRAPPER=false
-envsubst < $TESTDIR/../base/input.json.tmpl > input.json
-trace cat input.json
-$CLI bundle init . --config-file input.json
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy

acceptance/bundle/integration_whl/interactive_cluster/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import my_test_code
+
+setup(
+    name="my_test_code",
+    version=my_test_code.__version__,
+    author=my_test_code.__author__,
+    url="https://databricks.com",
+    author_email="john.doe@databricks.com",
+    description="my example wheel",
+    packages=find_packages(include=["my_test_code"]),
+    entry_points={"group1": "run=my_test_code.__main__:main"},
+    install_requires=["setuptools"],
+)

acceptance/bundle/integration_whl/interactive_cluster/template/setup.py.tmpl

@@ -1,15 +0,0 @@
-from setuptools import setup, find_packages
-
-import {{.project_name}}
-
-setup(
-    name="{{.project_name}}",
-    version={{.project_name}}.__version__,
-    author={{.project_name}}.__author__,
-    url="https://databricks.com",
-    author_email="john.doe@databricks.com",
-    description="my example wheel",
-    packages=find_packages(include=["{{.project_name}}"]),
-    entry_points={"group1": "run={{.project_name}}.__main__:main"},
-    install_requires=["setuptools"],
-)

acceptance/bundle/integration_whl/test.toml

@@ -7,8 +7,9 @@ Ignore = [
     "my_test_code",
     "my_test_code.egg-info",
     "setup.py",
-    "input.json",
     "databricks.yml",
+    "python_wheel_wrapper.yml",
+    "empty.yml",
 ]
 
 [[Repls]]

acceptance/bundle/integration_whl/wrapper/output.txt

@@ -1,15 +1,4 @@
 
->>> cat input.json
-{
-    "project_name": "my_test_code",
-    "spark_version": "12.2.x-scala2.12",
-    "node_type_id": "[NODE_TYPE_ID]",
-    "unique_id": "[UNIQUE_NAME]",
-    "python_wheel_wrapper": true,
-    "instance_pool_id": "[TEST_INSTANCE_POOL_ID]"
-}
-✨ Successfully initialized template
-
 >>> cat databricks.yml
 bundle:
   name: wheel-task
@@ -17,10 +6,8 @@ bundle:
 workspace:
   root_path: "~/.bundle/[UNIQUE_NAME]"
 
-
+include:
-experimental:
+  - python_wheel_wrapper.yml
-  python_wheel_wrapper: true
-
 
 resources:
   jobs:
@@ -30,10 +17,10 @@ resources:
         - task_key: TestTask
           new_cluster:
             num_workers: 1
-            spark_version: "12.2.x-scala2.12"
+            spark_version: 12.2.x-scala2.12
-            node_type_id: "[NODE_TYPE_ID]"
+            node_type_id: [NODE_TYPE_ID]
             data_security_mode: USER_ISOLATION
-            instance_pool_id: "[TEST_INSTANCE_POOL_ID]"
+            instance_pool_id: [TEST_INSTANCE_POOL_ID]
           python_wheel_task:
             package_name: my_test_code
             entry_point: run

acceptance/bundle/integration_whl/wrapper/script

@@ -2,11 +2,10 @@
 # But before users used older DBRs and python wheel tasks but installed it from DBFS.
 # We still want to support older DBRs and did the trampoline workaround (https://github.com/databricks/cli/pull/635)
 # Hence this is to test that python wheel tasks in DABs are working for older DBRs
-export SPARK_VERSION=12.2.x-scala2.12
+export DEFAULT_SPARK_VERSION=12.2.x-scala2.12
-export PYTHON_WHEEL_WRAPPER=true
+export EXTRA_CONFIG=python_wheel_wrapper.yml
-envsubst < $TESTDIR/../base/input.json.tmpl > input.json
+envsubst < $TESTDIR/../base/databricks.yml.tmpl > databricks.yml
-trace cat input.json
+cp -r $TESTDIR/../base/{$EXTRA_CONFIG,setup.py,my_test_code} .
-$CLI bundle init $TESTDIR/../base --config-file input.json
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy

acceptance/bundle/integration_whl/wrapper_custom_params/output.txt

@@ -1,15 +1,4 @@
 
->>> cat input.json
-{
-    "project_name": "my_test_code",
-    "spark_version": "12.2.x-scala2.12",
-    "node_type_id": "[NODE_TYPE_ID]",
-    "unique_id": "[UNIQUE_NAME]",
-    "python_wheel_wrapper": true,
-    "instance_pool_id": "[TEST_INSTANCE_POOL_ID]"
-}
-✨ Successfully initialized template
-
 >>> cat databricks.yml
 bundle:
   name: wheel-task
@@ -17,10 +6,8 @@ bundle:
 workspace:
   root_path: "~/.bundle/[UNIQUE_NAME]"
 
-
+include:
-experimental:
+  - python_wheel_wrapper.yml
-  python_wheel_wrapper: true
-
 
 resources:
   jobs:
@@ -30,10 +17,10 @@ resources:
         - task_key: TestTask
           new_cluster:
             num_workers: 1
-            spark_version: "12.2.x-scala2.12"
+            spark_version: 12.2.x-scala2.12
-            node_type_id: "[NODE_TYPE_ID]"
+            node_type_id: [NODE_TYPE_ID]
             data_security_mode: USER_ISOLATION
-            instance_pool_id: "[TEST_INSTANCE_POOL_ID]"
+            instance_pool_id: [TEST_INSTANCE_POOL_ID]
           python_wheel_task:
             package_name: my_test_code
             entry_point: run

acceptance/bundle/integration_whl/wrapper_custom_params/script

@@ -1,8 +1,7 @@
-export SPARK_VERSION=12.2.x-scala2.12
+export DEFAULT_SPARK_VERSION=12.2.x-scala2.12
-export PYTHON_WHEEL_WRAPPER=true
+export EXTRA_CONFIG=python_wheel_wrapper.yml
-envsubst < $TESTDIR/../base/input.json.tmpl > input.json
+envsubst < $TESTDIR/../base/databricks.yml.tmpl > databricks.yml
-trace cat input.json
+cp -r $TESTDIR/../base/{$EXTRA_CONFIG,setup.py,my_test_code} .
-$CLI bundle init $TESTDIR/../base --config-file input.json
 trace cat databricks.yml
 trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
 trace $CLI bundle deploy