Added E2E test to run Python wheels on interactive cluster created in bundle (#1864)

## Changes
Added E2E test to run python wheels on interactive cluster created in
bundle.

We had a gap in testing wheel on all purpose clusters, so this PR
addresses the gap

cmd/root/root.go

@@ -75,6 +75,7 @@ func New(ctx context.Context) *cobra.Command {
 
 		// Configure our user agent with the command that's about to be executed.
 		ctx = withCommandInUserAgent(ctx, cmd)
+		ctx = withCommandExecIdInUserAgent(ctx)
 		ctx = withUpstreamInUserAgent(ctx)
 		cmd.SetContext(ctx)
 		return nil

cmd/root/user_agent_command_exec_id.go

@@ -0,0 +1,14 @@
+package root
+
+import (
+	"context"
+
+	"github.com/databricks/databricks-sdk-go/useragent"
+	"github.com/google/uuid"
+)
+
+func withCommandExecIdInUserAgent(ctx context.Context) context.Context {
+	// A UUID that will allow us to correlate multiple API requests made by
+	// the same CLI invocation.
+	return useragent.InContext(ctx, "cmd-exec-id", uuid.New().String())
+}

cmd/root/user_agent_command_exec_id_test.go

@@ -0,0 +1,26 @@
+package root
+
+import (
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/useragent"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWithCommandExecIdInUserAgent(t *testing.T) {
+	ctx := withCommandExecIdInUserAgent(context.Background())
+
+	// Check that the command exec ID is in the user agent string.
+	ua := useragent.FromContext(ctx)
+	re := regexp.MustCompile(`cmd-exec-id/([a-f0-9-]+)`)
+	matches := re.FindAllStringSubmatch(ua, -1)
+
+	// Assert that we have exactly one match and that it's a valid UUID.
+	require.Len(t, matches, 1)
+	_, err := uuid.Parse(matches[0][1])
+	assert.NoError(t, err)
+}

cmd/root/user_agent_command_test.go

@@ -1,13 +1,15 @@
 package root
 
 import (
+	"context"
 	"testing"
 
+	"github.com/databricks/databricks-sdk-go/useragent"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCommandString(t *testing.T) {
+func TestWithCommandInUserAgent(t *testing.T) {
 	root := &cobra.Command{
 		Use: "root",
 	}
@@ -26,4 +28,9 @@ func TestCommandString(t *testing.T) {
 	assert.Equal(t, "root", commandString(root))
 	assert.Equal(t, "hello", commandString(hello))
 	assert.Equal(t, "hello_world", commandString(world))
+
+	ctx := withCommandInUserAgent(context.Background(), world)
+
+	ua := useragent.FromContext(ctx)
+	assert.Contains(t, ua, "cmd/hello_world")
 }

internal/bundle/bundles/python_wheel_task_with_cluster/databricks_template_schema.json

@@ -0,0 +1,25 @@
+{
+    "properties": {
+        "project_name": {
+            "type": "string",
+            "default": "my_test_code",
+            "description": "Unique name for this project"
+        },
+        "spark_version": {
+            "type": "string",
+            "description": "Spark version used for job cluster"
+        },
+        "node_type_id": {
+            "type": "string",
+            "description": "Node type id for job cluster"
+        },
+        "unique_id": {
+            "type": "string",
+            "description": "Unique ID for job name"
+        },
+        "instance_pool_id": {
+            "type": "string",
+            "description": "Instance pool id for job cluster"
+        }
+    }
+}

internal/bundle/bundles/python_wheel_task_with_cluster/template/databricks.yml.tmpl

@@ -0,0 +1,29 @@
+bundle:
+  name: wheel-task
+
+workspace:
+  root_path: "~/.bundle/{{.unique_id}}"
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: "test-cluster-{{.unique_id}}"
+      spark_version: "{{.spark_version}}"
+      node_type_id: "{{.node_type_id}}"
+      num_workers: 1
+      data_security_mode: USER_ISOLATION
+
+  jobs:
+    some_other_job:
+      name: "[${bundle.target}] Test Wheel Job {{.unique_id}}"
+      tasks:
+        - task_key: TestTask
+          existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}"
+          python_wheel_task:
+            package_name: my_test_code
+            entry_point: run
+            parameters:
+              - "one"
+              - "two"
+          libraries:
+            - whl: ./dist/*.whl

internal/bundle/bundles/python_wheel_task_with_cluster/template/setup.py.tmpl

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import {{.project_name}}
+
+setup(
+    name="{{.project_name}}",
+    version={{.project_name}}.__version__,
+    author={{.project_name}}.__author__,
+    url="https://databricks.com",
+    author_email="john.doe@databricks.com",
+    description="my example wheel",
+    packages=find_packages(include=["{{.project_name}}"]),
+    entry_points={"group1": "run={{.project_name}}.__main__:main"},
+    install_requires=["setuptools"],
+)

internal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.1"
+__author__ = "Databricks"

internal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__main__.py

@@ -0,0 +1,16 @@
+"""
+The entry point of the Python Wheel
+"""
+
+import sys
+
+
+def main():
+    # This method will print the provided arguments
+    print("Hello from my func")
+    print("Got arguments:")
+    print(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

internal/bundle/python_wheel_test.go

@@ -5,17 +5,18 @@ import (
 
 	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
-func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bool) {
+func runPythonWheelTest(t *testing.T, templateName string, sparkVersion string, pythonWheelWrapper bool) {
 	ctx, _ := acc.WorkspaceTest(t)
 
 	nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
 	instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
-	bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{
+	bundleRoot, err := initTestTemplate(t, ctx, templateName, map[string]any{
 		"node_type_id":         nodeTypeId,
 		"unique_id":            uuid.New().String(),
 		"spark_version":        sparkVersion,
@@ -45,9 +46,19 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo
 }
 
 func TestAccPythonWheelTaskDeployAndRunWithoutWrapper(t *testing.T) {
-	runPythonWheelTest(t, "13.3.x-snapshot-scala2.12", false)
+	runPythonWheelTest(t, "python_wheel_task", "13.3.x-snapshot-scala2.12", false)
 }
 
 func TestAccPythonWheelTaskDeployAndRunWithWrapper(t *testing.T) {
-	runPythonWheelTest(t, "12.2.x-scala2.12", true)
+	runPythonWheelTest(t, "python_wheel_task", "12.2.x-scala2.12", true)
+}
+
+func TestAccPythonWheelTaskDeployAndRunOnInteractiveCluster(t *testing.T) {
+	_, wt := acc.WorkspaceTest(t)
+
+	if testutil.IsAWSCloud(wt.T) {
+		t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters")
+	}
+
+	runPythonWheelTest(t, "python_wheel_task_with_cluster", defaultSparkVersion, false)
 }