Fix python wheel task integration tests (#1648)

## Changes
A new Service Control Policy has removed the `ec2.RunInstances`
permission from our service principal for our AWS integration tests.
This PR switches over to using the instance pool which does not require
creating new clusters.

## Tests
The integration tests pass now.
This commit is contained in:
shreyas-goenka 2024-08-01 19:25:22 +05:30 committed by GitHub
parent 630a56e41e
commit a13d77f8eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 7 additions and 0 deletions

View File

@ -20,6 +20,10 @@
"python_wheel_wrapper": {
"type": "boolean",
"description": "Whether or not to enable python wheel wrapper"
},
"instance_pool_id": {
"type": "string",
"description": "Instance pool id for job cluster"
}
}
}

View File

@ -20,6 +20,7 @@ resources:
spark_version: "{{.spark_version}}"
node_type_id: "{{.node_type_id}}"
data_security_mode: USER_ISOLATION
instance_pool_id: "{{.instance_pool_id}}"
python_wheel_task:
package_name: my_test_code
entry_point: run

View File

@ -14,11 +14,13 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo
ctx, _ := acc.WorkspaceTest(t)
nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{
"node_type_id": nodeTypeId,
"unique_id": uuid.New().String(),
"spark_version": sparkVersion,
"python_wheel_wrapper": pythonWheelWrapper,
"instance_pool_id": instancePoolId,
})
require.NoError(t, err)