steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix python wheel task integration tests (#1648) 

## Changes
A new Service Control Policy has removed the `ec2.RunInstances`
permission from our service principal for our AWS integration tests.
This PR switches over to using the instance pool which does not require
creating new clusters.

## Tests
The integration tests pass now.
This commit is contained in:
shreyas-goenka 2024-08-01 19:25:22 +05:30 committed by GitHub
parent 630a56e41e
commit a13d77f8eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/bundle/bundles/python_wheel_task/databricks_template_schema.json
View File

@ -20,6 +20,10 @@
"python_wheel_wrapper": { "python_wheel_wrapper": {
"type": "boolean", "type": "boolean",
"description": "Whether or not to enable python wheel wrapper" "description": "Whether or not to enable python wheel wrapper"
},
"instance_pool_id": {
"type": "string",
"description": "Instance pool id for job cluster"
} }
} }
} }

1
internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl
View File

@ -20,6 +20,7 @@ resources:
spark_version: "{{.spark_version}}" spark_version: "{{.spark_version}}"
node_type_id: "{{.node_type_id}}" node_type_id: "{{.node_type_id}}"
data_security_mode: USER_ISOLATION data_security_mode: USER_ISOLATION
instance_pool_id: "{{.instance_pool_id}}"
python_wheel_task: python_wheel_task:
package_name: my_test_code package_name: my_test_code
entry_point: run entry_point: run

2
internal/bundle/python_wheel_test.go
View File

@ -14,11 +14,13 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo
ctx, _ := acc.WorkspaceTest(t) ctx, _ := acc.WorkspaceTest(t)
nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{ bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{
"node_type_id": nodeTypeId, "node_type_id": nodeTypeId,
"unique_id": uuid.New().String(), "unique_id": uuid.New().String(),
"spark_version": sparkVersion, "spark_version": sparkVersion,
"python_wheel_wrapper": pythonWheelWrapper, "python_wheel_wrapper": pythonWheelWrapper,
"instance_pool_id": instancePoolId,
}) })
require.NoError(t, err) require.NoError(t, err)