--- [TESTROOT]/bundle/templates/default-python-classic/../default-python/output/my_default_python/resources/my_default_python.job.yml +++ output/my_default_python/resources/my_default_python.job.yml @@ -17,4 +17,5 @@ tasks: - task_key: notebook_task + job_cluster_key: job_cluster notebook_task: notebook_path: ../src/notebook.ipynb @@ -29,5 +30,5 @@ depends_on: - task_key: refresh_pipeline - environment_key: default + job_cluster_key: job_cluster python_wheel_task: package_name: my_default_python @@ -39,3 +40,10 @@ - whl: ../dist/*.whl - + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + autoscale: + min_workers: 1 + max_workers: 4