update with regard to libraries

This commit is contained in:
Denis Bilenko 2025-02-13 13:22:54 +01:00
parent a9c67164cd
commit 44bdd357ff
4 changed files with 58 additions and 16 deletions

View File

@ -6,17 +6,29 @@
+ job_cluster_key: job_cluster
notebook_task:
notebook_path: ../src/notebook.ipynb
@@ -29,5 +30,5 @@
@@ -29,16 +30,19 @@
depends_on:
- task_key: refresh_pipeline
- environment_key: default
+ job_cluster_key: job_cluster
python_wheel_task:
package_name: my_default_python
@@ -38,2 +39,11 @@
# for more information on how to add other libraries.
- whl: ../dist/*.whl
+
entry_point: main
- # A list of task execution environment specifications that can be referenced by tasks of this job.
- environments:
- - environment_key: default
-
- # Full documentation of this spec can be found at:
- # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
- spec:
- client: "1"
- dependencies:
- - ../dist/*.whl
+ libraries:
+ # By default we just include the .whl file generated for the my_default_python package.
+ # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+ # for more information on how to add other libraries.
+ - whl: ../dist/*.whl
+ job_clusters:
+ - job_cluster_key: job_cluster
+ new_cluster:

View File

@ -6,17 +6,29 @@
+ job_cluster_key: job_cluster
notebook_task:
notebook_path: ../src/notebook.ipynb
@@ -29,5 +30,5 @@
@@ -29,16 +30,19 @@
depends_on:
- task_key: refresh_pipeline
- environment_key: default
+ job_cluster_key: job_cluster
python_wheel_task:
package_name: my_default_python
@@ -38,2 +39,11 @@
# for more information on how to add other libraries.
- whl: ../dist/*.whl
+
entry_point: main
- # A list of task execution environment specifications that can be referenced by tasks of this job.
- environments:
- - environment_key: default
-
- # Full documentation of this spec can be found at:
- # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
- spec:
- client: "1"
- dependencies:
- - ../dist/*.whl
+ libraries:
+ # By default we just include the .whl file generated for the my_default_python package.
+ # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+ # for more information on how to add other libraries.
+ - whl: ../dist/*.whl
+ job_clusters:
+ - job_cluster_key: job_cluster
+ new_cluster:

View File

@ -32,8 +32,13 @@ resources:
python_wheel_task:
package_name: my_default_python
entry_point: main
libraries:
# By default we just include the .whl file generated for the my_default_python package.
# See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
# for more information on how to add other libraries.
- whl: ../dist/*.whl
# A list of task execution environment specifications that can be referenced by tasks of this job.
environments:
- environment_key: default
# Full documentation of this spec can be found at:
# https://docs.databricks.com/api/workspace/jobs/create#environments-spec
spec:
client: "1"
dependencies:
- ../dist/*.whl

View File

@ -64,14 +64,27 @@ resources:
python_wheel_task:
package_name: {{.project_name}}
entry_point: main
{{- if not $with_serverless }}
libraries:
# By default we just include the .whl file generated for the {{.project_name}} package.
# See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
# for more information on how to add other libraries.
- whl: ../dist/*.whl
{{- end -}}
{{else}}
{{- end -}}
{{- if not $with_serverless}}
{{- if $with_serverless}}
# A list of task execution environment specifications that can be referenced by tasks of this job.
environments:
- environment_key: default
# Full documentation of this spec can be found at:
# https://docs.databricks.com/api/workspace/jobs/create#environments-spec
spec:
client: "1"
dependencies:
- ../dist/*.whl
{{ else }}
job_clusters:
- job_cluster_key: job_cluster
new_cluster: