2024-04-03 10:40:29 +00:00
|
|
|
resources:
|
|
|
|
jobs:
|
|
|
|
job:
|
|
|
|
tasks:
|
|
|
|
- task_key: local
|
2025-01-17 10:22:49 +00:00
|
|
|
job_cluster_key: default
|
2024-04-03 10:40:29 +00:00
|
|
|
spark_python_task:
|
|
|
|
python_file: ../src/file1.py
|
|
|
|
|
|
|
|
- task_key: variable_reference
|
2025-01-17 10:22:49 +00:00
|
|
|
job_cluster_key: default
|
2024-04-03 10:40:29 +00:00
|
|
|
spark_python_task:
|
|
|
|
# Note: this is a pure variable reference yet needs to persist the location
|
|
|
|
# of the reference, not the location of the variable value.
|
|
|
|
# Also see https://github.com/databricks/cli/issues/1330.
|
|
|
|
python_file: ${var.file_path}
|
2025-01-17 10:22:49 +00:00
|
|
|
|
|
|
|
# Include a job cluster for completeness
|
|
|
|
job_clusters:
|
|
|
|
- job_cluster_key: default
|
|
|
|
new_cluster:
|
|
|
|
spark_version: 15.4.x-scala2.12
|