From 71b6394ccde7225ccc3a2fc57ce10c87cac4813a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 14 Feb 2025 17:11:22 +0100 Subject: [PATCH] add 'serverless: true' to pipeline --- .../default-python/classic/out.compare-vs-serverless.diff | 8 ++++++++ .../resources/my_default_python.pipeline.yml | 1 + .../resources/{{.project_name}}.pipeline.yml.tmpl | 7 +++++++ 3 files changed, 16 insertions(+) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 9f2378f30..95ed062e4 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -37,3 +37,11 @@ + autoscale: + min_workers: 1 + max_workers: 4 +--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml ++++ output/my_default_python/resources/my_default_python.pipeline.yml +@@ -6,5 +6,4 @@ + catalog: main + target: my_default_python_${bundle.target} +- serverless: true + libraries: + - notebook: diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml index f9e083f4f..4f880ba48 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,6 +5,7 @@ resources: name: my_default_python_pipeline catalog: main target: my_default_python_${bundle.target} + serverless: true libraries: - notebook: path: ../src/dlt_pipeline.ipynb diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 50f11fe2c..67eff5b3e 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -1,3 +1,7 @@ +{{$with_serverless := (eq .serverless "yes") -}} +{{if (eq .serverless "auto") -}} + {{$with_serverless = is_serverless_supported -}} +{{end -}} # The main pipeline for {{.project_name}} resources: pipelines: @@ -10,6 +14,9 @@ resources: catalog: {{default_catalog}} {{- end}} target: {{.project_name}}_${bundle.target} + {{- if $with_serverless }} + serverless: true + {{- end}} libraries: - notebook: path: ../src/dlt_pipeline.ipynb