From 9e56bed5935df915d5f8cb6c2b673a3ed7c3d462 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 11 Sep 2023 09:36:44 +0200 Subject: [PATCH] Minor default template tweaks (#758) Minor template tweaks, mostly making the imports section for DLT notebooks a bit more elegant. Tested with DAB deployment + in-workspace UI. --- .../default-python/template/__preamble.tmpl | 2 +- .../template/{{.project_name}}/README.md.tmpl | 2 +- .../{{.project_name}}/fixtures/.gitkeep.tmpl | 2 +- .../{{.project_name}}_pipeline.yml.tmpl | 6 +++--- .../src/dlt_pipeline.ipynb.tmpl | 20 +++++-------------- .../{{.project_name}}/src/notebook.ipynb.tmpl | 2 +- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/libs/template/templates/default-python/template/__preamble.tmpl b/libs/template/templates/default-python/template/__preamble.tmpl index 95c61333..a86d3bff 100644 --- a/libs/template/templates/default-python/template/__preamble.tmpl +++ b/libs/template/templates/default-python/template/__preamble.tmpl @@ -17,7 +17,7 @@ This file only template directives; it is skipped for the actual output. {{if $notPython}} {{skip "{{.project_name}}/src/{{.project_name}}"}} - {{skip "{{.project_name}}/tests/test_main.py"}} + {{skip "{{.project_name}}/tests/main_test.py"}} {{skip "{{.project_name}}/setup.py"}} {{skip "{{.project_name}}/pytest.ini"}} {{end}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl index 7c8876e7..1bcd7af4 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl @@ -20,7 +20,7 @@ The '{{.project_name}}' project was generated by using the default-python templa This deploys everything that's defined for this project. For example, the default template would deploy a job called - `[dev yourname] {{.project_name}}-job` to your workspace. + `[dev yourname] {{.project_name}}_job` to your workspace. You can find that job by opening your workpace and clicking on **Workflows**. 4. Similarly, to deploy a production copy, type: diff --git a/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl index 361c681f..ee957030 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/fixtures/.gitkeep.tmpl @@ -17,7 +17,7 @@ def get_absolute_path(*relative_parts): if 'dbutils' in globals(): base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore path = os.path.normpath(os.path.join(base_dir, *relative_parts)) - return path if path.startswith("/Workspace") else os.path.join("/Workspace", path) + return path if path.startswith("/Workspace") else "/Workspace" + path else: return os.path.join(*relative_parts) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index ffe400cb..4b8f74d1 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -2,11 +2,11 @@ resources: pipelines: {{.project_name}}_pipeline: - name: "{{.project_name}}_pipeline" - target: "{{.project_name}}_${bundle.environment}" + name: {{.project_name}}_pipeline + target: {{.project_name}}_${bundle.environment} libraries: - notebook: path: ../src/dlt_pipeline.ipynb configuration: - "bundle.sourcePath": "/Workspace/${workspace.file_path}/src" + bundle.sourcePath: /Workspace/${workspace.file_path}/src diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl index 74893238..4f50294f 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# DLT pipeline\n", "\n", - "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.my_project}}_pipeline.yml." + "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}_pipeline.yml." ] }, { @@ -27,28 +27,18 @@ "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", "showTitle": false, "title": "" - }, - "jupyter": { - {{- /* Collapse this cell by default. Just boring imports here! */}} - "source_hidden": true } }, "outputs": [], "source": [ {{- if (eq .include_python "yes") }} - "# Import DLT and make sure 'my_project' is on the Python path\n", + "# Import DLT and src/{{.project_name}}\n", "import dlt\n", - "from pyspark.sql.functions import expr\n", - "from pyspark.sql import SparkSession\n", - "spark = SparkSession.builder.getOrCreate()\n", "import sys\n", - "try:\n", - " sys.path.append(spark.conf.get(\"bundle.sourcePath\"))\n", - "except:\n", - " pass\n", - "from my_project import main" + "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", + "from pyspark.sql.functions import expr\n", + "from {{.project_name}} import main" {{else}} - "# Import DLT\n", "import dlt\n", "from pyspark.sql.functions import expr\n", "from pyspark.sql import SparkSession\n", diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl index 8423ecf8..0ab61db2 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# Default notebook\n", "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/{{.my_project}}_job.yml." + "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}_job.yml." ] }, {