Minor default template tweaks (#758)

Minor template tweaks, mostly making the imports section for DLT
notebooks a bit more elegant.

Tested with DAB deployment + in-workspace UI.
This commit is contained in:
Lennart Kats (databricks) 2023-09-11 09:36:44 +02:00 committed by GitHub
parent d9a276b17d
commit 9e56bed593
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 12 additions and 22 deletions

View File

@ -17,7 +17,7 @@ This file only template directives; it is skipped for the actual output.
{{if $notPython}} {{if $notPython}}
{{skip "{{.project_name}}/src/{{.project_name}}"}} {{skip "{{.project_name}}/src/{{.project_name}}"}}
{{skip "{{.project_name}}/tests/test_main.py"}} {{skip "{{.project_name}}/tests/main_test.py"}}
{{skip "{{.project_name}}/setup.py"}} {{skip "{{.project_name}}/setup.py"}}
{{skip "{{.project_name}}/pytest.ini"}} {{skip "{{.project_name}}/pytest.ini"}}
{{end}} {{end}}

View File

@ -20,7 +20,7 @@ The '{{.project_name}}' project was generated by using the default-python templa
This deploys everything that's defined for this project. This deploys everything that's defined for this project.
For example, the default template would deploy a job called For example, the default template would deploy a job called
`[dev yourname] {{.project_name}}-job` to your workspace. `[dev yourname] {{.project_name}}_job` to your workspace.
You can find that job by opening your workpace and clicking on **Workflows**. You can find that job by opening your workpace and clicking on **Workflows**.
4. Similarly, to deploy a production copy, type: 4. Similarly, to deploy a production copy, type:

View File

@ -17,7 +17,7 @@ def get_absolute_path(*relative_parts):
if 'dbutils' in globals(): if 'dbutils' in globals():
base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
path = os.path.normpath(os.path.join(base_dir, *relative_parts)) path = os.path.normpath(os.path.join(base_dir, *relative_parts))
return path if path.startswith("/Workspace") else os.path.join("/Workspace", path) return path if path.startswith("/Workspace") else "/Workspace" + path
else: else:
return os.path.join(*relative_parts) return os.path.join(*relative_parts)

View File

@ -2,11 +2,11 @@
resources: resources:
pipelines: pipelines:
{{.project_name}}_pipeline: {{.project_name}}_pipeline:
name: "{{.project_name}}_pipeline" name: {{.project_name}}_pipeline
target: "{{.project_name}}_${bundle.environment}" target: {{.project_name}}_${bundle.environment}
libraries: libraries:
- notebook: - notebook:
path: ../src/dlt_pipeline.ipynb path: ../src/dlt_pipeline.ipynb
configuration: configuration:
"bundle.sourcePath": "/Workspace/${workspace.file_path}/src" bundle.sourcePath: /Workspace/${workspace.file_path}/src

View File

@ -14,7 +14,7 @@
"source": [ "source": [
"# DLT pipeline\n", "# DLT pipeline\n",
"\n", "\n",
"This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.my_project}}_pipeline.yml." "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}_pipeline.yml."
] ]
}, },
{ {
@ -27,28 +27,18 @@
"nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f",
"showTitle": false, "showTitle": false,
"title": "" "title": ""
},
"jupyter": {
{{- /* Collapse this cell by default. Just boring imports here! */}}
"source_hidden": true
} }
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
{{- if (eq .include_python "yes") }} {{- if (eq .include_python "yes") }}
"# Import DLT and make sure 'my_project' is on the Python path\n", "# Import DLT and src/{{.project_name}}\n",
"import dlt\n", "import dlt\n",
"from pyspark.sql.functions import expr\n",
"from pyspark.sql import SparkSession\n",
"spark = SparkSession.builder.getOrCreate()\n",
"import sys\n", "import sys\n",
"try:\n", "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
" sys.path.append(spark.conf.get(\"bundle.sourcePath\"))\n", "from pyspark.sql.functions import expr\n",
"except:\n", "from {{.project_name}} import main"
" pass\n",
"from my_project import main"
{{else}} {{else}}
"# Import DLT\n",
"import dlt\n", "import dlt\n",
"from pyspark.sql.functions import expr\n", "from pyspark.sql.functions import expr\n",
"from pyspark.sql import SparkSession\n", "from pyspark.sql import SparkSession\n",

View File

@ -14,7 +14,7 @@
"source": [ "source": [
"# Default notebook\n", "# Default notebook\n",
"\n", "\n",
"This default notebook is executed using Databricks Workflows as defined in resources/{{.my_project}}_job.yml." "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}_job.yml."
] ]
}, },
{ {