diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 711576247..6f457a6ae 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -4,7 +4,7 @@ "project_name": { "type": "string", "default": "my_project", - "description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project", + "description": "\nPlease provide a unique name for this project.\nproject_name", "order": 1, "pattern": "^[A-Za-z0-9_]+$", "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." @@ -13,7 +13,7 @@ "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'", + "description": "\n\nInclude a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'", "order": 2 }, "include_dlt": { diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/apply_defaults.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/apply_defaults.ipynb.tmpl deleted file mode 100644 index a68e7d822..000000000 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/apply_defaults.ipynb.tmpl +++ /dev/null @@ -1,83 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# helper notebook: apply_defaults\n", - "\n", - "This helper notebook is used to create widgets that configure the default catalog\n", - "and schema.\n", - "\n", - "Usage:\n", - "\n", - "```\n", - "% run ../relative/path/to/apply_defaults\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Load default catalog and schema as widget and set their values as the default catalog / schema\n", - "dbutils.widgets.text('catalog', '{{.default_catalog}}')\n", - "dbutils.widgets.text('schema', 'default')\n", - "catalog = dbutils.widgets.get('catalog')\n", - "schema = dbutils.widgets.get('schema')\n", - "spark.sql(f'USE {catalog}.{schema}')" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Automatically reload imported modules when they change\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "dlt_pipeline", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl index 95f6b16a3..4d3c398c9 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl @@ -24,7 +24,9 @@ "outputs": [], "source": [ "# Load default catalog and schema as widget and set their values as the default catalog / schema\n", - "%run ./apply_defaults" + "catalog = dbutils.widgets.get('catalog')\n", + "schema = dbutils.widgets.get('schema')\n", + "spark.sql(f'USE {catalog}.{schema}')" ] }, { @@ -62,7 +64,64 @@ "pythonIndentUnit": 2 }, "notebookName": "notebook", - "widgets": {} + "widgets": { + "catalog": { + "currentValue": "{{.default_catalog}}", + "nuid": "3965fc9c-8080-45b1-bee3-f75cef7685b4", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "{{.default_catalog}}", + "label": null, + "name": "catalog", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "{{.default_catalog}}", + "label": null, + "name": "catalog", + "options": { + "widgetType": "text", + "autoCreated": null, + "validationRegex": null + } + } + }, +{{- $dev_schema := .shared_schema }} +{{- if (regexp "^yes").MatchString .personal_schemas}} + {{- $dev_schema = "{{short_name}}"}} +{{- end}} + "schema": { + "currentValue": "{{$dev_schema}}", + "nuid": "6ec0d70f-39bf-4859-a510-02c3e3d59bff", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "{{$dev_schema}}", + "label": null, + "name": "schema", + "options": { + "widgetDisplayType": "Text", + "validationRegex": null + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "widgetType": "text", + "defaultValue": "{{$dev_schema}}", + "label": null, + "name": "schema", + "options": { + "widgetType": "text", + "autoCreated": null, + "validationRegex": null + } + } + } + } }, "kernelspec": { "display_name": "Python 3", diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl index 80d447f6c..45af6fa86 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/{{.project_name}}/main.py.tmpl @@ -1,4 +1,5 @@ from pyspark.sql import SparkSession, DataFrame +import argparse def get_spark() -> SparkSession: """