mirror of https://github.com/databricks/cli.git
WIP
This commit is contained in:
Lennart Kats
parent
3ca2e674be
commit
487884b6a5
|
|
@ -4,7 +4,7 @@
|
|||
"project_name": {
|
||||
"type": "string",
|
||||
"default": "my_project",
|
||||
"description": "Please provide the following details to tailor the template to your preferences.\n\nUnique name for this project",
|
||||
"description": "\nPlease provide a unique name for this project.\nproject_name",
|
||||
"order": 1,
|
||||
"pattern": "^[A-Za-z0-9_]+$",
|
||||
"pattern_match_failure_message": "Name must consist of letters, numbers, and underscores."
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
"type": "string",
|
||||
"default": "yes",
|
||||
"enum": ["yes", "no"],
|
||||
"description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'",
|
||||
"description": "\n\nInclude a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'",
|
||||
"order": 2
|
||||
},
|
||||
"include_dlt": {
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"# helper notebook: apply_defaults\n",
|
||||
"\n",
|
||||
"This helper notebook is used to create widgets that configure the default catalog\n",
|
||||
"and schema.\n",
|
||||
"\n",
|
||||
"Usage:\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"% run ../relative/path/to/apply_defaults\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 0,
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+cell": {
|
||||
"cellMetadata": {},
|
||||
"inputWidgets": {},
|
||||
"nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f",
|
||||
"showTitle": false,
|
||||
"title": ""
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load default catalog and schema as widget and set their values as the default catalog / schema\n",
|
||||
"dbutils.widgets.text('catalog', '{{.default_catalog}}')\n",
|
||||
"dbutils.widgets.text('schema', 'default')\n",
|
||||
"catalog = dbutils.widgets.get('catalog')\n",
|
||||
"schema = dbutils.widgets.get('schema')\n",
|
||||
"spark.sql(f'USE {catalog}.{schema}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically reload imported modules when they change\n",
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"application/vnd.databricks.v1+notebook": {
|
||||
"dashboards": [],
|
||||
"language": "python",
|
||||
"notebookMetadata": {
|
||||
"pythonIndentUnit": 2
|
||||
},
|
||||
"notebookName": "dlt_pipeline",
|
||||
"widgets": {}
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
|
|
@ -24,7 +24,9 @@
|
|||
"outputs": [],
|
||||
"source": [
|
||||
"# Load default catalog and schema as widget and set their values as the default catalog / schema\n",
|
||||
"%run ./apply_defaults"
|
||||
"catalog = dbutils.widgets.get('catalog')\n",
|
||||
"schema = dbutils.widgets.get('schema')\n",
|
||||
"spark.sql(f'USE {catalog}.{schema}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -62,7 +64,64 @@
|
|||
"pythonIndentUnit": 2
|
||||
},
|
||||
"notebookName": "notebook",
|
||||
"widgets": {}
|
||||
"widgets": {
|
||||
"catalog": {
|
||||
"currentValue": "{{.default_catalog}}",
|
||||
"nuid": "3965fc9c-8080-45b1-bee3-f75cef7685b4",
|
||||
"typedWidgetInfo": {
|
||||
"autoCreated": false,
|
||||
"defaultValue": "{{.default_catalog}}",
|
||||
"label": null,
|
||||
"name": "catalog",
|
||||
"options": {
|
||||
"widgetDisplayType": "Text",
|
||||
"validationRegex": null
|
||||
},
|
||||
"parameterDataType": "String"
|
||||
},
|
||||
"widgetInfo": {
|
||||
"widgetType": "text",
|
||||
"defaultValue": "{{.default_catalog}}",
|
||||
"label": null,
|
||||
"name": "catalog",
|
||||
"options": {
|
||||
"widgetType": "text",
|
||||
"autoCreated": null,
|
||||
"validationRegex": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{{- $dev_schema := .shared_schema }}
|
||||
{{- if (regexp "^yes").MatchString .personal_schemas}}
|
||||
{{- $dev_schema = "{{short_name}}"}}
|
||||
{{- end}}
|
||||
"schema": {
|
||||
"currentValue": "{{$dev_schema}}",
|
||||
"nuid": "6ec0d70f-39bf-4859-a510-02c3e3d59bff",
|
||||
"typedWidgetInfo": {
|
||||
"autoCreated": false,
|
||||
"defaultValue": "{{$dev_schema}}",
|
||||
"label": null,
|
||||
"name": "schema",
|
||||
"options": {
|
||||
"widgetDisplayType": "Text",
|
||||
"validationRegex": null
|
||||
},
|
||||
"parameterDataType": "String"
|
||||
},
|
||||
"widgetInfo": {
|
||||
"widgetType": "text",
|
||||
"defaultValue": "{{$dev_schema}}",
|
||||
"label": null,
|
||||
"name": "schema",
|
||||
"options": {
|
||||
"widgetType": "text",
|
||||
"autoCreated": null,
|
||||
"validationRegex": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from pyspark.sql import SparkSession, DataFrame
|
||||
import argparse
|
||||
|
||||
def get_spark() -> SparkSession:
|
||||
"""
|
||||
|
|
|
|||
Loading…
Reference in New Issue