Add an experimental dbt-sql template (#1059)

## Changes

This adds a new dbt-sql template. This work requires the new WorkspaceFS
support for dbt tasks.

In this latest revision, I've hidden the new template from the list so
we can merge it, iterate over it, and propertly release the template at
the right time.

Blockers:
- [x] WorkspaceFS support for dbt projects is in prod
- [x] Move dbt files into a subdirectory
- [ ] Wait until the next (>1.7.4) release of the dbt plugin which will
have major improvements!
- _Rather than wait, this template is hidden from the list of
templates._
- [x] SQL extension is preconfigured based on extension settings (if
possible)
- MV / streaming tables:
  - [x] Add to template
- [x] Fix https://github.com/databricks/dbt-databricks/issues/535 (to be
released with in 1.7.4)
- [x] Merge https://github.com/databricks/dbt-databricks/pull/338 (to be
released with in 1.7.4)
- [ ] Fix "too many 503 errors" issue
(https://github.com/databricks/dbt-databricks/issues/570, internal
tracker: ES-1009215, ES-1014138)
  - [x] Support ANSI mode in the template
- [ ] Streaming tables support is either ungated or the template
provides instructions about signup
- _Mitigation for now: this template is hidden from the list of
templates._
- [x] Support non-workspace-admin deployment
- [x] Make sure `data_security_mode: SINGLE_USER` works on non-UC
workspaces (it's required to be explicitly specified on UC workspaces
with single-node clusters)
- [x] Support non-UC workspaces

## Tests

- [x] Unit tests
- [x] Manual testing
- [x] More manual testing
- [ ] Reviewer manual testing
  - _I'd like to do a small bug bash post-merging._
- [x] Unit tests
This commit is contained in:
Lennart Kats (databricks) 2024-02-19 10:15:17 +01:00 committed by GitHub
parent f70ec359dc
commit 1c680121c8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 569 additions and 7 deletions

View File

@ -25,6 +25,7 @@ type nativeTemplate struct {
gitUrl string
description string
aliases []string
hidden bool
}
const customTemplate = "custom..."
@ -34,6 +35,11 @@ var nativeTemplates = []nativeTemplate{
name: "default-python",
description: "The default Python template for Notebooks / Delta Live Tables / Workflows",
},
{
name: "dbt-sql",
description: "The dbt SQL template (https://www.databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)",
hidden: true,
},
{
name: "mlops-stacks",
gitUrl: "https://github.com/databricks/mlops-stacks",
@ -50,7 +56,7 @@ var nativeTemplates = []nativeTemplate{
func nativeTemplateHelpDescriptions() string {
var lines []string
for _, template := range nativeTemplates {
if template.name != customTemplate {
if template.name != customTemplate && !template.hidden {
lines = append(lines, fmt.Sprintf("- %s: %s", template.name, template.description))
}
}
@ -61,6 +67,9 @@ func nativeTemplateHelpDescriptions() string {
func nativeTemplateOptions() []cmdio.Tuple {
names := make([]cmdio.Tuple, 0, len(nativeTemplates))
for _, template := range nativeTemplates {
if template.hidden {
continue
}
tuple := cmdio.Tuple{
Name: template.name,
Id: template.description,

View File

@ -11,6 +11,7 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/iam"
)
@ -29,6 +30,7 @@ type pair struct {
var cachedUser *iam.User
var cachedIsServicePrincipal *bool
var cachedCatalog *string
func loadHelpers(ctx context.Context) template.FuncMap {
w := root.WorkspaceClient(ctx)
@ -108,6 +110,25 @@ func loadHelpers(ctx context.Context) template.FuncMap {
}
return auth.GetShortUserName(cachedUser.UserName), nil
},
// Get the default workspace catalog. If there is no default, or if
// Unity Catalog is not enabled, return an empty string.
"default_catalog": func() (string, error) {
if cachedCatalog == nil {
metastore, err := w.Metastores.Current(ctx)
if err != nil {
var aerr *apierr.APIError
if errors.As(err, &aerr) && aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" {
// Workspace doesn't have a metastore assigned, ignore error
empty_default := ""
cachedCatalog = &empty_default
return "", nil
}
return "", err
}
cachedCatalog = &metastore.DefaultCatalogName
}
return *cachedCatalog, nil
},
"is_service_principal": func() (bool, error) {
if cachedIsServicePrincipal != nil {
return *cachedIsServicePrincipal, nil

View File

@ -37,10 +37,10 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
assert.Equal(t, perm, info.Mode().Perm())
}
func assertBuiltinTemplateValid(t *testing.T, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
ctx := context.Background()
templatePath, err := prepareBuiltinTemplates("default-python", tempDir)
templatePath, err := prepareBuiltinTemplates(template, tempDir)
require.NoError(t, err)
libraryPath := filepath.Join(templatePath, "library")
@ -50,6 +50,9 @@ func assertBuiltinTemplateValid(t *testing.T, settings map[string]any, target st
// Prepare helpers
cachedUser = &iam.User{UserName: "user@domain.com"}
if isServicePrincipal {
cachedUser.UserName = "1d410060-a513-496f-a197-23cc82e5f46d"
}
cachedIsServicePrincipal = &isServicePrincipal
ctx = root.SetWorkspaceClient(ctx, w)
helpers := loadHelpers(ctx)
@ -102,11 +105,13 @@ func TestPrepareBuiltInTemplatesWithRelativePaths(t *testing.T) {
assert.Equal(t, "./default-python", dir)
}
func TestBuiltinTemplateValid(t *testing.T) {
func TestBuiltinPythonTemplateValid(t *testing.T) {
// Test option combinations
options := []string{"yes", "no"}
isServicePrincipal := false
build := false
catalog := "hive_metastore"
cachedCatalog = &catalog
for _, includeNotebook := range options {
for _, includeDlt := range options {
for _, includePython := range options {
@ -118,7 +123,7 @@ func TestBuiltinTemplateValid(t *testing.T) {
"include_python": includePython,
}
tempDir := t.TempDir()
assertBuiltinTemplateValid(t, config, "dev", isServicePrincipal, build, tempDir)
assertBuiltinTemplateValid(t, "default-python", config, "dev", isServicePrincipal, build, tempDir)
}
}
}
@ -140,10 +145,28 @@ func TestBuiltinTemplateValid(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tempDir)
assertBuiltinTemplateValid(t, config, "prod", isServicePrincipal, build, tempDir)
assertBuiltinTemplateValid(t, "default-python", config, "prod", isServicePrincipal, build, tempDir)
defer os.RemoveAll(tempDir)
}
func TestBuiltinDbtTemplateValid(t *testing.T) {
for _, personal_schemas := range []string{"yes", "no"} {
for _, target := range []string{"dev", "prod"} {
for _, isServicePrincipal := range []bool{true, false} {
config := map[string]any{
"project_name": "my_project",
"http_path": "/sql/1.0/warehouses/123",
"default_catalog": "hive_metastore",
"personal_schemas": personal_schemas,
"shared_schema": "lennart",
}
build := false
assertBuiltinTemplateValid(t, "dbt-sql", config, target, isServicePrincipal, build, t.TempDir())
}
}
}
}
func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
tmpDir := t.TempDir()

View File

@ -0,0 +1,9 @@
# dbt template
This folder provides a template for using dbt-core with Databricks Asset Bundles.
It leverages dbt-core for local development and relies on Databricks Asset Bundles
for deployment (either manually or with CI/CD). In production,
dbt is executed using Databricks Workflows.
* Learn more about the dbt and its standard project structure here: https://docs.getdbt.com/docs/build/projects.
* Learn more about Databricks Asset Bundles here: https://docs.databricks.com/en/dev-tools/bundles/index.html

View File

@ -0,0 +1,53 @@
{
"welcome_message": "\nWelcome to the (EXPERIMENTAL) dbt template for Databricks Asset Bundles!",
"properties": {
"project_name": {
"type": "string",
"pattern": "^[A-Za-z_][A-Za-z0-9_]+$",
"pattern_match_failure_message": "Name must consist of letters, numbers, and underscores.",
"default": "dbt_project",
"description": "\nPlease provide a unique name for this project.\nproject_name",
"order": 1
},
"http_path": {
"type": "string",
"pattern": "^/sql/.\\../warehouses/[a-z0-9]+$",
"pattern_match_failure_message": "Path must be of the form /sql/1.0/warehouses/<warehouse id>",
"description": " \nPlease provide the HTTP Path of the SQL warehouse you would like to use with dbt during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]",
"order": 2
},
"default_catalog": {
"type": "string",
"default": "{{default_catalog}}",
"pattern": "^\\w*$",
"pattern_match_failure_message": "Invalid catalog name.",
"description": "\nPlease provide an initial catalog{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}.\ndefault_catalog",
"order": 3
},
"personal_schemas": {
"type": "string",
"description": "\nWould you like to use a personal schema for each user working on this project? (e.g., 'catalog.{{short_name}}')\npersonal_schemas",
"enum": [
"yes, use a schema based on the current user name during development",
"no, use a shared schema during development"
],
"order": 4
},
"shared_schema": {
"skip_prompt_if": {
"properties": {
"personal_schemas": {
"const": "yes, use a schema based on the current user name during development"
}
}
},
"type": "string",
"default": "default",
"pattern": "^\\w+$",
"pattern_match_failure_message": "Invalid schema name.",
"description": "\nPlease provide an initial schema during development.\ndefault_schema",
"order": 5
}
},
"success_message": "\n📊 Your new project has been created in the '{{.project_name}}' directory!\nIf you already have dbt installed, just type 'cd {{.project_name}}; dbt init' to get started.\nRefer to the README.md file for full \"getting started\" guide and production setup instructions.\n"
}

View File

@ -0,0 +1,7 @@
{{define "latest_lts_dbr_version" -}}
13.3.x-scala2.12
{{- end}}
{{define "latest_lts_db_connect_version_spec" -}}
>=13.3,<13.4
{{- end}}

View File

@ -0,0 +1,9 @@
# Preamble
This file only template directives; it is skipped for the actual output.
{{skip "__preamble"}}
{{if eq .project_name "dbt"}}
{{fail "Project name 'dbt' is not supported"}}
{{end}}

View File

@ -0,0 +1,3 @@
# Typings for Pylance in Visual Studio Code
# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
from databricks.sdk.runtime import *

View File

@ -0,0 +1,6 @@
{
"recommendations": [
"redhat.vscode-yaml",
"innoverio.vscode-dbt-power-user",
]
}

View File

@ -0,0 +1,33 @@
{
"python.analysis.stubPath": ".vscode",
"databricks.python.envFile": "${workspaceFolder}/.env",
"jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
"jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
"python.testing.pytestArgs": [
"."
],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true,
"python.analysis.extraPaths": ["src"],
"files.exclude": {
"**/*.egg-info": true,
"**/__pycache__": true,
".pytest_cache": true,
},
"python.envFile": "${workspaceFolder}/.databricks/.databricks.env",
"python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
"sqltools.connections": [
{
"connectionMethod": "VS Code Extension (beta)",
"catalog": "hive_metastore",
"previewLimit": 50,
"driver": "Databricks",
"name": "databricks",
"path": "{{.http_path}}"
}
],
"sqltools.autoConnectTo": "",
"[jinja-sql]": {
"editor.defaultFormatter": "innoverio.vscode-dbt-power-user"
}
}

View File

@ -0,0 +1,138 @@
# {{.project_name}}
The '{{.project_name}}' project was generated by using the dbt template for
Databricks Asset Bundles. It follows the standard dbt project structure
and has an additional `resources` directory to define Databricks resources such as jobs
that run dbt models.
* Learn more about dbt and its standard project structure here: https://docs.getdbt.com/docs/build/projects.
* Learn more about Databricks Asset Bundles here: https://docs.databricks.com/en/dev-tools/bundles/index.html
The remainder of this file includes instructions for local development (using dbt)
and deployment to production (using Databricks Asset Bundles).
## Development setup
1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
2. Authenticate to your Databricks workspace, if you have not done so already:
```
$ databricks configure
```
3. Install dbt
To install dbt, you need a recent version of Python. For the instructions below,
we assume `python3` refers to the Python version you want to use. On some systems,
you may need to refer to a different Python version, e.g. `python` or `/usr/bin/python`.
Run these instructions from the `{{.project_name}}` directory. We recommend making
use of a Python virtual environment and installing dbt as follows:
```
$ python3 -m venv .venv
$ . .venv/bin/activate
$ pip install -r requirements-dev.txt
```
4. Initialize your dbt profile
Use `dbt init` to initialize your profile.
```
$ dbt init
```
Note that dbt authentication uses personal access tokens by default
(see https://docs.databricks.com/dev-tools/auth/pat.html).
You can use OAuth as an alternative, but this currently requires manual configuration.
See https://github.com/databricks/dbt-databricks/blob/main/docs/oauth.md
for general instructions, or https://community.databricks.com/t5/technical-blog/using-dbt-core-with-oauth-on-azure-databricks/ba-p/46605
for advice on setting up OAuth for Azure Databricks.
To setup up additional profiles, such as a 'prod' profile,
see https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles.
5. Activate dbt so it can be used from the terminal
```
$ . .venv/bin/activate
```
## Local development with dbt
Use `dbt` to [run this project locally using a SQL warehouse](https://docs.databricks.com/partners/prep/dbt.html):
```
$ dbt seed
$ dbt run
```
(Did you get an error that the dbt command could not be found? You may need
to try the last step from the development setup above to re-activate
your Python virtual environment!)
To just evaluate a single model defined in a file called orders.sql, use:
```
$ dbt run --model orders
```
Use `dbt test` to run tests generated from yml files such as `models/schema.yml`
and any SQL tests from `tests/`
```
$ dbt test
```
## Production setup
Your production dbt profiles are defined in dbt_profiles/profiles.yml.
These profiles define the default catalog, schema, and any other
target-specific settings. Read more about dbt profiles on Databricks at
https://docs.databricks.com/en/workflows/jobs/how-to/use-dbt-in-workflows.html#advanced-run-dbt-with-a-custom-profile.
The target workspaces for staging and prod are defined in databricks.yml.
You can manaully deploy based on these configurations (see below).
Or you can use CI/CD to automate deployment. See
https://docs.databricks.com/dev-tools/bundles/ci-cd.html for documentation
on CI/CD setup.
## Manually deploying to Databricks with Databricks Asset Bundles
Databricks Asset Bundles can be used to deploy to Databricks and to execute
dbt commands as a job using Databricks Workflows. See
https://docs.databricks.com/dev-tools/bundles/index.html to learn more.
Use the Databricks CLI to deploy a development copy of this project to a workspace:
```
$ databricks bundle deploy --target dev
```
(Note that "dev" is the default target, so the `--target` parameter
is optional here.)
This deploys everything that's defined for this project.
For example, the default template would deploy a job called
`[dev yourname] {{.project_name}}_job` to your workspace.
You can find that job by opening your workpace and clicking on **Workflows**.
You can also deploy to your production target directly from the command-line.
The warehouse, catalog, and schema for that target are configured in databricks.yml.
When deploying to this target, note that the default job at resources/{{.project_name}}_job.yml
has a schedule set that runs every day. The schedule is paused when deploying in development mode
(see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
To deploy a production copy, type:
```
$ databricks bundle deploy --target prod
```
## IDE support
Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
https://docs.databricks.com/dev-tools/vscode-ext.html. Third-party extensions
related to dbt may further enhance your dbt development experience!

View File

@ -0,0 +1,32 @@
# This file defines the structure of this project and how it is deployed
# to production using Databricks Asset Bundles.
# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
bundle:
name: {{.project_name}}
include:
- resources/*.yml
# Deployment targets.
# The default schema, catalog, etc. for dbt are defined in dbt_profiles/profiles.yml
targets:
dev:
default: true
# We use 'mode: development' to indicate this is a personal development copy.
# Any job schedules and triggers are paused by default.
mode: development
workspace:
host: {{workspace_host}}
prod:
mode: production
workspace:
host: {{workspace_host}}
# We always use /Users/{{user_name}} for all resources to make sure we only have a single copy.
root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
{{- if not is_service_principal}}
run_as:
# This runs as {{user_name}} in production. We could also use a service principal here
# using service_principal_name (see the Databricks documentation).
user_name: {{user_name}}
{{- end}}

View File

@ -0,0 +1,36 @@
{{- $catalog := .default_catalog}}
{{- if eq .default_catalog ""}}
{{- $catalog = "\"\" # workspace default"}}
{{- end}}
# This file defines dbt profiles for deployed dbt jobs.
# Note that for local development you should create your own, local profile.
# (see README.md).
my_dbt_project:
target: dev # default target
outputs:
dev:
type: databricks
method: http
catalog: {{$catalog}}
schema: "{{"{{"}} var('dev_schema') {{"}}"}}"
http_path: {{.http_path}}
# The workspace host / token are provided by Databricks
# see databricks.yml for the host used for 'dev'
host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}"
token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}"
prod:
type: databricks
method: http
catalog: {{$catalog}}
schema: {{.shared_schema}}
http_path: {{.http_path}}
# The workspace host / token are provided by Databricks
# see databricks.yml for the host used for 'dev'
host: "{{"{{"}} env_var('DBT_HOST') {{"}}"}}"
token: "{{"{{"}} env_var('DBT_ACCESS_TOKEN') {{"}}"}}"

View File

@ -0,0 +1,32 @@
name: '{{.project_name}}'
version: '1.0.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project.
profile: '{{.project_name}}'
# These configurations specify where dbt should look for different types of files.
# For Databricks asset bundles, we put everything in src, as you may have
# non-dbt resources in your project.
model-paths: ["src/models"]
analysis-paths: ["src/analyses"]
test-paths: ["src/tests"]
seed-paths: ["src/seeds"]
macro-paths: ["src/macros"]
snapshot-paths: ["src/snapshots"]
clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_packages"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
# In this example config, we tell dbt to build all models in the example/
# directory as views by default. These settings can be overridden in the
# individual model files using the `{{"{{"}} config(...) {{"}}"}}` macro.
models:
{{.project_name}}:
# Config indicated by + and applies to all files under models/example/
example:
+materialized: view

View File

@ -0,0 +1,37 @@
# This file defines prompts with defaults for dbt initializaton.
# It is used when the `dbt init` command is invoked.
#
fixed:
type: databricks
prompts:
host:
default: {{(regexp "^https?://").ReplaceAllString workspace_host ""}}
token:
hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
hide_input: true
http_path:
hint: 'HTTP path of SQL warehouse to use'
default: {{.http_path}}
{{- if eq .default_catalog ""}}
_choose_unity_catalog:
'use the default workspace catalog (or do not use Unity Catalog)':
_fixed_catalog: null
'specify a default catalog':
catalog:
hint: 'initial catalog'
{{- else}}
catalog:
hint: 'initial catalog'
default: {{.default_catalog}}
{{- end}}
schema:
{{- if (regexp "^yes").MatchString .personal_schemas}}
hint: 'personal schema where dbt will build objects during development, example: {{short_name}}'
{{- else}}
hint: 'default schema where dbt will build objects'
default: {{.shared_schema}}
{{- end}}
threads:
hint: 'threads to use during development, 1 or more'
type: 'int'
default: 4

View File

@ -0,0 +1,3 @@
## requirements-dev.txt: dependencies for local development.
dbt-databricks>=1.0.0,<2.0.0

View File

@ -0,0 +1,45 @@
resources:
jobs:
{{.project_name}}_job:
name: {{.project_name}}_job
schedule:
# Run every day at 9:27 AM
quartz_cron_expression: 21 27 9 * * ?
timezone_id: UTC
email_notifications:
on_failure:
- {{user_name}}
{{- $dev_schema := .shared_schema }}
{{- if (regexp "^yes").MatchString .personal_schemas}}
{{- $dev_schema = "${workspace.current_user.short_name}"}}
{{- end}}
tasks:
- task_key: dbt
dbt_task:
project_directory: ../
# The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml
profiles_directory: dbt_profiles/
commands:
- 'dbt deps --target=${bundle.target}'
- 'dbt seed --target=${bundle.target} --vars "{ dev_schema: {{$dev_schema}} }"'
- 'dbt run --target=${bundle.target} --vars "{ dev_schema: {{$dev_schema}} }"'
libraries:
- pypi:
package: dbt-databricks>=1.0.0,<2.0.0
new_cluster:
spark_version: {{template "latest_lts_dbr_version"}}
node_type_id: {{smallest_node_type}}
data_security_mode: SINGLE_USER
num_workers: 0
spark_conf:
spark.master: "local[*, 4]"
spark.databricks.cluster.profile: singleNode
custom_tags:
ResourceClass: SingleNode

View File

@ -0,0 +1,24 @@
{{- if eq (default_catalog) ""}}
{{- /* This workspace might not have Unity Catalog, */}}
{{- /* so let's not show both materialized views and streaming tables. */}}
{{- /* They're not supported without Unity Catalog! */}}
-- This model file defines a table called 'orders_daily'
{{"{{"}} config(materialized = 'table') {{"}}"}}
{{- else}}
-- This model file defines a materialized view called 'orders_daily'
--
-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables
-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.
{{"{{"}} config(materialized = 'materialized_view') {{"}}"}}
{{- end}}
select order_date, count(*) AS number_of_orders
from {{"{{"}} ref('orders_raw') {{"}}"}}
-- During development, only process a smaller range of data
{% if target.name != 'prod' %}
where order_date >= '2019-08-01' and order_date < '2019-09-01'
{% endif %}
group by order_date

View File

@ -0,0 +1,16 @@
-- This model file defines a streaming table called 'orders_raw'
--
-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/
-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables
-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.
{{"{{"}} config(materialized = 'streaming_table') {{"}}"}}
select
customer_name,
date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date,
order_number
from stream read_files(
"/databricks-datasets/retail-org/sales_orders/",
format => "json",
header => true
)

View File

@ -0,0 +1,21 @@
version: 2
models:
- name: orders_raw
description: "Raw ingested orders"
columns:
- name: customer_name
description: "The name of a customer"
tests:
- unique
- not_null
- name: orders_daily
description: "Number of orders by day"
columns:
- name: order_date
description: "The date on which orders took place"
tests:
- unique
- not_null

View File

@ -6,7 +6,7 @@ The '{{.project_name}}' project was generated by using the default-python templa
1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
2. Authenticate to your Databricks workspace:
2. Authenticate to your Databricks workspace, if you have not done so already:
```
$ databricks configure
```
@ -28,6 +28,11 @@ The '{{.project_name}}' project was generated by using the default-python templa
$ databricks bundle deploy --target prod
```
Note that the default job from the template has a schedule that runs every day
(defined in resources/{{.project_name}}_job.yml). The schedule
is paused when deploying in development mode (see
https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
5. To run a job or pipeline, use the "run" command:
```
$ databricks bundle run