From c3c4dcd8947c5af12ad142d3931158e2cf6d1107 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 15 Jan 2025 10:35:58 +0100 Subject: [PATCH] Update --- .../bundle/init/default-python/output.txt | 30 ---- .../{init => templates}/dbt-sql/input.json | 0 .../{init => templates}/dbt-sql/output.txt | 9 + .../dbt-sql/output/my_dbt_sql/.gitignore | 0 .../my_dbt_sql/.vscode/__builtins__.pyi | 0 .../output/my_dbt_sql/.vscode/extensions.json | 0 .../output/my_dbt_sql/.vscode/settings.json | 0 .../dbt-sql/output/my_dbt_sql/README.md | 2 +- .../dbt-sql/output/my_dbt_sql/databricks.yml | 2 +- .../my_dbt_sql/dbt_profiles/profiles.yml | 0 .../dbt-sql/output/my_dbt_sql/dbt_project.yml | 0 .../output/my_dbt_sql/profile_template.yml | 0 .../output/my_dbt_sql/requirements-dev.txt | 0 .../my_dbt_sql/resources/my_dbt_sql.job.yml | 0 .../output/my_dbt_sql/src/analyses/.gitkeep | 0 .../output/my_dbt_sql/src/macros/.gitkeep | 0 .../src/models/example/orders_daily.sql | 4 +- .../src/models/example/orders_raw.sql | 2 +- .../my_dbt_sql/src/models/example/schema.yml | 0 .../output/my_dbt_sql/src/seeds/.gitkeep | 0 .../output/my_dbt_sql/src/snapshots/.gitkeep | 0 .../output/my_dbt_sql/src/tests/.gitkeep | 0 .../bundle/{init => templates}/dbt-sql/script | 0 .../default-python/input.json | 0 .../templates/default-python/output.txt | 157 ++++++++++++++++++ .../output/my_default_python/.gitignore | 0 .../.vscode/__builtins__.pyi | 0 .../my_default_python/.vscode/extensions.json | 0 .../my_default_python/.vscode/settings.json | 0 .../output/my_default_python/README.md | 0 .../output/my_default_python/databricks.yml | 2 +- .../my_default_python/fixtures/.gitkeep | 0 .../output/my_default_python/pytest.ini | 0 .../my_default_python/requirements-dev.txt | 0 .../resources/my_default_python.job.yml | 0 .../resources/my_default_python.pipeline.yml | 0 .../my_default_python/scratch/README.md | 0 .../output/my_default_python/setup.py | 2 +- .../my_default_python/src/dlt_pipeline.ipynb | 6 +- .../src/my_default_python/__init__.py | 0 .../src/my_default_python/main.py | 0 .../my_default_python/src/notebook.ipynb | 8 +- .../my_default_python/tests/main_test.py | 0 .../{init => templates}/default-python/script | 0 .../default-sql/input.json | 0 .../default-sql/output.txt | 10 ++ .../output/my_default_sql/.gitignore | 0 .../my_default_sql/.vscode/extensions.json | 0 .../my_default_sql/.vscode/settings.json | 0 .../output/my_default_sql/README.md | 0 .../output/my_default_sql/databricks.yml | 2 +- .../resources/my_default_sql_sql.job.yml | 0 .../output/my_default_sql/scratch/README.md | 0 .../my_default_sql/scratch/exploration.ipynb | 2 +- .../my_default_sql/src/orders_daily.sql | 2 +- .../output/my_default_sql/src/orders_raw.sql | 0 .../{init => templates}/default-sql/script | 0 57 files changed, 193 insertions(+), 47 deletions(-) delete mode 100644 acceptance/bundle/init/default-python/output.txt rename acceptance/bundle/{init => templates}/dbt-sql/input.json (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output.txt (88%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/.gitignore (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/.vscode/extensions.json (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/.vscode/settings.json (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/README.md (99%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/databricks.yml (96%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/dbt_project.yml (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/profile_template.yml (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/requirements-dev.txt (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql (86%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql (96%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep (100%) rename acceptance/bundle/{init => templates}/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep (100%) rename acceptance/bundle/{init => templates}/dbt-sql/script (100%) rename acceptance/bundle/{init => templates}/default-python/input.json (100%) create mode 100644 acceptance/bundle/templates/default-python/output.txt rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/.gitignore (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/.vscode/__builtins__.pyi (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/.vscode/extensions.json (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/.vscode/settings.json (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/README.md (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/databricks.yml (95%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/fixtures/.gitkeep (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/pytest.ini (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/requirements-dev.txt (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/resources/my_default_python.job.yml (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/resources/my_default_python.pipeline.yml (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/scratch/README.md (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/setup.py (96%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/src/dlt_pipeline.ipynb (91%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/src/my_default_python/__init__.py (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/src/my_default_python/main.py (100%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/src/notebook.ipynb (89%) rename acceptance/bundle/{init => templates}/default-python/output/my_default_python/tests/main_test.py (100%) rename acceptance/bundle/{init => templates}/default-python/script (100%) rename acceptance/bundle/{init => templates}/default-sql/input.json (100%) rename acceptance/bundle/{init => templates}/default-sql/output.txt (82%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/.gitignore (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/.vscode/extensions.json (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/.vscode/settings.json (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/README.md (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/databricks.yml (96%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/scratch/README.md (100%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/scratch/exploration.ipynb (92%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/src/orders_daily.sql (86%) rename acceptance/bundle/{init => templates}/default-sql/output/my_default_sql/src/orders_raw.sql (100%) rename acceptance/bundle/{init => templates}/default-sql/script (100%) diff --git a/acceptance/bundle/init/default-python/output.txt b/acceptance/bundle/init/default-python/output.txt deleted file mode 100644 index a6e98a7d5..000000000 --- a/acceptance/bundle/init/default-python/output.txt +++ /dev/null @@ -1,30 +0,0 @@ - ->>> $CLI bundle init default-python --config-file ./input.json --output-dir output - -Welcome to the default Python template for Databricks Asset Bundles! -Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): http://$DATABRICKS_HOST - -✨ Your new project has been created in the 'my_default_python' directory! - -Please refer to the README.md file for "getting started" instructions. -See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. - ->>> $CLI bundle validate -t dev -Name: my_default_python -Target: dev -Workspace: - Host: http://$DATABRICKS_HOST - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev - -Validation OK! - ->>> $CLI bundle validate -t prod -Name: my_default_python -Target: prod -Workspace: - Host: http://$DATABRICKS_HOST - User: $USERNAME - Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod - -Validation OK! diff --git a/acceptance/bundle/init/dbt-sql/input.json b/acceptance/bundle/templates/dbt-sql/input.json similarity index 100% rename from acceptance/bundle/init/dbt-sql/input.json rename to acceptance/bundle/templates/dbt-sql/input.json diff --git a/acceptance/bundle/init/dbt-sql/output.txt b/acceptance/bundle/templates/dbt-sql/output.txt similarity index 88% rename from acceptance/bundle/init/dbt-sql/output.txt rename to acceptance/bundle/templates/dbt-sql/output.txt index 8e9273f5d..0459528a9 100644 --- a/acceptance/bundle/init/dbt-sql/output.txt +++ b/acceptance/bundle/templates/dbt-sql/output.txt @@ -30,3 +30,12 @@ Workspace: Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod Validation OK! + +>>> ruff format --diff +warning: No Python files found under the given path(s) + +Exit code: 0 + +>>> ruff clean + +Exit code: 0 diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.gitignore rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/__builtins__.pyi diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/extensions.json b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/extensions.json rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/extensions.json diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/settings.json b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/.vscode/settings.json rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.vscode/settings.json diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/README.md b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md similarity index 99% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/README.md rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md index 756a2eda4..7211bea15 100644 --- a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/README.md +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/README.md @@ -47,7 +47,7 @@ and deployment to production (using Databricks Asset Bundles). (see https://docs.databricks.com/dev-tools/auth/pat.html). You can use OAuth as an alternative, but this currently requires manual configuration. See https://github.com/databricks/dbt-databricks/blob/main/docs/oauth.md - for general instructions, or https://community.databricks.com/t5/technical-blog/using-dbt-core-with-oauth-on-azure-databricks/ba-p/46605 + for general instructions, or https://community.databricks.com/t5/technical-blog/using-dbt-core-with-oauth-on-azure-databricks/ba-p/ for advice on setting up OAuth for Azure Databricks. To setup up additional profiles, such as a 'prod' profile, diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/databricks.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml similarity index 96% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/databricks.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml index 803c4cf38..0405960d2 100644 --- a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/databricks.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/databricks.yml @@ -3,7 +3,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_dbt_sql - uuid: 9d4ad8f6-850c-45fb-b51a-04fa314564de + uuid: include: - resources/*.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_profiles/profiles.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/dbt_project.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/dbt_project.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/dbt_project.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/profile_template.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/profile_template.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/profile_template.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/requirements-dev.txt b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/requirements-dev.txt rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/requirements-dev.txt diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/analyses/.gitkeep diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/macros/.gitkeep diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql similarity index 86% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql index e32736ceb..93e8b4b35 100644 --- a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_daily.sql @@ -2,7 +2,7 @@ -- This model file defines a materialized view called 'orders_daily' -- -- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables --- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/. {{ config(materialized = 'materialized_view') }} select order_date, count(*) AS number_of_orders @@ -11,7 +11,7 @@ from {{ ref('orders_raw') }} -- During development, only process a smaller range of data {% if target.name != 'prod' %} -where order_date >= '2019-08-01' and order_date < '2019-09-01' +where order_date >= '-08-01' and order_date < '-09-01' {% endif %} group by order_date diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql similarity index 96% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql index 8faf8f38b..193d7b888 100644 --- a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/orders_raw.sql @@ -2,7 +2,7 @@ -- -- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ -- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables --- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/. {{ config(materialized = 'streaming_table') }} select diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/models/example/schema.yml diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/seeds/.gitkeep diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/snapshots/.gitkeep diff --git a/acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep similarity index 100% rename from acceptance/bundle/init/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/src/tests/.gitkeep diff --git a/acceptance/bundle/init/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script similarity index 100% rename from acceptance/bundle/init/dbt-sql/script rename to acceptance/bundle/templates/dbt-sql/script diff --git a/acceptance/bundle/init/default-python/input.json b/acceptance/bundle/templates/default-python/input.json similarity index 100% rename from acceptance/bundle/init/default-python/input.json rename to acceptance/bundle/templates/default-python/input.json diff --git a/acceptance/bundle/templates/default-python/output.txt b/acceptance/bundle/templates/default-python/output.txt new file mode 100644 index 000000000..f1f2ba5c2 --- /dev/null +++ b/acceptance/bundle/templates/default-python/output.txt @@ -0,0 +1,157 @@ + +>>> $CLI bundle init default-python --config-file ./input.json --output-dir output + +Welcome to the default Python template for Databricks Asset Bundles! +Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): http://$DATABRICKS_HOST + +✨ Your new project has been created in the 'my_default_python' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> $CLI bundle validate -t dev +Name: my_default_python +Target: dev +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev + +Validation OK! + +>>> $CLI bundle validate -t prod +Name: my_default_python +Target: prod +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod + +Validation OK! + +>>> ruff format --diff +--- scratch/exploration.ipynb:cell 1 ++++ scratch/exploration.ipynb:cell 1 +--- scratch/exploration.ipynb:cell 2 ++++ scratch/exploration.ipynb:cell 2 +@@ -1,5 +1,6 @@ + import sys +-sys.path.append('../src') ++ ++sys.path.append("../src") + from my_default_python import main + + main.get_taxis(spark).show(10) + +--- setup.py ++++ setup.py +@@ -5,11 +5,13 @@ + be executed directly. See README.md for how to deploy, test, and run + the my_default_python project. + """ ++ + from setuptools import setup, find_packages + + import sys +-sys.path.append('./src') + ++sys.path.append("./src") ++ + import datetime + import my_default_python + +@@ -17,17 +19,15 @@ + name="my_default_python", + # We use timestamp as Local version identifier (https://peps.python.org/pep-/#local-version-identifiers.) + # to ensure that changes to wheel package are picked up when used on all-purpose clusters +- version=my_default_python.__version__ + "+" + datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"), ++ version=my_default_python.__version__ ++ + "+" ++ + datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"), + url="https://databricks.com", + author="$USERNAME", + description="wheel file based on my_default_python/src", +- packages=find_packages(where='./src'), +- package_dir={'': 'src'}, +- entry_points={ +- "packages": [ +- "main=my_default_python.main:main" +- ] +- }, ++ packages=find_packages(where="./src"), ++ package_dir={"": "src"}, ++ entry_points={"packages": ["main=my_default_python.main:main"]}, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + +--- src/dlt_pipeline.ipynb:cell 2 ++++ src/dlt_pipeline.ipynb:cell 2 +@@ -1,6 +1,7 @@ + # Import DLT and src/my_default_python + import dlt + import sys ++ + sys.path.append(spark.conf.get("bundle.sourcePath", ".")) + from pyspark.sql.functions import expr + from my_default_python import main +--- src/dlt_pipeline.ipynb:cell 3 ++++ src/dlt_pipeline.ipynb:cell 3 +@@ -1,7 +1,8 @@ + @dlt.view + def taxi_raw(): +- return main.get_taxis(spark) ++ return main.get_taxis(spark) ++ + + @dlt.table + def filtered_taxis(): +- return dlt.read("taxi_raw").filter(expr("fare_amount < 30")) ++ return dlt.read("taxi_raw").filter(expr("fare_amount < 30")) + +--- src/my_default_python/main.py ++++ src/my_default_python/main.py +@@ -1,21 +1,25 @@ + from pyspark.sql import SparkSession, DataFrame + ++ + def get_taxis(spark: SparkSession) -> DataFrame: +- return spark.read.table("samples.nyctaxi.trips") ++ return spark.read.table("samples.nyctaxi.trips") + + + # Create a new Databricks Connect session. If this fails, + # check that you have configured Databricks Connect correctly. + # See https://docs.databricks.com/dev-tools/databricks-connect.html. + def get_spark() -> SparkSession: +- try: +- from databricks.connect import DatabricksSession +- return DatabricksSession.builder.getOrCreate() +- except ImportError: +- return SparkSession.builder.getOrCreate() ++ try: ++ from databricks.connect import DatabricksSession ++ ++ return DatabricksSession.builder.getOrCreate() ++ except ImportError: ++ return SparkSession.builder.getOrCreate() ++ + + def main(): +- get_taxis(get_spark()).show(5) ++ get_taxis(get_spark()).show(5) ++ + +-if __name__ == '__main__': +- main() ++if __name__ == "__main__": ++ main() + +4 files would be reformatted, 3 files already formatted + +Exit code: 1 + +>>> ruff clean +Removing cache at: .ruff_cache + +Exit code: 0 diff --git a/acceptance/bundle/init/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/.gitignore rename to acceptance/bundle/templates/default-python/output/my_default_python/.gitignore diff --git a/acceptance/bundle/init/default-python/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/.vscode/__builtins__.pyi rename to acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi diff --git a/acceptance/bundle/init/default-python/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/.vscode/extensions.json rename to acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json diff --git a/acceptance/bundle/init/default-python/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/.vscode/settings.json rename to acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json diff --git a/acceptance/bundle/init/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/README.md similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/README.md rename to acceptance/bundle/templates/default-python/output/my_default_python/README.md diff --git a/acceptance/bundle/init/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml similarity index 95% rename from acceptance/bundle/init/default-python/output/my_default_python/databricks.yml rename to acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml index 4a3ec2b41..94fc6107c 100644 --- a/acceptance/bundle/init/default-python/output/my_default_python/databricks.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml @@ -2,7 +2,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_default_python - uuid: 3c5cdb6b-9e42-46f3-a33c-54769acda6bf + uuid: include: - resources/*.yml diff --git a/acceptance/bundle/init/default-python/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/fixtures/.gitkeep rename to acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep diff --git a/acceptance/bundle/init/default-python/output/my_default_python/pytest.ini b/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/pytest.ini rename to acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini diff --git a/acceptance/bundle/init/default-python/output/my_default_python/requirements-dev.txt b/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/requirements-dev.txt rename to acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt diff --git a/acceptance/bundle/init/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/resources/my_default_python.job.yml rename to acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml diff --git a/acceptance/bundle/init/default-python/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/resources/my_default_python.pipeline.yml rename to acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml diff --git a/acceptance/bundle/init/default-python/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/scratch/README.md rename to acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md diff --git a/acceptance/bundle/init/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py similarity index 96% rename from acceptance/bundle/init/default-python/output/my_default_python/setup.py rename to acceptance/bundle/templates/default-python/output/my_default_python/setup.py index 05114b037..d2db5af0d 100644 --- a/acceptance/bundle/init/default-python/output/my_default_python/setup.py +++ b/acceptance/bundle/templates/default-python/output/my_default_python/setup.py @@ -15,7 +15,7 @@ import my_default_python setup( name="my_default_python", - # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.) + # We use timestamp as Local version identifier (https://peps.python.org/pep-/#local-version-identifiers.) # to ensure that changes to wheel package are picked up when used on all-purpose clusters version=my_default_python.__version__ + "+" + datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S"), url="https://databricks.com", diff --git a/acceptance/bundle/init/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb similarity index 91% rename from acceptance/bundle/init/default-python/output/my_default_python/src/dlt_pipeline.ipynb rename to acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb index 569617cfb..71b5bee6e 100644 --- a/acceptance/bundle/init/default-python/output/my_default_python/src/dlt_pipeline.ipynb +++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb @@ -6,7 +6,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec", + "nuid": "", "showTitle": false, "title": "" } @@ -24,7 +24,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", + "nuid": "", "showTitle": false, "title": "" } @@ -46,7 +46,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "3fc19dba-61fd-4a89-8f8c-24fee63bfb14", + "nuid": "", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/init/default-python/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/src/my_default_python/__init__.py rename to acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py diff --git a/acceptance/bundle/init/default-python/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/src/my_default_python/main.py rename to acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py diff --git a/acceptance/bundle/init/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb similarity index 89% rename from acceptance/bundle/init/default-python/output/my_default_python/src/notebook.ipynb rename to acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb index 1afb15b6b..8e38f12cb 100644 --- a/acceptance/bundle/init/default-python/output/my_default_python/src/notebook.ipynb +++ b/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb @@ -6,7 +6,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "ee353e42-ff58-4955-9608-12865bd0950e", + "nuid": "", "showTitle": false, "title": "" } @@ -33,11 +33,11 @@ "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 + "byteLimit": , + "rowLimit": }, "inputWidgets": {}, - "nuid": "6bca260b-13d1-448f-8082-30b60a85c9ae", + "nuid": "", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/init/default-python/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py similarity index 100% rename from acceptance/bundle/init/default-python/output/my_default_python/tests/main_test.py rename to acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py diff --git a/acceptance/bundle/init/default-python/script b/acceptance/bundle/templates/default-python/script similarity index 100% rename from acceptance/bundle/init/default-python/script rename to acceptance/bundle/templates/default-python/script diff --git a/acceptance/bundle/init/default-sql/input.json b/acceptance/bundle/templates/default-sql/input.json similarity index 100% rename from acceptance/bundle/init/default-sql/input.json rename to acceptance/bundle/templates/default-sql/input.json diff --git a/acceptance/bundle/init/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt similarity index 82% rename from acceptance/bundle/init/default-sql/output.txt rename to acceptance/bundle/templates/default-sql/output.txt index 8a6e15bac..85f63507f 100644 --- a/acceptance/bundle/init/default-sql/output.txt +++ b/acceptance/bundle/templates/default-sql/output.txt @@ -30,3 +30,13 @@ Workspace: Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod Validation OK! + +>>> ruff format --diff +error: Failed to parse scratch/exploration.ipynb:1:2:15: Simple statements must be separated by newlines or semicolons + +Exit code: 2 + +>>> ruff clean +Removing cache at: .ruff_cache + +Exit code: 0 diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/.gitignore rename to acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/.vscode/extensions.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/.vscode/extensions.json rename to acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/extensions.json diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/.vscode/settings.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/.vscode/settings.json rename to acceptance/bundle/templates/default-sql/output/my_default_sql/.vscode/settings.json diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/README.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/README.md similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/README.md rename to acceptance/bundle/templates/default-sql/output/my_default_sql/README.md diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/databricks.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml similarity index 96% rename from acceptance/bundle/init/default-sql/output/my_default_sql/databricks.yml rename to acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml index ac8266b89..16687ac47 100644 --- a/acceptance/bundle/init/default-sql/output/my_default_sql/databricks.yml +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/databricks.yml @@ -2,7 +2,7 @@ # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: name: my_default_sql - uuid: 631398bf-1d77-42ce-ba4f-9bb29dd64b5a + uuid: include: - resources/*.yml diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml b/acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml rename to acceptance/bundle/templates/default-sql/output/my_default_sql/resources/my_default_sql_sql.job.yml diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/scratch/README.md b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/scratch/README.md rename to acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/README.md diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/scratch/exploration.ipynb b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb similarity index 92% rename from acceptance/bundle/init/default-sql/output/my_default_sql/scratch/exploration.ipynb rename to acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb index becee5fba..c3fd072e5 100644 --- a/acceptance/bundle/init/default-sql/output/my_default_sql/scratch/exploration.ipynb +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/scratch/exploration.ipynb @@ -7,7 +7,7 @@ "application/vnd.databricks.v1+cell": { "cellMetadata": {}, "inputWidgets": {}, - "nuid": "dc8c630c-1ea0-42e4-873f-e4dec4d3d416", + "nuid": "", "showTitle": false, "title": "" } diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/src/orders_daily.sql b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql similarity index 86% rename from acceptance/bundle/init/default-sql/output/my_default_sql/src/orders_daily.sql rename to acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql index ea7b80b54..437de30d9 100644 --- a/acceptance/bundle/init/default-sql/output/my_default_sql/src/orders_daily.sql +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_daily.sql @@ -15,7 +15,7 @@ WHERE if( true, -- During development, only process a smaller range of data - order_date >= '2019-08-01' AND order_date < '2019-09-01' + order_date >= '-08-01' AND order_date < '-09-01' ) GROUP BY order_date diff --git a/acceptance/bundle/init/default-sql/output/my_default_sql/src/orders_raw.sql b/acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql similarity index 100% rename from acceptance/bundle/init/default-sql/output/my_default_sql/src/orders_raw.sql rename to acceptance/bundle/templates/default-sql/output/my_default_sql/src/orders_raw.sql diff --git a/acceptance/bundle/init/default-sql/script b/acceptance/bundle/templates/default-sql/script similarity index 100% rename from acceptance/bundle/init/default-sql/script rename to acceptance/bundle/templates/default-sql/script