From 2d65e719799c7194848ecc41d7927e5425df8661 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 14 Feb 2025 10:37:00 +0100 Subject: [PATCH] reorg tests --- .../out.compare-vs-serverless.diff | 39 ------------------- .../default-python-classic/test.toml | 7 ---- .../classic}/input.json | 3 +- .../classic/out.compare-vs-serverless.diff | 0 .../classic}/output.txt | 0 .../classic}/script | 2 +- .../default-python/classic/test.toml | 1 + .../{ => serverless}/input.json | 3 +- .../{ => serverless}/output.txt | 0 .../.vscode/__builtins__.pyi | 0 .../my_default_python/.vscode/extensions.json | 0 .../my_default_python/.vscode/settings.json | 0 .../output/my_default_python/README.md | 0 .../output/my_default_python/databricks.yml | 0 .../my_default_python/fixtures/.gitkeep | 0 .../output/my_default_python/out.gitignore | 0 .../output/my_default_python/pytest.ini | 0 .../my_default_python/requirements-dev.txt | 0 .../resources/my_default_python.job.yml | 0 .../resources/my_default_python.pipeline.yml | 0 .../my_default_python/scratch/README.md | 0 .../scratch/exploration.ipynb | 0 .../output/my_default_python/setup.py | 0 .../my_default_python/src/dlt_pipeline.ipynb | 0 .../src/my_default_python/__init__.py | 0 .../src/my_default_python/main.py | 0 .../my_default_python/src/notebook.ipynb | 0 .../my_default_python/tests/main_test.py | 0 .../default-python/{ => serverless}/script | 0 29 files changed, 6 insertions(+), 49 deletions(-) delete mode 100644 acceptance/bundle/templates/default-python-classic/out.compare-vs-serverless.diff delete mode 100644 acceptance/bundle/templates/default-python-classic/test.toml rename acceptance/bundle/templates/{default-python-classic => default-python/classic}/input.json (66%) create mode 100644 acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff rename acceptance/bundle/templates/{default-python-classic => default-python/classic}/output.txt (100%) rename acceptance/bundle/templates/{default-python-classic => default-python/classic}/script (81%) create mode 100644 acceptance/bundle/templates/default-python/classic/test.toml rename acceptance/bundle/templates/default-python/{ => serverless}/input.json (65%) rename acceptance/bundle/templates/default-python/{ => serverless}/output.txt (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/.vscode/__builtins__.pyi (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/.vscode/extensions.json (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/.vscode/settings.json (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/README.md (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/databricks.yml (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/fixtures/.gitkeep (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/out.gitignore (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/pytest.ini (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/requirements-dev.txt (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/resources/my_default_python.job.yml (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/resources/my_default_python.pipeline.yml (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/scratch/README.md (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/scratch/exploration.ipynb (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/setup.py (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/src/dlt_pipeline.ipynb (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/src/my_default_python/__init__.py (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/src/my_default_python/main.py (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/src/notebook.ipynb (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/output/my_default_python/tests/main_test.py (100%) rename acceptance/bundle/templates/default-python/{ => serverless}/script (100%) diff --git a/acceptance/bundle/templates/default-python-classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python-classic/out.compare-vs-serverless.diff deleted file mode 100644 index 41d68edd0..000000000 --- a/acceptance/bundle/templates/default-python-classic/out.compare-vs-serverless.diff +++ /dev/null @@ -1,39 +0,0 @@ ---- [TESTROOT]/bundle/templates/default-python-classic/../default-python/output/my_default_python/resources/my_default_python.job.yml -+++ output/my_default_python/resources/my_default_python.job.yml -@@ -17,4 +17,5 @@ - tasks: - - task_key: notebook_task -+ job_cluster_key: job_cluster - notebook_task: - notebook_path: ../src/notebook.ipynb -@@ -29,16 +30,19 @@ - depends_on: - - task_key: refresh_pipeline -- environment_key: default -+ job_cluster_key: job_cluster - python_wheel_task: - package_name: my_default_python - entry_point: main -- # A list of task execution environment specifications that can be referenced by tasks of this job. -- environments: -- - environment_key: default -- -- # Full documentation of this spec can be found at: -- # https://docs.databricks.com/api/workspace/jobs/create#environments-spec -- spec: -- client: "1" -- dependencies: -- - ../dist/*.whl -+ libraries: -+ # By default we just include the .whl file generated for the my_default_python package. -+ # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html -+ # for more information on how to add other libraries. -+ - whl: ../dist/*.whl -+ job_clusters: -+ - job_cluster_key: job_cluster -+ new_cluster: -+ spark_version: 15.4.x-scala2.12 -+ node_type_id: i3.xlarge -+ autoscale: -+ min_workers: 1 -+ max_workers: 4 diff --git a/acceptance/bundle/templates/default-python-classic/test.toml b/acceptance/bundle/templates/default-python-classic/test.toml deleted file mode 100644 index 6d62af2b9..000000000 --- a/acceptance/bundle/templates/default-python-classic/test.toml +++ /dev/null @@ -1,7 +0,0 @@ -# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. -LocalOnly = true -SaveRepls = true - -[[Server]] -Pattern = "GET /api/2.0/settings-api/workspace/{workspaceId}/serverless_job_nb" -Response.Body = '{}' diff --git a/acceptance/bundle/templates/default-python-classic/input.json b/acceptance/bundle/templates/default-python/classic/input.json similarity index 66% rename from acceptance/bundle/templates/default-python-classic/input.json rename to acceptance/bundle/templates/default-python/classic/input.json index 3e1d79c68..2c4416c00 100644 --- a/acceptance/bundle/templates/default-python-classic/input.json +++ b/acceptance/bundle/templates/default-python/classic/input.json @@ -2,5 +2,6 @@ "project_name": "my_default_python", "include_notebook": "yes", "include_dlt": "yes", - "include_python": "yes" + "include_python": "yes", + "serverless": "no" } diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance/bundle/templates/default-python-classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt similarity index 100% rename from acceptance/bundle/templates/default-python-classic/output.txt rename to acceptance/bundle/templates/default-python/classic/output.txt diff --git a/acceptance/bundle/templates/default-python-classic/script b/acceptance/bundle/templates/default-python/classic/script similarity index 81% rename from acceptance/bundle/templates/default-python-classic/script rename to acceptance/bundle/templates/default-python/classic/script index 655c5bcb7..b203c7e5f 100644 --- a/acceptance/bundle/templates/default-python-classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -10,6 +10,6 @@ mv .gitignore out.gitignore cd ../../ # Calculate the difference from the standard (serverless) template -diff.py $TESTDIR/../default-python/output output/ > out.compare-vs-serverless.diff +diff.py $TESTDIR/../serverless/output output/ > out.compare-vs-serverless.diff rm -fr output diff --git a/acceptance/bundle/templates/default-python/classic/test.toml b/acceptance/bundle/templates/default-python/classic/test.toml new file mode 100644 index 000000000..d5712aea5 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/test.toml @@ -0,0 +1 @@ +SaveRepls = true diff --git a/acceptance/bundle/templates/default-python/input.json b/acceptance/bundle/templates/default-python/serverless/input.json similarity index 65% rename from acceptance/bundle/templates/default-python/input.json rename to acceptance/bundle/templates/default-python/serverless/input.json index 3e1d79c68..b1ae9a2ba 100644 --- a/acceptance/bundle/templates/default-python/input.json +++ b/acceptance/bundle/templates/default-python/serverless/input.json @@ -2,5 +2,6 @@ "project_name": "my_default_python", "include_notebook": "yes", "include_dlt": "yes", - "include_python": "yes" + "include_python": "yes", + "serverless": "yes" } diff --git a/acceptance/bundle/templates/default-python/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt similarity index 100% rename from acceptance/bundle/templates/default-python/output.txt rename to acceptance/bundle/templates/default-python/serverless/output.txt diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/README.md rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/setup.py similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/setup.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/setup.py diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/dlt_pipeline.ipynb similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/dlt_pipeline.ipynb diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/__init__.py similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/__init__.py diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/serverless/script similarity index 100% rename from acceptance/bundle/templates/default-python/script rename to acceptance/bundle/templates/default-python/serverless/script