steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

acc: test "deploy" and "validate -o json" in default-sql

This commit is contained in:
Denis Bilenko 2025-02-10 10:36:11 +01:00
parent c9beddc97a
commit c05b780c80
7 changed files with 538 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
acceptance/bundle/templates/default-sql/out.validate.dev.json Normal file
View File

@ -0,0 +1,140 @@
{
"bundle": {
"deployment": {
"lock": {
"enabled": false
}
},
"environment": "[TARGET]",
"git": {
"bundle_root_path": "."
},
"mode": "[TARGET]elopment",
"name": "my_default_sql",
"target": "[TARGET]",
"terraform": {
"exec_path": "[TERRAFORM]"
},
"uuid": "[UUID]"
},
"include": [
"resources/my_default_sql_sql.job.yml"
],
"presets": {
"jobs_max_concurrent_runs": 4,
"name_prefix": "[[TARGET] [USERNAME]] ",
"pipelines_[TARGET]elopment": true,
"tags": {
"[TARGET]": "[USERNAME]"
},
"trigger_pause_status": "PAUSED"
},
"resources": {
"jobs": {
"my_default_sql_sql_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {
"on_failure": [
"[USERNAME]"
]
},
"format": "MULTI_TASK",
"max_concurrent_runs": 4,
"name": "[[TARGET] [USERNAME]] my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
"default": "[USERNAME]",
"name": "schema"
},
{
"default": "[TARGET]",
"name": "bundle_target"
}
],
"permissions": [],
"queue": {
"enabled": true
},
"tags": {
"[TARGET]": "[USERNAME]"
},
"tasks": [
{
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_raw"
},
{
"depends_on": [
{
"task_key": "orders_raw"
}
],
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_daily"
}
],
"trigger": {
"pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
}
}
}
}
},
"sync": {
"paths": [
"."
]
},
"targets": null,
"variables": {
"catalog": {
"default": "main",
"description": "The catalog to use",
"value": "main"
},
"schema": {
"default": "[USERNAME]",
"description": "The schema to use",
"value": "[USERNAME]"
},
"warehouse_id": {
"default": "f00dcafe",
"description": "The warehouse to use",
"value": "f00dcafe"
}
},
"workspace": {
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts",
"current_user": {
"id": "[USERID]",
"short_name": "[USERNAME]",
"userName": "[USERNAME]"
},
"file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files",
"host": "[DATABRICKS_URL]",
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources",
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]",
"state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state"
}
}

134
acceptance/bundle/templates/default-sql/out.validate.prod.json Normal file
View File

@ -0,0 +1,134 @@
{
"bundle": {
"environment": "[TARGET]",
"git": {
"bundle_root_path": "."
},
"mode": "[TARGET]uction",
"name": "my_default_sql",
"target": "[TARGET]",
"terraform": {
"exec_path": "[TERRAFORM]"
},
"uuid": "[UUID]"
},
"include": [
"resources/my_default_sql_sql.job.yml"
],
"permissions": [
{
"level": "CAN_MANAGE",
"user_name": "[USERNAME]"
}
],
"resources": {
"jobs": {
"my_default_sql_sql_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {
"on_failure": [
"[USERNAME]"
]
},
"format": "MULTI_TASK",
"name": "my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
"default": "default",
"name": "schema"
},
{
"default": "[TARGET]",
"name": "bundle_target"
}
],
"permissions": [],
"queue": {
"enabled": true
},
"run_as": {
"user_name": "[USERNAME]"
},
"tags": {},
"tasks": [
{
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_raw"
},
{
"depends_on": [
{
"task_key": "orders_raw"
}
],
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_daily"
}
],
"trigger": {
"periodic": {
"interval": 1,
"unit": "DAYS"
}
}
}
}
},
"run_as": {
"user_name": "[USERNAME]"
},
"sync": {
"paths": [
"."
]
},
"targets": null,
"variables": {
"catalog": {
"default": "main",
"description": "The catalog to use",
"value": "main"
},
"schema": {
"default": "default",
"description": "The schema to use",
"value": "default"
},
"warehouse_id": {
"default": "f00dcafe",
"description": "The warehouse to use",
"value": "f00dcafe"
}
},
"workspace": {
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts",
"current_user": {
"id": "[USERID]",
"short_name": "[USERNAME]",
"userName": "[USERNAME]"
},
"file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files",
"host": "[DATABRICKS_URL]",
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources",
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]",
"state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state"
}
}

140
acceptance/bundle/templates/default-sql/output.txt
View File

@ -30,3 +30,143 @@ Workspace:
Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod
Validation OK!
>>> [CLI] bundle validate -t dev -o json
>>> [CLI] bundle validate -t prod -o json
>>> diff -u ../../out.validate.dev.json ../../out.validate.prod.json
--- ../../out.validate.dev.json [TIMESTAMP]
+++ ../../out.validate.prod.json [TIMESTAMP]
@@ -1,15 +1,10 @@
{
"bundle": {
- "deployment": {
- "lock": {
- "enabled": false
- }
- },
"environment": "[TARGET]",
"git": {
"bundle_root_path": "."
},
- "mode": "[TARGET]elopment",
+ "mode": "[TARGET]uction",
"name": "my_default_sql",
"target": "[TARGET]",
"terraform": {
\ No newline at end of file
@@ -20,15 +15,12 @@
"include": [
"resources/my_default_sql_sql.job.yml"
],
- "presets": {
- "jobs_max_concurrent_runs": 4,
- "name_prefix": "[[TARGET] [USERNAME]] ",
- "pipelines_[TARGET]elopment": true,
- "tags": {
- "[TARGET]": "[USERNAME]"
- },
- "trigger_pause_status": "PAUSED"
- },
+ "permissions": [
+ {
+ "level": "CAN_MANAGE",
+ "user_name": "[USERNAME]"
+ }
+ ],
"resources": {
"jobs": {
"my_default_sql_sql_job": {
\ No newline at end of file
@@ -43,15 +35,14 @@
]
},
"format": "MULTI_TASK",
- "max_concurrent_runs": 4,
- "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job",
+ "name": "my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
- "default": "[USERNAME]",
+ "default": "default",
"name": "schema"
},
{
\ No newline at end of file
@@ -63,9 +54,10 @@
"queue": {
"enabled": true
},
- "tags": {
- "[TARGET]": "[USERNAME]"
+ "run_as": {
+ "user_name": "[USERNAME]"
},
+ "tags": {},
"tasks": [
{
"sql_task": {
\ No newline at end of file
@@ -92,7 +84,6 @@
}
],
"trigger": {
- "pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
\ No newline at end of file
@@ -101,6 +92,9 @@
}
}
},
+ "run_as": {
+ "user_name": "[USERNAME]"
+ },
"sync": {
"paths": [
"."
\ No newline at end of file
@@ -114,9 +108,9 @@
"value": "main"
},
"schema": {
- "default": "[USERNAME]",
+ "default": "default",
"description": "The schema to use",
- "value": "[USERNAME]"
+ "value": "default"
},
"warehouse_id": {
"default": "f00dcafe",
\ No newline at end of file
Exit code: 1
>>> [CLI] bundle deploy -t dev
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files...
Deploying resources...
Updating deployment state...
Deployment complete!
>>> [CLI] bundle deploy -t prod
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files...
Error:
Exit code: 1
=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference
>>> [CLI] bundle validate -t dev -o json
>>> diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json
=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference
>>> [CLI] bundle validate -t prod -o json
>>> diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json

41
acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/dev/sync-snapshots/snapshot.json Normal file
View File

@ -0,0 +1,41 @@
{
"version": "v1",
"host": "[DATABRICKS_URL]",
"remote_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files",
"last_modified_times": {
".gitignore": "[TIMESTAMP]",
".vscode/extensions.json": "[TIMESTAMP]",
".vscode/settings.json": "[TIMESTAMP]",
"README.md": "[TIMESTAMP]",
"databricks.yml": "[TIMESTAMP]",
"resources/my_default_sql_sql.job.yml": "[TIMESTAMP]",
"scratch/README.md": "[TIMESTAMP]",
"scratch/exploration.ipynb": "[TIMESTAMP]",
"src/orders_daily.sql": "[TIMESTAMP]",
"src/orders_raw.sql": "[TIMESTAMP]"
},
"local_to_remote_names": {
".gitignore": ".gitignore",
".vscode/extensions.json": ".vscode/extensions.json",
".vscode/settings.json": ".vscode/settings.json",
"README.md": "README.md",
"databricks.yml": "databricks.yml",
"resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml",
"scratch/README.md": "scratch/README.md",
"scratch/exploration.ipynb": "scratch/exploration",
"src/orders_daily.sql": "src/orders_daily.sql",
"src/orders_raw.sql": "src/orders_raw.sql"
},
"remote_to_local_names": {
".gitignore": ".gitignore",
".vscode/extensions.json": ".vscode/extensions.json",
".vscode/settings.json": ".vscode/settings.json",
"README.md": "README.md",
"databricks.yml": "databricks.yml",
"resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml",
"scratch/README.md": "scratch/README.md",
"scratch/exploration": "scratch/exploration.ipynb",
"src/orders_daily.sql": "src/orders_daily.sql",
"src/orders_raw.sql": "src/orders_raw.sql"
}
}

41
acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/prod/sync-snapshots/snapshot.json Normal file
View File

@ -0,0 +1,41 @@
{
"version": "v1",
"host": "[DATABRICKS_URL]",
"remote_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files",
"last_modified_times": {
".gitignore": "[TIMESTAMP]",
".vscode/extensions.json": "[TIMESTAMP]",
".vscode/settings.json": "[TIMESTAMP]",
"README.md": "[TIMESTAMP]",
"databricks.yml": "[TIMESTAMP]",
"resources/my_default_sql_sql.job.yml": "[TIMESTAMP]",
"scratch/README.md": "[TIMESTAMP]",
"scratch/exploration.ipynb": "[TIMESTAMP]",
"src/orders_daily.sql": "[TIMESTAMP]",
"src/orders_raw.sql": "[TIMESTAMP]"
},
"local_to_remote_names": {
".gitignore": ".gitignore",
".vscode/extensions.json": ".vscode/extensions.json",
".vscode/settings.json": ".vscode/settings.json",
"README.md": "README.md",
"databricks.yml": "databricks.yml",
"resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml",
"scratch/README.md": "scratch/README.md",
"scratch/exploration.ipynb": "scratch/exploration",
"src/orders_daily.sql": "src/orders_daily.sql",
"src/orders_raw.sql": "src/orders_raw.sql"
},
"remote_to_local_names": {
".gitignore": ".gitignore",
".vscode/extensions.json": ".vscode/extensions.json",
".vscode/settings.json": ".vscode/settings.json",
"README.md": "README.md",
"databricks.yml": "databricks.yml",
"resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml",
"scratch/README.md": "scratch/README.md",
"scratch/exploration": "scratch/exploration.ipynb",
"src/orders_daily.sql": "src/orders_daily.sql",
"src/orders_raw.sql": "src/orders_raw.sql"
}
}

38
acceptance/bundle/templates/default-sql/script
View File

@ -4,5 +4,43 @@ cd output/my_default_sql
trace $CLI bundle validate -t dev
trace $CLI bundle validate -t prod
replace-target() {
sed "s/$1/[TARGET]/"
}
trace $CLI bundle validate -t dev -o json | replace-target dev > ../../out.validate.dev.json
trace $CLI bundle validate -t prod -o json | replace-target prod > ../../out.validate.prod.json
errcode trace diff -u ../../out.validate.dev.json ../../out.validate.prod.json
errcode trace $CLI bundle deploy -t dev
mv .databricks/bundle/dev/sync-snapshots/*.json .databricks/bundle/dev/sync-snapshots/snapshot.json
rm -r .databricks/bundle/dev/terraform
# TODO: This would be nice to include but .files currently has randomized order
#jq < .databricks/bundle/dev/deployment.json > .databricks/bundle/dev/deployment.jq.json
rm .databricks/bundle/dev/deployment.json
errcode trace $CLI bundle deploy -t prod
mv .databricks/bundle/prod/sync-snapshots/*.json .databricks/bundle/prod/sync-snapshots/snapshot.json
rm -r .databricks/bundle/prod/terraform
#jq < .databricks/bundle/prod/deployment.json > .databricks/bundle/prod/deployment.jq.json
rm .databricks/bundle/prod/deployment.json
#errcode trace $CLI bundle summary -t dev
#errcode trace $CLI bundle summary -t prod
#errcode trace $CLI bundle summary -t dev -o json > ../../out.summary.dev.json
#errcode trace $CLI bundle summary -t prod -o json > ../../out.summary.prod.json
printf "\n=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference"
errcode trace $CLI bundle validate -t dev -o json | replace-target dev > ../../out.validate.2nd.dev.json
errcode trace diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json # should be the same
rm ../../out.validate.2nd.dev.json
printf "\n=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference"
errcode trace $CLI bundle validate -t prod -o json | replace-target prod > ../../out.validate.2nd.prod.json
errcode trace diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json # should be the same
rm ../../out.validate.2nd.prod.json
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore

4
acceptance/bundle/templates/test.toml
View File

@ -1,2 +1,6 @@
# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on.
LocalOnly = true
[[Repls]]
Old = '\d\d\d\d-\d\d-\d\d(T| )\d\d:\d\d:\d\d(\.\d+)?(\+\d\d:\d\d)?Z?'
New = '[TIMESTAMP]'