steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix default-sql

This commit is contained in:
Denis Bilenko 2025-02-12 11:40:54 +01:00
parent adee4c9998
commit e97d060478
4 changed files with 432 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
acceptance/bundle/templates/default-sql/out.summary.dev.json Normal file
View File

@ -0,0 +1,140 @@
{
"variables": {
"catalog": {
"default": "main",
"description": "The catalog to use",
"value": "main"
},
"schema": {
"default": "[USERNAME]",
"description": "The schema to use",
"value": "[USERNAME]"
},
"warehouse_id": {
"default": "f00dcafe",
"description": "The warehouse to use",
"value": "f00dcafe"
}
},
"bundle": {
"name": "my_default_sql",
"target": "[TARGET]",
"environment": "[TARGET]",
"terraform": {
"exec_path": "[TERRAFORM]"
},
"git": {
"bundle_root_path": "."
},
"mode": "[TARGET]elopment",
"deployment": {
"lock": {
"enabled": false
}
},
"uuid": "[UUID]"
},
"include": [
"resources/my_default_sql_sql.job.yml"
],
"workspace": {
"host": "[DATABRICKS_URL]",
"current_user": {
"id": "[USERID]",
"short_name": "[USERNAME]",
"userName": "[USERNAME]"
},
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]",
"file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files",
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources",
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts",
"state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state"
},
"resources": {
"jobs": {
"my_default_sql_sql_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {
"on_failure": [
"[USERNAME]"
]
},
"format": "MULTI_TASK",
"id": "0",
"max_concurrent_runs": 4,
"name": "[[TARGET] [USERNAME]] my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
"default": "[USERNAME]",
"name": "schema"
},
{
"default": "[TARGET]",
"name": "bundle_target"
}
],
"queue": {
"enabled": true
},
"tags": {
"[TARGET]": "[USERNAME]"
},
"tasks": [
{
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_raw"
},
{
"depends_on": [
{
"task_key": "orders_raw"
}
],
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_daily"
}
],
"trigger": {
"pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
}
},
"url": "[DATABRICKS_URL]/jobs/0?o=900800700600"
}
}
},
"sync": {
"paths": [
"."
]
},
"presets": {
"name_prefix": "[[TARGET] [USERNAME]] ",
"pipelines_[TARGET]elopment": true,
"trigger_pause_status": "PAUSED",
"jobs_max_concurrent_runs": 4,
"tags": {
"[TARGET]": "[USERNAME]"
}
}
}

137
acceptance/bundle/templates/default-sql/out.summary.prod.json Normal file
View File

@ -0,0 +1,137 @@
{
"variables": {
"catalog": {
"default": "main",
"description": "The catalog to use",
"value": "main"
},
"schema": {
"default": "default",
"description": "The schema to use",
"value": "default"
},
"warehouse_id": {
"default": "f00dcafe",
"description": "The warehouse to use",
"value": "f00dcafe"
}
},
"bundle": {
"name": "my_default_sql",
"target": "[TARGET]",
"environment": "[TARGET]",
"terraform": {
"exec_path": "[TERRAFORM]"
},
"git": {
"bundle_root_path": "."
},
"mode": "[TARGET]uction",
"deployment": {
"lock": {}
},
"uuid": "[UUID]"
},
"include": [
"resources/my_default_sql_sql.job.yml"
],
"workspace": {
"host": "[DATABRICKS_URL]",
"current_user": {
"id": "[USERID]",
"short_name": "[USERNAME]",
"userName": "[USERNAME]"
},
"root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]",
"file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files",
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources",
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts",
"state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state"
},
"resources": {
"jobs": {
"my_default_sql_sql_job": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {
"on_failure": [
"[USERNAME]"
]
},
"format": "MULTI_TASK",
"id": "0",
"name": "my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
"default": "default",
"name": "schema"
},
{
"default": "[TARGET]",
"name": "bundle_target"
}
],
"queue": {
"enabled": true
},
"run_as": {
"user_name": "[USERNAME]"
},
"tasks": [
{
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_raw"
},
{
"depends_on": [
{
"task_key": "orders_raw"
}
],
"sql_task": {
"file": {
"path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql"
},
"warehouse_id": "f00dcafe"
},
"task_key": "orders_daily"
}
],
"trigger": {
"periodic": {
"interval": 1,
"unit": "DAYS"
}
},
"url": "[DATABRICKS_URL]/jobs/0?o=900800700600"
}
}
},
"sync": {
"paths": [
"."
]
},
"run_as": {
"user_name": "[USERNAME]"
},
"presets": {},
"permissions": [
{
"level": "CAN_MANAGE",
"user_name": "[USERNAME]"
}
]
}

152
acceptance/bundle/templates/default-sql/output.txt
View File

@ -31,6 +31,7 @@ Workspace:
Validation OK! Validation OK!
=== Testing validate
>>> [CLI] bundle validate -t dev -o json >>> [CLI] bundle validate -t dev -o json
>>> [CLI] bundle validate -t prod -o json >>> [CLI] bundle validate -t prod -o json
@ -141,24 +142,171 @@ Validation OK!
Exit code: 1 Exit code: 1
=== Testing deploy
>>> [CLI] bundle deploy -t dev >>> [CLI] bundle deploy -t dev
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files...
Deploying resources... Deploying resources...
Updating deployment state... Updating deployment state...
Deployment complete! Deployment complete!
Error: terraform apply: exit status 1
Error: cannot read job:
with databricks_job.my_default_sql_sql_job,
on bundle.tf.json line 81, in resource.databricks_job.my_default_sql_sql_job:
81: }
Exit code: 1
>>> [CLI] bundle deploy -t prod >>> [CLI] bundle deploy -t prod
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files...
Deploying resources... Deploying resources...
Updating deployment state... Updating deployment state...
Deployment complete! Deployment complete!
Error: terraform apply: exit status 1
=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference Error: cannot read job:
with databricks_job.my_default_sql_sql_job,
on bundle.tf.json line 80, in resource.databricks_job.my_default_sql_sql_job:
80: }
Exit code: 1
=== Testing summary, both JSON and regular output
>>> [CLI] bundle summary -t dev
Name: my_default_sql
Target: dev
Workspace:
Host: [DATABRICKS_URL]
User: [USERNAME]
Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev
Resources:
Jobs:
my_default_sql_sql_job:
Name: [dev [USERNAME]] my_default_sql_sql_job
URL: [DATABRICKS_URL]/jobs/0?o=900800700600
>>> [CLI] bundle summary -t dev -o json
>>> [CLI] bundle summary -t prod -o json
>>> [CLI] bundle summary -t prod
Name: my_default_sql
Target: prod
Workspace:
Host: [DATABRICKS_URL]
User: [USERNAME]
Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod
Resources:
Jobs:
my_default_sql_sql_job:
Name: my_default_sql_sql_job
URL: [DATABRICKS_URL]/jobs/0?o=900800700600
>>> diff -u ../../out.summary.dev.json ../../out.summary.prod.json
--- ../../out.summary.dev.json [TIMESTAMP]
+++ ../../out.summary.prod.json [TIMESTAMP]
@@ -6,9 +6,9 @@
"value": "main"
},
"schema": {
- "default": "[USERNAME]",
+ "default": "default",
"description": "The schema to use",
- "value": "[USERNAME]"
+ "value": "default"
},
"warehouse_id": {
"default": "f00dcafe",
@@ -26,11 +26,9 @@
"git": {
"bundle_root_path": "."
},
- "mode": "[TARGET]elopment",
+ "mode": "[TARGET]uction",
"deployment": {
- "lock": {
- "enabled": false
- }
+ "lock": {}
},
"uuid": "[UUID]"
},
@@ -65,15 +63,14 @@
},
"format": "MULTI_TASK",
"id": "0",
- "max_concurrent_runs": 4,
- "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job",
+ "name": "my_default_sql_sql_job",
"parameters": [
{
"default": "main",
"name": "catalog"
},
{
- "default": "[USERNAME]",
+ "default": "default",
"name": "schema"
},
{
@@ -84,8 +81,8 @@
"queue": {
"enabled": true
},
- "tags": {
- "[TARGET]": "[USERNAME]"
+ "run_as": {
+ "user_name": "[USERNAME]"
},
"tasks": [
{
@@ -113,7 +110,6 @@
}
],
"trigger": {
- "pause_status": "PAUSED",
"periodic": {
"interval": 1,
"unit": "DAYS"
@@ -128,13 +124,14 @@
"."
]
},
- "presets": {
- "name_prefix": "[[TARGET] [USERNAME]] ",
- "pipelines_[TARGET]elopment": true,
- "trigger_pause_status": "PAUSED",
- "jobs_max_concurrent_runs": 4,
- "tags": {
- "[TARGET]": "[USERNAME]"
+ "run_as": {
+ "user_name": "[USERNAME]"
+ },
+ "presets": {},
+ "permissions": [
+ {
+ "level": "CAN_MANAGE",
+ "user_name": "[USERNAME]"
}
- }
+ ]
}
Exit code: 1
=== Testing validate second time, there should be no difference
>>> [CLI] bundle validate -t dev -o json >>> [CLI] bundle validate -t dev -o json
>>> diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json >>> diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json
=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference
>>> [CLI] bundle validate -t prod -o json >>> [CLI] bundle validate -t prod -o json
>>> diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json >>> diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json

8
acceptance/bundle/templates/default-sql/script
View File

@ -31,13 +31,15 @@ rm .databricks/bundle/prod/deployment.json
printf "\n=== Testing summary, both JSON and regular output" printf "\n=== Testing summary, both JSON and regular output"
errcode trace $CLI bundle summary -t dev --debug errcode trace $CLI bundle summary -t dev
errcode trace $CLI bundle summary -t dev -o json > ../../out.summary.dev.json errcode trace $CLI bundle summary -t dev -o json | replace-target dev > ../../out.summary.dev.json
errcode trace $CLI bundle summary -t prod -o json > ../../out.summary.prod.json errcode trace $CLI bundle summary -t prod -o json | replace-target prod > ../../out.summary.prod.json
errcode trace $CLI bundle summary -t prod errcode trace $CLI bundle summary -t prod
errcode trace diff -u ../../out.summary.dev.json ../../out.summary.prod.json
printf "\n=== Testing validate second time, there should be no difference" printf "\n=== Testing validate second time, there should be no difference"