diff --git a/acceptance/bundle/templates/default-sql/out.summary.dev.json b/acceptance/bundle/templates/default-sql/out.summary.dev.json new file mode 100644 index 000000000..6f8c3fb72 --- /dev/null +++ b/acceptance/bundle/templates/default-sql/out.summary.dev.json @@ -0,0 +1,140 @@ +{ + "variables": { + "catalog": { + "default": "main", + "description": "The catalog to use", + "value": "main" + }, + "schema": { + "default": "[USERNAME]", + "description": "The schema to use", + "value": "[USERNAME]" + }, + "warehouse_id": { + "default": "f00dcafe", + "description": "The warehouse to use", + "value": "f00dcafe" + } + }, + "bundle": { + "name": "my_default_sql", + "target": "[TARGET]", + "environment": "[TARGET]", + "terraform": { + "exec_path": "[TERRAFORM]" + }, + "git": { + "bundle_root_path": "." + }, + "mode": "[TARGET]elopment", + "deployment": { + "lock": { + "enabled": false + } + }, + "uuid": "[UUID]" + }, + "include": [ + "resources/my_default_sql_sql.job.yml" + ], + "workspace": { + "host": "[DATABRICKS_URL]", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]", + "file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources", + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state" + }, + "resources": { + "jobs": { + "my_default_sql_sql_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": { + "on_failure": [ + "[USERNAME]" + ] + }, + "format": "MULTI_TASK", + "id": "0", + "max_concurrent_runs": 4, + "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + }, + { + "default": "[TARGET]", + "name": "bundle_target" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "[TARGET]": "[USERNAME]" + }, + "tasks": [ + { + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_raw" + }, + { + "depends_on": [ + { + "task_key": "orders_raw" + } + ], + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_daily" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "url": "[DATABRICKS_URL]/jobs/0?o=900800700600" + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "presets": { + "name_prefix": "[[TARGET] [USERNAME]] ", + "pipelines_[TARGET]elopment": true, + "trigger_pause_status": "PAUSED", + "jobs_max_concurrent_runs": 4, + "tags": { + "[TARGET]": "[USERNAME]" + } + } +} diff --git a/acceptance/bundle/templates/default-sql/out.summary.prod.json b/acceptance/bundle/templates/default-sql/out.summary.prod.json new file mode 100644 index 000000000..d24d3c262 --- /dev/null +++ b/acceptance/bundle/templates/default-sql/out.summary.prod.json @@ -0,0 +1,137 @@ +{ + "variables": { + "catalog": { + "default": "main", + "description": "The catalog to use", + "value": "main" + }, + "schema": { + "default": "default", + "description": "The schema to use", + "value": "default" + }, + "warehouse_id": { + "default": "f00dcafe", + "description": "The warehouse to use", + "value": "f00dcafe" + } + }, + "bundle": { + "name": "my_default_sql", + "target": "[TARGET]", + "environment": "[TARGET]", + "terraform": { + "exec_path": "[TERRAFORM]" + }, + "git": { + "bundle_root_path": "." + }, + "mode": "[TARGET]uction", + "deployment": { + "lock": {} + }, + "uuid": "[UUID]" + }, + "include": [ + "resources/my_default_sql_sql.job.yml" + ], + "workspace": { + "host": "[DATABRICKS_URL]", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]", + "file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources", + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state" + }, + "resources": { + "jobs": { + "my_default_sql_sql_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": { + "on_failure": [ + "[USERNAME]" + ] + }, + "format": "MULTI_TASK", + "id": "0", + "name": "my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { + "default": "default", + "name": "schema" + }, + { + "default": "[TARGET]", + "name": "bundle_target" + } + ], + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "tasks": [ + { + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_raw" + }, + { + "depends_on": [ + { + "task_key": "orders_raw" + } + ], + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_daily" + } + ], + "trigger": { + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "url": "[DATABRICKS_URL]/jobs/0?o=900800700600" + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "presets": {}, + "permissions": [ + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] +} diff --git a/acceptance/bundle/templates/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt index b7f8d34af..1246a0c84 100644 --- a/acceptance/bundle/templates/default-sql/output.txt +++ b/acceptance/bundle/templates/default-sql/output.txt @@ -31,6 +31,7 @@ Workspace: Validation OK! +=== Testing validate >>> [CLI] bundle validate -t dev -o json >>> [CLI] bundle validate -t prod -o json @@ -141,24 +142,171 @@ Validation OK! Exit code: 1 +=== Testing deploy >>> [CLI] bundle deploy -t dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files... Deploying resources... Updating deployment state... Deployment complete! +Error: terraform apply: exit status 1 + +Error: cannot read job: + + with databricks_job.my_default_sql_sql_job, + on bundle.tf.json line 81, in resource.databricks_job.my_default_sql_sql_job: + 81: } + + + + +Exit code: 1 >>> [CLI] bundle deploy -t prod Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files... Deploying resources... Updating deployment state... Deployment complete! +Error: terraform apply: exit status 1 -=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference +Error: cannot read job: + + with databricks_job.my_default_sql_sql_job, + on bundle.tf.json line 80, in resource.databricks_job.my_default_sql_sql_job: + 80: } + + + + +Exit code: 1 + +=== Testing summary, both JSON and regular output +>>> [CLI] bundle summary -t dev +Name: my_default_sql +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev +Resources: + Jobs: + my_default_sql_sql_job: + Name: [dev [USERNAME]] my_default_sql_sql_job + URL: [DATABRICKS_URL]/jobs/0?o=900800700600 + +>>> [CLI] bundle summary -t dev -o json + +>>> [CLI] bundle summary -t prod -o json + +>>> [CLI] bundle summary -t prod +Name: my_default_sql +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod +Resources: + Jobs: + my_default_sql_sql_job: + Name: my_default_sql_sql_job + URL: [DATABRICKS_URL]/jobs/0?o=900800700600 + +>>> diff -u ../../out.summary.dev.json ../../out.summary.prod.json +--- ../../out.summary.dev.json [TIMESTAMP] ++++ ../../out.summary.prod.json [TIMESTAMP] +@@ -6,9 +6,9 @@ + "value": "main" + }, + "schema": { +- "default": "[USERNAME]", ++ "default": "default", + "description": "The schema to use", +- "value": "[USERNAME]" ++ "value": "default" + }, + "warehouse_id": { + "default": "f00dcafe", +@@ -26,11 +26,9 @@ + "git": { + "bundle_root_path": "." + }, +- "mode": "[TARGET]elopment", ++ "mode": "[TARGET]uction", + "deployment": { +- "lock": { +- "enabled": false +- } ++ "lock": {} + }, + "uuid": "[UUID]" + }, +@@ -65,15 +63,14 @@ + }, + "format": "MULTI_TASK", + "id": "0", +- "max_concurrent_runs": 4, +- "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job", ++ "name": "my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { +- "default": "[USERNAME]", ++ "default": "default", + "name": "schema" + }, + { +@@ -84,8 +81,8 @@ + "queue": { + "enabled": true + }, +- "tags": { +- "[TARGET]": "[USERNAME]" ++ "run_as": { ++ "user_name": "[USERNAME]" + }, + "tasks": [ + { +@@ -113,7 +110,6 @@ + } + ], + "trigger": { +- "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" +@@ -128,13 +124,14 @@ + "." + ] + }, +- "presets": { +- "name_prefix": "[[TARGET] [USERNAME]] ", +- "pipelines_[TARGET]elopment": true, +- "trigger_pause_status": "PAUSED", +- "jobs_max_concurrent_runs": 4, +- "tags": { +- "[TARGET]": "[USERNAME]" ++ "run_as": { ++ "user_name": "[USERNAME]" ++ }, ++ "presets": {}, ++ "permissions": [ ++ { ++ "level": "CAN_MANAGE", ++ "user_name": "[USERNAME]" + } +- } ++ ] + } + +Exit code: 1 + +=== Testing validate second time, there should be no difference >>> [CLI] bundle validate -t dev -o json >>> diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json -=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference >>> [CLI] bundle validate -t prod -o json >>> diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index a5ec29058..f8c1c909d 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -31,13 +31,15 @@ rm .databricks/bundle/prod/deployment.json printf "\n=== Testing summary, both JSON and regular output" -errcode trace $CLI bundle summary -t dev --debug +errcode trace $CLI bundle summary -t dev -errcode trace $CLI bundle summary -t dev -o json > ../../out.summary.dev.json -errcode trace $CLI bundle summary -t prod -o json > ../../out.summary.prod.json +errcode trace $CLI bundle summary -t dev -o json | replace-target dev > ../../out.summary.dev.json +errcode trace $CLI bundle summary -t prod -o json | replace-target prod > ../../out.summary.prod.json errcode trace $CLI bundle summary -t prod +errcode trace diff -u ../../out.summary.dev.json ../../out.summary.prod.json + printf "\n=== Testing validate second time, there should be no difference"