diff --git a/acceptance/bundle/templates/default-sql/out.validate.dev.json b/acceptance/bundle/templates/default-sql/out.validate.dev.json new file mode 100644 index 000000000..237b7a5af --- /dev/null +++ b/acceptance/bundle/templates/default-sql/out.validate.dev.json @@ -0,0 +1,140 @@ +{ + "bundle": { + "deployment": { + "lock": { + "enabled": false + } + }, + "environment": "[TARGET]", + "git": { + "bundle_root_path": "." + }, + "mode": "[TARGET]elopment", + "name": "my_default_sql", + "target": "[TARGET]", + "terraform": { + "exec_path": "[TERRAFORM]" + }, + "uuid": "[UUID]" + }, + "include": [ + "resources/my_default_sql_sql.job.yml" + ], + "presets": { + "jobs_max_concurrent_runs": 4, + "name_prefix": "[[TARGET] [USERNAME]] ", + "pipelines_[TARGET]elopment": true, + "tags": { + "[TARGET]": "[USERNAME]" + }, + "trigger_pause_status": "PAUSED" + }, + "resources": { + "jobs": { + "my_default_sql_sql_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": { + "on_failure": [ + "[USERNAME]" + ] + }, + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + }, + { + "default": "[TARGET]", + "name": "bundle_target" + } + ], + "permissions": [], + "queue": { + "enabled": true + }, + "tags": { + "[TARGET]": "[USERNAME]" + }, + "tasks": [ + { + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_raw" + }, + { + "depends_on": [ + { + "task_key": "orders_raw" + } + ], + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_daily" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "targets": null, + "variables": { + "catalog": { + "default": "main", + "description": "The catalog to use", + "value": "main" + }, + "schema": { + "default": "[USERNAME]", + "description": "The schema to use", + "value": "[USERNAME]" + }, + "warehouse_id": { + "default": "f00dcafe", + "description": "The warehouse to use", + "value": "f00dcafe" + } + }, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files", + "host": "[DATABRICKS_URL]", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state" + } +} \ No newline at end of file diff --git a/acceptance/bundle/templates/default-sql/out.validate.prod.json b/acceptance/bundle/templates/default-sql/out.validate.prod.json new file mode 100644 index 000000000..eefbb297c --- /dev/null +++ b/acceptance/bundle/templates/default-sql/out.validate.prod.json @@ -0,0 +1,134 @@ +{ + "bundle": { + "environment": "[TARGET]", + "git": { + "bundle_root_path": "." + }, + "mode": "[TARGET]uction", + "name": "my_default_sql", + "target": "[TARGET]", + "terraform": { + "exec_path": "[TERRAFORM]" + }, + "uuid": "[UUID]" + }, + "include": [ + "resources/my_default_sql_sql.job.yml" + ], + "permissions": [ + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ], + "resources": { + "jobs": { + "my_default_sql_sql_job": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": { + "on_failure": [ + "[USERNAME]" + ] + }, + "format": "MULTI_TASK", + "name": "my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { + "default": "default", + "name": "schema" + }, + { + "default": "[TARGET]", + "name": "bundle_target" + } + ], + "permissions": [], + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "tags": {}, + "tasks": [ + { + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_raw.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_raw" + }, + { + "depends_on": [ + { + "task_key": "orders_raw" + } + ], + "sql_task": { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files/src/orders_daily.sql" + }, + "warehouse_id": "f00dcafe" + }, + "task_key": "orders_daily" + } + ], + "trigger": { + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + } + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "sync": { + "paths": [ + "." + ] + }, + "targets": null, + "variables": { + "catalog": { + "default": "main", + "description": "The catalog to use", + "value": "main" + }, + "schema": { + "default": "default", + "description": "The schema to use", + "value": "default" + }, + "warehouse_id": { + "default": "f00dcafe", + "description": "The warehouse to use", + "value": "f00dcafe" + } + }, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/artifacts", + "current_user": { + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/files", + "host": "[DATABRICKS_URL]", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/[TARGET]/state" + } +} \ No newline at end of file diff --git a/acceptance/bundle/templates/default-sql/output.txt b/acceptance/bundle/templates/default-sql/output.txt index 06eff962b..a28bd0f86 100644 --- a/acceptance/bundle/templates/default-sql/output.txt +++ b/acceptance/bundle/templates/default-sql/output.txt @@ -30,3 +30,143 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod Validation OK! + +>>> [CLI] bundle validate -t dev -o json + +>>> [CLI] bundle validate -t prod -o json + +>>> diff -u ../../out.validate.dev.json ../../out.validate.prod.json +--- ../../out.validate.dev.json [TIMESTAMP] ++++ ../../out.validate.prod.json [TIMESTAMP] +@@ -1,15 +1,10 @@ + { + "bundle": { +- "deployment": { +- "lock": { +- "enabled": false +- } +- }, + "environment": "[TARGET]", + "git": { + "bundle_root_path": "." + }, +- "mode": "[TARGET]elopment", ++ "mode": "[TARGET]uction", + "name": "my_default_sql", + "target": "[TARGET]", + "terraform": { +\ No newline at end of file +@@ -20,15 +15,12 @@ + "include": [ + "resources/my_default_sql_sql.job.yml" + ], +- "presets": { +- "jobs_max_concurrent_runs": 4, +- "name_prefix": "[[TARGET] [USERNAME]] ", +- "pipelines_[TARGET]elopment": true, +- "tags": { +- "[TARGET]": "[USERNAME]" +- }, +- "trigger_pause_status": "PAUSED" +- }, ++ "permissions": [ ++ { ++ "level": "CAN_MANAGE", ++ "user_name": "[USERNAME]" ++ } ++ ], + "resources": { + "jobs": { + "my_default_sql_sql_job": { +\ No newline at end of file +@@ -43,15 +35,14 @@ + ] + }, + "format": "MULTI_TASK", +- "max_concurrent_runs": 4, +- "name": "[[TARGET] [USERNAME]] my_default_sql_sql_job", ++ "name": "my_default_sql_sql_job", + "parameters": [ + { + "default": "main", + "name": "catalog" + }, + { +- "default": "[USERNAME]", ++ "default": "default", + "name": "schema" + }, + { +\ No newline at end of file +@@ -63,9 +54,10 @@ + "queue": { + "enabled": true + }, +- "tags": { +- "[TARGET]": "[USERNAME]" ++ "run_as": { ++ "user_name": "[USERNAME]" + }, ++ "tags": {}, + "tasks": [ + { + "sql_task": { +\ No newline at end of file +@@ -92,7 +84,6 @@ + } + ], + "trigger": { +- "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" +\ No newline at end of file +@@ -101,6 +92,9 @@ + } + } + }, ++ "run_as": { ++ "user_name": "[USERNAME]" ++ }, + "sync": { + "paths": [ + "." +\ No newline at end of file +@@ -114,9 +108,9 @@ + "value": "main" + }, + "schema": { +- "default": "[USERNAME]", ++ "default": "default", + "description": "The schema to use", +- "value": "[USERNAME]" ++ "value": "default" + }, + "warehouse_id": { + "default": "f00dcafe", +\ No newline at end of file + +Exit code: 1 + +>>> [CLI] bundle deploy -t dev +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle deploy -t prod +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files... +Error: + + +Exit code: 1 + +=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference +>>> [CLI] bundle validate -t dev -o json + +>>> diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json + +=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference +>>> [CLI] bundle validate -t prod -o json + +>>> diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/dev/sync-snapshots/snapshot.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/dev/sync-snapshots/snapshot.json new file mode 100644 index 000000000..05ecebd33 --- /dev/null +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/dev/sync-snapshots/snapshot.json @@ -0,0 +1,41 @@ +{ + "version": "v1", + "host": "[DATABRICKS_URL]", + "remote_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/dev/files", + "last_modified_times": { + ".gitignore": "[TIMESTAMP]", + ".vscode/extensions.json": "[TIMESTAMP]", + ".vscode/settings.json": "[TIMESTAMP]", + "README.md": "[TIMESTAMP]", + "databricks.yml": "[TIMESTAMP]", + "resources/my_default_sql_sql.job.yml": "[TIMESTAMP]", + "scratch/README.md": "[TIMESTAMP]", + "scratch/exploration.ipynb": "[TIMESTAMP]", + "src/orders_daily.sql": "[TIMESTAMP]", + "src/orders_raw.sql": "[TIMESTAMP]" + }, + "local_to_remote_names": { + ".gitignore": ".gitignore", + ".vscode/extensions.json": ".vscode/extensions.json", + ".vscode/settings.json": ".vscode/settings.json", + "README.md": "README.md", + "databricks.yml": "databricks.yml", + "resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml", + "scratch/README.md": "scratch/README.md", + "scratch/exploration.ipynb": "scratch/exploration", + "src/orders_daily.sql": "src/orders_daily.sql", + "src/orders_raw.sql": "src/orders_raw.sql" + }, + "remote_to_local_names": { + ".gitignore": ".gitignore", + ".vscode/extensions.json": ".vscode/extensions.json", + ".vscode/settings.json": ".vscode/settings.json", + "README.md": "README.md", + "databricks.yml": "databricks.yml", + "resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml", + "scratch/README.md": "scratch/README.md", + "scratch/exploration": "scratch/exploration.ipynb", + "src/orders_daily.sql": "src/orders_daily.sql", + "src/orders_raw.sql": "src/orders_raw.sql" + } +} \ No newline at end of file diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/prod/sync-snapshots/snapshot.json b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/prod/sync-snapshots/snapshot.json new file mode 100644 index 000000000..a44c54d3b --- /dev/null +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/bundle/prod/sync-snapshots/snapshot.json @@ -0,0 +1,41 @@ +{ + "version": "v1", + "host": "[DATABRICKS_URL]", + "remote_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_sql/prod/files", + "last_modified_times": { + ".gitignore": "[TIMESTAMP]", + ".vscode/extensions.json": "[TIMESTAMP]", + ".vscode/settings.json": "[TIMESTAMP]", + "README.md": "[TIMESTAMP]", + "databricks.yml": "[TIMESTAMP]", + "resources/my_default_sql_sql.job.yml": "[TIMESTAMP]", + "scratch/README.md": "[TIMESTAMP]", + "scratch/exploration.ipynb": "[TIMESTAMP]", + "src/orders_daily.sql": "[TIMESTAMP]", + "src/orders_raw.sql": "[TIMESTAMP]" + }, + "local_to_remote_names": { + ".gitignore": ".gitignore", + ".vscode/extensions.json": ".vscode/extensions.json", + ".vscode/settings.json": ".vscode/settings.json", + "README.md": "README.md", + "databricks.yml": "databricks.yml", + "resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml", + "scratch/README.md": "scratch/README.md", + "scratch/exploration.ipynb": "scratch/exploration", + "src/orders_daily.sql": "src/orders_daily.sql", + "src/orders_raw.sql": "src/orders_raw.sql" + }, + "remote_to_local_names": { + ".gitignore": ".gitignore", + ".vscode/extensions.json": ".vscode/extensions.json", + ".vscode/settings.json": ".vscode/settings.json", + "README.md": "README.md", + "databricks.yml": "databricks.yml", + "resources/my_default_sql_sql.job.yml": "resources/my_default_sql_sql.job.yml", + "scratch/README.md": "scratch/README.md", + "scratch/exploration": "scratch/exploration.ipynb", + "src/orders_daily.sql": "src/orders_daily.sql", + "src/orders_raw.sql": "src/orders_raw.sql" + } +} \ No newline at end of file diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index 7ea0d863c..b0e6de388 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -4,5 +4,43 @@ cd output/my_default_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod +replace-target() { + sed "s/$1/[TARGET]/" +} + +trace $CLI bundle validate -t dev -o json | replace-target dev > ../../out.validate.dev.json +trace $CLI bundle validate -t prod -o json | replace-target prod > ../../out.validate.prod.json +errcode trace diff -u ../../out.validate.dev.json ../../out.validate.prod.json + +errcode trace $CLI bundle deploy -t dev +mv .databricks/bundle/dev/sync-snapshots/*.json .databricks/bundle/dev/sync-snapshots/snapshot.json +rm -r .databricks/bundle/dev/terraform +# TODO: This would be nice to include but .files currently has randomized order +#jq < .databricks/bundle/dev/deployment.json > .databricks/bundle/dev/deployment.jq.json +rm .databricks/bundle/dev/deployment.json + + +errcode trace $CLI bundle deploy -t prod +mv .databricks/bundle/prod/sync-snapshots/*.json .databricks/bundle/prod/sync-snapshots/snapshot.json +rm -r .databricks/bundle/prod/terraform +#jq < .databricks/bundle/prod/deployment.json > .databricks/bundle/prod/deployment.jq.json +rm .databricks/bundle/prod/deployment.json + +#errcode trace $CLI bundle summary -t dev +#errcode trace $CLI bundle summary -t prod + +#errcode trace $CLI bundle summary -t dev -o json > ../../out.summary.dev.json +#errcode trace $CLI bundle summary -t prod -o json > ../../out.summary.prod.json + +printf "\n=== Running validate -t dev -o json second time and comparing output to first time - there should be no difference" +errcode trace $CLI bundle validate -t dev -o json | replace-target dev > ../../out.validate.2nd.dev.json +errcode trace diff -u ../../out.validate.dev.json ../../out.validate.2nd.dev.json # should be the same +rm ../../out.validate.2nd.dev.json + +printf "\n=== Running validate -t prod -o json second time and comparing output to first time - there should be no difference" +errcode trace $CLI bundle validate -t prod -o json | replace-target prod > ../../out.validate.2nd.prod.json +errcode trace diff -u ../../out.validate.prod.json ../../out.validate.2nd.prod.json # should be the same +rm ../../out.validate.2nd.prod.json + # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 90539263d..af1833bfa 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,2 +1,6 @@ # At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. LocalOnly = true + +[[Repls]] +Old = '\d\d\d\d-\d\d-\d\d(T| )\d\d:\d\d:\d\d(\.\d+)?(\+\d\d:\d\d)?Z?' +New = '[TIMESTAMP]'