From 82e35530b02d1b2246a012ef3a1720c3791acb16 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 14 Jan 2025 19:23:34 +0100 Subject: [PATCH] Add acceptance tests for builtin templates (#2135) ## Changes To accommodate: * Add the server URL to the set of output replacements * Include a call to the permissions API to the dummy server * Run the main script in a subshell to isolate working directory changes --- acceptance/acceptance_test.go | 7 +++- acceptance/bundle/init/dbt-sql/input.json | 6 ++++ acceptance/bundle/init/dbt-sql/output.txt | 32 +++++++++++++++++++ acceptance/bundle/init/dbt-sql/script | 5 +++ acceptance/bundle/init/dbt-sql/script.cleanup | 1 + .../bundle/init/default-python/input.json | 6 ++++ .../bundle/init/default-python/output.txt | 30 +++++++++++++++++ acceptance/bundle/init/default-python/script | 5 +++ .../bundle/init/default-python/script.cleanup | 1 + acceptance/bundle/init/default-sql/input.json | 6 ++++ acceptance/bundle/init/default-sql/output.txt | 32 +++++++++++++++++++ acceptance/bundle/init/default-sql/script | 5 +++ .../bundle/init/default-sql/script.cleanup | 1 + acceptance/server_test.go | 30 ++++++++++++++--- 14 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/init/dbt-sql/input.json create mode 100644 acceptance/bundle/init/dbt-sql/output.txt create mode 100644 acceptance/bundle/init/dbt-sql/script create mode 100644 acceptance/bundle/init/dbt-sql/script.cleanup create mode 100644 acceptance/bundle/init/default-python/input.json create mode 100644 acceptance/bundle/init/default-python/output.txt create mode 100644 acceptance/bundle/init/default-python/script create mode 100644 acceptance/bundle/init/default-python/script.cleanup create mode 100644 acceptance/bundle/init/default-sql/input.json create mode 100644 acceptance/bundle/init/default-sql/output.txt create mode 100644 acceptance/bundle/init/default-sql/script create mode 100644 acceptance/bundle/init/default-sql/script.cleanup diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 4b5146503..62c56a325 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -66,7 +66,7 @@ func TestAccept(t *testing.T) { server := StartServer(t) AddHandlers(server) // Redirect API access to local server: - t.Setenv("DATABRICKS_HOST", fmt.Sprintf("http://127.0.0.1:%d", server.Port)) + t.Setenv("DATABRICKS_HOST", server.URL) t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() @@ -217,6 +217,11 @@ func doComparison(t *testing.T, pathExpected, pathNew, valueNew string) { // Note, cleanups are not executed if main script fails; that's not a huge issue, since it runs it temp dir. func readMergedScriptContents(t *testing.T, dir string) string { scriptContents := testutil.ReadFile(t, filepath.Join(dir, EntryPointScript)) + + // Wrap script contents in a subshell such that changing the working + // directory only affects the main script and not cleanup. + scriptContents = "(\n" + scriptContents + ")\n" + prepares := []string{} cleanups := []string{} diff --git a/acceptance/bundle/init/dbt-sql/input.json b/acceptance/bundle/init/dbt-sql/input.json new file mode 100644 index 000000000..201ac9667 --- /dev/null +++ b/acceptance/bundle/init/dbt-sql/input.json @@ -0,0 +1,6 @@ +{ + "project_name": "my_dbt_sql", + "http_path": "/sql/2.0/warehouses/f00dcafe", + "default_catalog": "main", + "personal_schemas": "yes, use a schema based on the current user name during development" +} diff --git a/acceptance/bundle/init/dbt-sql/output.txt b/acceptance/bundle/init/dbt-sql/output.txt new file mode 100644 index 000000000..384013d14 --- /dev/null +++ b/acceptance/bundle/init/dbt-sql/output.txt @@ -0,0 +1,32 @@ + +>>> $CLI bundle init dbt-sql --config-file ./input.json + +Welcome to the dbt template for Databricks Asset Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: http://$DATABRICKS_HOST + +📊 Your new project has been created in the 'my_dbt_sql' directory! +If you already have dbt installed, just type 'cd my_dbt_sql; dbt init' to get started. +Refer to the README.md file for full "getting started" guide and production setup instructions. + + +>>> $CLI bundle validate -t dev +Name: my_dbt_sql +Target: dev +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/dev + +Validation OK! + +>>> $CLI bundle validate -t prod +Name: my_dbt_sql +Target: prod +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_dbt_sql/prod + +Validation OK! diff --git a/acceptance/bundle/init/dbt-sql/script b/acceptance/bundle/init/dbt-sql/script new file mode 100644 index 000000000..5f6efda99 --- /dev/null +++ b/acceptance/bundle/init/dbt-sql/script @@ -0,0 +1,5 @@ +trace $CLI bundle init dbt-sql --config-file ./input.json + +cd my_dbt_sql +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod diff --git a/acceptance/bundle/init/dbt-sql/script.cleanup b/acceptance/bundle/init/dbt-sql/script.cleanup new file mode 100644 index 000000000..b717c8d4a --- /dev/null +++ b/acceptance/bundle/init/dbt-sql/script.cleanup @@ -0,0 +1 @@ +rm -fr my_dbt_sql diff --git a/acceptance/bundle/init/default-python/input.json b/acceptance/bundle/init/default-python/input.json new file mode 100644 index 000000000..3e1d79c68 --- /dev/null +++ b/acceptance/bundle/init/default-python/input.json @@ -0,0 +1,6 @@ +{ + "project_name": "my_default_python", + "include_notebook": "yes", + "include_dlt": "yes", + "include_python": "yes" +} diff --git a/acceptance/bundle/init/default-python/output.txt b/acceptance/bundle/init/default-python/output.txt new file mode 100644 index 000000000..c389ef0ad --- /dev/null +++ b/acceptance/bundle/init/default-python/output.txt @@ -0,0 +1,30 @@ + +>>> $CLI bundle init default-python --config-file ./input.json + +Welcome to the default Python template for Databricks Asset Bundles! +Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): http://$DATABRICKS_HOST + +✨ Your new project has been created in the 'my_default_python' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> $CLI bundle validate -t dev +Name: my_default_python +Target: dev +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/dev + +Validation OK! + +>>> $CLI bundle validate -t prod +Name: my_default_python +Target: prod +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_python/prod + +Validation OK! diff --git a/acceptance/bundle/init/default-python/script b/acceptance/bundle/init/default-python/script new file mode 100644 index 000000000..84834ce3f --- /dev/null +++ b/acceptance/bundle/init/default-python/script @@ -0,0 +1,5 @@ +trace $CLI bundle init default-python --config-file ./input.json + +cd my_default_python +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod diff --git a/acceptance/bundle/init/default-python/script.cleanup b/acceptance/bundle/init/default-python/script.cleanup new file mode 100644 index 000000000..4fd2e4aa7 --- /dev/null +++ b/acceptance/bundle/init/default-python/script.cleanup @@ -0,0 +1 @@ +rm -fr my_default_python diff --git a/acceptance/bundle/init/default-sql/input.json b/acceptance/bundle/init/default-sql/input.json new file mode 100644 index 000000000..c728d25de --- /dev/null +++ b/acceptance/bundle/init/default-sql/input.json @@ -0,0 +1,6 @@ +{ + "project_name": "my_default_sql", + "http_path": "/sql/2.0/warehouses/f00dcafe", + "default_catalog": "main", + "personal_schemas": "yes, automatically use a schema based on the current user name during development" +} diff --git a/acceptance/bundle/init/default-sql/output.txt b/acceptance/bundle/init/default-sql/output.txt new file mode 100644 index 000000000..3605206dc --- /dev/null +++ b/acceptance/bundle/init/default-sql/output.txt @@ -0,0 +1,32 @@ + +>>> $CLI bundle init default-sql --config-file ./input.json + +Welcome to the default SQL template for Databricks Asset Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: http://$DATABRICKS_HOST + +✨ Your new project has been created in the 'my_default_sql' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> $CLI bundle validate -t dev +Name: my_default_sql +Target: dev +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/dev + +Validation OK! + +>>> $CLI bundle validate -t prod +Name: my_default_sql +Target: prod +Workspace: + Host: http://$DATABRICKS_HOST + User: $USERNAME + Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod + +Validation OK! diff --git a/acceptance/bundle/init/default-sql/script b/acceptance/bundle/init/default-sql/script new file mode 100644 index 000000000..6f7332a39 --- /dev/null +++ b/acceptance/bundle/init/default-sql/script @@ -0,0 +1,5 @@ +trace $CLI bundle init default-sql --config-file ./input.json + +cd my_default_sql +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod diff --git a/acceptance/bundle/init/default-sql/script.cleanup b/acceptance/bundle/init/default-sql/script.cleanup new file mode 100644 index 000000000..c46d9ad60 --- /dev/null +++ b/acceptance/bundle/init/default-sql/script.cleanup @@ -0,0 +1 @@ +rm -fr my_default_sql diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 7b21e198f..0d10fbea1 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -2,11 +2,11 @@ package acceptance_test import ( "encoding/json" - "net" "net/http" "net/http/httptest" "testing" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -14,8 +14,7 @@ import ( type TestServer struct { *httptest.Server - Mux *http.ServeMux - Port int + Mux *http.ServeMux } type HandlerFunc func(r *http.Request) (any, error) @@ -23,12 +22,10 @@ type HandlerFunc func(r *http.Request) (any, error) func NewTestServer() *TestServer { mux := http.NewServeMux() server := httptest.NewServer(mux) - port := server.Listener.Addr().(*net.TCPAddr).Port return &TestServer{ Server: server, Mux: mux, - Port: port, } } @@ -126,4 +123,27 @@ func AddHandlers(server *TestServer) { ResourceId: "1001", }, nil }) + + server.Handle("/api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) { + return catalog.MetastoreAssignment{ + DefaultCatalogName: "main", + }, nil + }) + + server.Handle("/api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) { + return workspace.WorkspaceObjectPermissions{ + ObjectId: "1001", + ObjectType: "DIRECTORY", + AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "tester@databricks.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + { + PermissionLevel: "CAN_MANAGE", + }, + }, + }, + }, + }, nil + }) }