From ce965b22b2dc194ce90d101bf52f892d18e3773d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 29 Jan 2025 16:55:53 +0100 Subject: [PATCH 1/8] [Release] Release v0.240.0 (#2264) Bundles: * Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)). * Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)). * Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)). * libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)). API Changes: * Added `databricks access-control` command group. * Added `databricks serving-endpoints http-request` command. * Changed `databricks serving-endpoints create` command with new required argument order. * Changed `databricks serving-endpoints get-open-api` command return type to become non-empty. * Changed `databricks recipients update` command return type to become non-empty. OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22) Dependency updates: * Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)). * Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)). --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 255bfb0a8..449c30288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Version changelog +## [Release] Release v0.240.0 + +Bundles: + * Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)). + * Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)). + * Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)). + * libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)). + +API Changes: + * Added `databricks access-control` command group. + * Added `databricks serving-endpoints http-request` command. + * Changed `databricks serving-endpoints create` command with new required argument order. + * Changed `databricks serving-endpoints get-open-api` command return type to become non-empty. + * Changed `databricks recipients update` command return type to become non-empty. + +OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22) +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)). + * Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)). + ## [Release] Release v0.239.1 CLI: From 55c03cc119c2792b084928f12a15b93b3a6ed14a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 29 Jan 2025 21:24:33 +0530 Subject: [PATCH 2/8] Always close test HTTP server during cleanup (#2261) ## Changes This PR registers the `server.Close()` function to be run during test cleanup in the server initialization function. This ensures that all test servers are closed as soon as the test they are scoped to finish. Motivated by https://github.com/databricks/cli/pull/2255/files where a regression was introduced where we did not close the test server. ## Tests N/A --- acceptance/cmd_server_test.go | 3 ++- acceptance/server_test.go | 9 --------- libs/testserver/server.go | 5 +---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 3f5a6356e..9af63d0db 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -13,7 +13,8 @@ import ( ) func StartCmdServer(t *testing.T) *testserver.Server { - server := StartServer(t) + server := testserver.New(t) + server.Handle("/", func(r *http.Request) (any, error) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 66de5dcbf..98e351739 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -2,7 +2,6 @@ package acceptance_test import ( "net/http" - "testing" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -11,14 +10,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) -func StartServer(t *testing.T) *testserver.Server { - server := testserver.New(t) - t.Cleanup(func() { - server.Close() - }) - return server -} - func AddHandlers(server *testserver.Server) { server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) { return compute.ListPoliciesResponse{ diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 10269af8f..9ebfe3ba0 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -18,6 +18,7 @@ type Server struct { func New(t testutil.TestingT) *Server { mux := http.NewServeMux() server := httptest.NewServer(mux) + t.Cleanup(server.Close) return &Server{ Server: server, @@ -28,10 +29,6 @@ func New(t testutil.TestingT) *Server { type HandlerFunc func(req *http.Request) (resp any, err error) -func (s *Server) Close() { - s.Server.Close() -} - func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { resp, err := handler(r) From 58ef34f320f4865d2a62d08b80798fdbdcf7294d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 29 Jan 2025 18:35:03 +0100 Subject: [PATCH 3/8] acc: Include "id" into /api/2.0/preview/scim/v2/Me response (#2266) This is something terraform provider expects. Related to https://github.com/databricks/cli/pull/2242 --- acceptance/bundle/variables/git-branch/output.txt | 2 ++ acceptance/bundle/variables/prepend-workspace-var/output.txt | 1 + acceptance/bundle/variables/resolve-builtin/output.txt | 1 + .../bundle/variables/resolve-vars-in-root-path/output.txt | 1 + acceptance/server_test.go | 1 + 5 files changed, 6 insertions(+) diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index d6d824394..fb3ab805a 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -29,6 +29,7 @@ "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, @@ -78,6 +79,7 @@ Validation OK! "workspace": { "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index 706d134ff..fcaa25b4a 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -54,6 +54,7 @@ "workspace": { "artifact_path": "/Users/$USERNAME/path/to/root/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/resolve-builtin/output.txt b/acceptance/bundle/variables/resolve-builtin/output.txt index f060c472e..0c1678f84 100644 --- a/acceptance/bundle/variables/resolve-builtin/output.txt +++ b/acceptance/bundle/variables/resolve-builtin/output.txt @@ -1,6 +1,7 @@ { "artifact_path": "TestResolveVariableReferences/bar/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt index c56fbe415..51eb40c91 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/output.txt @@ -1,6 +1,7 @@ { "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts", "current_user": { + "id": "$USER.Id", "short_name": "$USERNAME", "userName": "$USERNAME" }, diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 98e351739..4957a7668 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -54,6 +54,7 @@ func AddHandlers(server *testserver.Server) { server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) { return iam.User{ + Id: "1000012345", UserName: "tester@databricks.com", }, nil }) From a03ea730112c7f51c8fc82d55ab8d11908ef8e9f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 30 Jan 2025 10:52:41 +0100 Subject: [PATCH 4/8] Add ruff.toml with increased line-length (#2268) The default is 88 which reformats too much. This has no effect on templates but affects Python script in this PR https://github.com/databricks/cli/pull/2267 For context, we do not set any line length for golang and have 177 .go files with max line length 150 or more. --- ruff.toml | 1 + 1 file changed, 1 insertion(+) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..802a3ca67 --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 150 From f1efbd7d9fb19c4f9457999e5ab8740293f34502 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 30 Jan 2025 11:38:54 +0100 Subject: [PATCH 5/8] acc: add -norepl flag that disables replacements (for debugging) (#2269) --- acceptance/acceptance_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 91ad09e9e..e7104a1c1 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -25,7 +25,10 @@ import ( "github.com/stretchr/testify/require" ) -var KeepTmp bool +var ( + KeepTmp bool + NoRepl bool +) // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. @@ -40,6 +43,7 @@ var InprocessMode bool func init() { flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)") flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run") + flag.BoolVar(&NoRepl, "norepl", false, "Do not apply any replacements (for debugging)") } const ( @@ -272,7 +276,9 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // Apply replacements to the new value only. // The reference value is stored after applying replacements. - valueNew = repls.Replace(valueNew) + if !NoRepl { + valueNew = repls.Replace(valueNew) + } // The test did not produce an expected output file. if okRef && !okNew { From 3c6eacb05b6980c59324b1da4ac091f22cb5733d Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:13:07 +0530 Subject: [PATCH 6/8] Add feature to mock server APIs in acceptance tests (#2226) ## Changes This PR allows us to define custom server stubs in a `test.toml` file. Note: A followup PR will add functionality to do assertions on the API request itself. ## Tests New acceptance test. --- acceptance/acceptance_test.go | 25 +++++++++++++++++---- acceptance/config_test.go | 23 +++++++++++++++++++ acceptance/workspace/jobs/create/output.txt | 5 +++++ acceptance/workspace/jobs/create/script | 1 + acceptance/workspace/jobs/create/test.toml | 7 ++++++ 5 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 acceptance/workspace/jobs/create/output.txt create mode 100644 acceptance/workspace/jobs/create/script create mode 100644 acceptance/workspace/jobs/create/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e7104a1c1..60f7945df 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -112,10 +113,10 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cloudEnv := os.Getenv("CLOUD_ENV") if cloudEnv == "" { - server := testserver.New(t) - AddHandlers(server) + defaultServer := testserver.New(t) + AddHandlers(defaultServer) // Redirect API access to local server: - t.Setenv("DATABRICKS_HOST", server.URL) + t.Setenv("DATABRICKS_HOST", defaultServer.URL) t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() @@ -214,6 +215,22 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont args := []string{"bash", "-euo", "pipefail", EntryPointScript} cmd := exec.Command(args[0], args[1:]...) + cmd.Env = os.Environ() + + // Start a new server with a custom configuration if the acceptance test + // specifies a custom server stubs. + if len(config.Server) > 0 { + server := testserver.New(t) + + for _, stub := range config.Server { + require.NotEmpty(t, stub.Pattern) + server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) { + return stub.Response.Body, nil + }) + } + cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL) + } + if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one // results in sporadic failures like this one (only if tests are running in parallel): @@ -221,7 +238,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--")) err := os.MkdirAll(coverDir, os.ModePerm) require.NoError(t, err) - cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir) + cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } // Write combined output to a file diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 41866c4a7..f340f0367 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -29,6 +29,29 @@ type TestConfig struct { // List of additional replacements to apply on this test. // Old is a regexp, New is a replacement expression. Repls []testdiff.Replacement + + // List of server stubs to load. Example configuration: + // + // [[Server]] + // Pattern = "POST /api/2.1/jobs/create" + // Response.Body = ''' + // { + // "job_id": 1111 + // } + // ''' + Server []ServerStub +} + +type ServerStub struct { + // The HTTP method and path to match. Examples: + // 1. /api/2.0/clusters/list (matches all methods) + // 2. GET /api/2.0/clusters/list + Pattern string + + // The response body to return. + Response struct { + Body string + } } // FindConfig finds the closest config file. diff --git a/acceptance/workspace/jobs/create/output.txt b/acceptance/workspace/jobs/create/output.txt new file mode 100644 index 000000000..a9487fe5b --- /dev/null +++ b/acceptance/workspace/jobs/create/output.txt @@ -0,0 +1,5 @@ + +>>> $CLI jobs create --json {"name":"abc"} +{ + "job_id":1111 +} diff --git a/acceptance/workspace/jobs/create/script b/acceptance/workspace/jobs/create/script new file mode 100644 index 000000000..9ff7b5b87 --- /dev/null +++ b/acceptance/workspace/jobs/create/script @@ -0,0 +1 @@ +trace $CLI jobs create --json '{"name":"abc"}' diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml new file mode 100644 index 000000000..94e5eee13 --- /dev/null +++ b/acceptance/workspace/jobs/create/test.toml @@ -0,0 +1,7 @@ +[[Server]] +Pattern = "POST /api/2.1/jobs/create" +Response.Body = ''' +{ + "job_id": 1111 +} +''' From 787dbe909912ab5c4894cb21218cb39d6626e95a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:01:23 +0530 Subject: [PATCH 7/8] Add request body assertions to acceptance tests (#2263) ## Changes With this PR, any acceptance tests that define custom server stubs in `test.toml` will automatically record all HTTP requests made and assert on them. Builds on top of https://github.com/databricks/cli/pull/2226 ## Tests Modifying existing acceptance test. --- acceptance/acceptance_test.go | 37 ++++++++++++++++++- acceptance/config_test.go | 4 ++ .../workspace/jobs/create/out.requests.txt | 1 + acceptance/workspace/jobs/create/test.toml | 2 + libs/testserver/server.go | 25 +++++++++++++ 5 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 acceptance/workspace/jobs/create/out.requests.txt diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 60f7945df..bb1d0f44f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -2,6 +2,7 @@ package acceptance_test import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -219,8 +220,21 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // Start a new server with a custom configuration if the acceptance test // specifies a custom server stubs. - if len(config.Server) > 0 { - server := testserver.New(t) + var server *testserver.Server + + // Start a new server for this test if either: + // 1. A custom server spec is defined in the test configuration. + // 2. The test is configured to record requests and assert on them. We need + // a duplicate of the default server to record requests because the default + // server otherwise is a shared resource. + if len(config.Server) > 0 || config.RecordRequests { + server = testserver.New(t) + server.RecordRequests = config.RecordRequests + + // If no custom server stubs are defined, add the default handlers. + if len(config.Server) == 0 { + AddHandlers(server) + } for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) @@ -249,6 +263,25 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Dir = tmpDir err = cmd.Run() + // Write the requests made to the server to a output file if the test is + // configured to record requests. + if config.RecordRequests { + f, err := os.OpenFile(filepath.Join(tmpDir, "out.requests.txt"), os.O_CREATE|os.O_WRONLY, 0o644) + require.NoError(t, err) + + for _, req := range server.Requests { + reqJson, err := json.Marshal(req) + require.NoError(t, err) + + line := fmt.Sprintf("%s\n", reqJson) + _, err = f.WriteString(line) + require.NoError(t, err) + } + + err = f.Close() + require.NoError(t, err) + } + // Include exit code in output (if non-zero) formatOutput(out, err) require.NoError(t, out.Close()) diff --git a/acceptance/config_test.go b/acceptance/config_test.go index f340f0367..beceb6a08 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -40,6 +40,10 @@ type TestConfig struct { // } // ''' Server []ServerStub + + // Record the requests made to the server and write them as output to + // out.requests.txt + RecordRequests bool } type ServerStub struct { diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt new file mode 100644 index 000000000..b22876b70 --- /dev/null +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -0,0 +1 @@ +{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index 94e5eee13..1bf36547b 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,3 +1,5 @@ +RecordRequests = true + [[Server]] Pattern = "POST /api/2.1/jobs/create" Response.Body = ''' diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 9ebfe3ba0..2e8dbdfda 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -2,9 +2,12 @@ package testserver import ( "encoding/json" + "io" "net/http" "net/http/httptest" + "github.com/stretchr/testify/assert" + "github.com/databricks/cli/internal/testutil" ) @@ -13,6 +16,16 @@ type Server struct { Mux *http.ServeMux t testutil.TestingT + + RecordRequests bool + + Requests []Request +} + +type Request struct { + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -37,6 +50,18 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { return } + if s.RecordRequests { + body, err := io.ReadAll(r.Body) + assert.NoError(s.t, err) + + s.Requests = append(s.Requests, Request{ + Method: r.Method, + Path: r.URL.Path, + Body: json.RawMessage(body), + }) + + } + w.Header().Set("Content-Type", "application/json") var respBytes []byte From e5730bf57ecb39049047ba394734831589a81bbb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 31 Jan 2025 14:53:13 +0100 Subject: [PATCH 8/8] Use real terraform in acceptance tests (#2267) ## Changes - Add a script install_terraform.py that downloads terraform and provider and generates a config to use, inspired by https://gist.github.com/pietern/1cb6b6f3e0a452328e13cdc75031105e - Make acceptance tests run this script once before running the tests and set the required env vars to make cli use this terraform installation. - Use OS-specific directory for things that are build by acceptance test runner (CLI and terraform). This enables acceptance tests against cloud #2242 and local test for bundle deploy #2254. ## Tests - Add an acceptance test for standalone terraform. This is useful to debug terraform with TF_LOG=DEBUG to see that it uses local provider. - Other acceptance tests are updated with regard to terraform exec path. - The overall time for tests locally is unchanged (if terraform is already fetched). --- acceptance/.gitignore | 1 + acceptance/acceptance_test.go | 61 ++++++--- acceptance/build/.gitignore | 1 - .../bundle/variables/git-branch/output.txt | 4 +- .../prepend-workspace-var/output.txt | 2 +- acceptance/install_terraform.py | 122 ++++++++++++++++++ acceptance/terraform/main.tf | 25 ++++ acceptance/terraform/output.txt | 51 ++++++++ acceptance/terraform/script | 14 ++ 9 files changed, 256 insertions(+), 25 deletions(-) create mode 100644 acceptance/.gitignore delete mode 100644 acceptance/build/.gitignore create mode 100755 acceptance/install_terraform.py create mode 100644 acceptance/terraform/main.tf create mode 100644 acceptance/terraform/output.txt create mode 100644 acceptance/terraform/script diff --git a/acceptance/.gitignore b/acceptance/.gitignore new file mode 100644 index 000000000..378eac25d --- /dev/null +++ b/acceptance/.gitignore @@ -0,0 +1 @@ +build diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index bb1d0f44f..5f0030eec 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -77,6 +77,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { cwd, err := os.Getwd() require.NoError(t, err) + buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + + // Download terraform and provider and create config; this also creates build directory. + RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".") + coverDir := os.Getenv("CLI_GOCOVERDIR") if coverDir != "" { @@ -93,7 +98,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { t.Setenv("CMD_SERVER_URL", cmdServer.URL) execPath = filepath.Join(cwd, "bin", "callserver.py") } else { - execPath = BuildCLI(t, cwd, coverDir) + execPath = BuildCLI(t, buildDir, coverDir) } t.Setenv("CLI", execPath) @@ -123,11 +128,24 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { homeDir := t.TempDir() // Do not read user's ~/.databrickscfg t.Setenv(env.HomeEnvVar(), homeDir) - - // Prevent CLI from downloading terraform in each test: - t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir) } + terraformrcPath := filepath.Join(buildDir, ".terraformrc") + t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath) + t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) + repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE") + + terraformExecPath := filepath.Join(buildDir, "terraform") + if runtime.GOOS == "windows" { + terraformExecPath += ".exe" + } + t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) + t.Setenv("TERRAFORM", terraformExecPath) + repls.SetPath(terraformExecPath, "$TERRAFORM") + + // do it last so that full paths match first: + repls.SetPath(buildDir, "$BUILD_DIR") + workspaceClient, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -406,13 +424,12 @@ func readMergedScriptContents(t *testing.T, dir string) string { return strings.Join(prepares, "\n") } -func BuildCLI(t *testing.T, cwd, coverDir string) string { - execPath := filepath.Join(cwd, "build", "databricks") +func BuildCLI(t *testing.T, buildDir, coverDir string) string { + execPath := filepath.Join(buildDir, "databricks") if runtime.GOOS == "windows" { execPath += ".exe" } - start := time.Now() args := []string{ "go", "build", "-mod", "vendor", @@ -430,20 +447,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string { args = append(args, "-buildvcs=false") } - cmd := exec.Command(args[0], args[1:]...) - cmd.Dir = ".." - out, err := cmd.CombinedOutput() - elapsed := time.Since(start) - t.Logf("%s took %s", args, elapsed) - require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out) - if len(out) > 0 { - t.Logf("go build output: %s: %s", args, out) - } - - // Quick check + warm up cache: - cmd = exec.Command(execPath, "--version") - out, err = cmd.CombinedOutput() - require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out) + RunCommand(t, args, "..") + RunCommand(t, []string{execPath, "--version"}, ".") return execPath } @@ -581,3 +586,17 @@ func getUVDefaultCacheDir(t *testing.T) string { return cacheDir + "/uv" } } + +func RunCommand(t *testing.T, args []string, dir string) { + start := time.Now() + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + elapsed := time.Since(start) + t.Logf("%s took %s", args, elapsed) + + require.NoError(t, err, "%s failed: %s\n%s", args, err, out) + if len(out) > 0 { + t.Logf("%s output: %s", args, out) + } +} diff --git a/acceptance/build/.gitignore b/acceptance/build/.gitignore deleted file mode 100644 index a48b4db25..000000000 --- a/acceptance/build/.gitignore +++ /dev/null @@ -1 +0,0 @@ -databricks diff --git a/acceptance/bundle/variables/git-branch/output.txt b/acceptance/bundle/variables/git-branch/output.txt index fb3ab805a..5e7664f61 100644 --- a/acceptance/bundle/variables/git-branch/output.txt +++ b/acceptance/bundle/variables/git-branch/output.txt @@ -11,7 +11,7 @@ "name": "git", "target": "prod", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { @@ -61,7 +61,7 @@ Validation OK! "name": "git", "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "sync": { diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index fcaa25b4a..ed6c2b2af 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -7,7 +7,7 @@ }, "target": "dev", "terraform": { - "exec_path": "$TMPHOME" + "exec_path": "$TERRAFORM" } }, "resources": { diff --git a/acceptance/install_terraform.py b/acceptance/install_terraform.py new file mode 100755 index 000000000..4cf6a9729 --- /dev/null +++ b/acceptance/install_terraform.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +Script to set up terraform and databricks terraform provider in a local directory: + +- Download terraform. +- Download databricks provider. +- Write a .terraformrc config file that uses this directory. +- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider. +""" + +import os +import platform +import zipfile +import argparse +import json +from pathlib import Path +from urllib.request import urlretrieve + +os_name = platform.system().lower() + +arch = platform.machine().lower() +arch = {"x86_64": "amd64"}.get(arch, arch) +if os_name == "windows" and arch not in ("386", "amd64"): + # terraform 1.5.5 only has builds for these two. + arch = "amd64" + +terraform_version = "1.5.5" +terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip" +terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}" +terraform_binary = "terraform.exe" if os_name == "windows" else "terraform" + + +def retrieve(url, path): + if not path.exists(): + print(f"Downloading {url} -> {path}") + urlretrieve(url, path) + + +def read_version(path): + for line in path.open(): + if "ProviderVersion" in line: + # Expecting 'const ProviderVersion = "1.64.1"' + items = line.strip().split() + assert len(items) >= 3, items + assert items[-3:-1] == ["ProviderVersion", "="], items + version = items[-1].strip('"') + assert version, items + return version + raise SystemExit(f"Could not find ProviderVersion in {path}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--targetdir", default="build", type=Path) + parser.add_argument("--provider-version") + args = parser.parse_args() + target = args.targetdir + + if not args.provider_version: + version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go" + assert version_file.exists(), version_file + terraform_provider_version = read_version(version_file) + print(f"Read version {terraform_provider_version} from {version_file}") + else: + terraform_provider_version = args.provider_version + + terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip" + terraform_provider_url = ( + f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}" + ) + + target.mkdir(exist_ok=True, parents=True) + + zip_path = target / terraform_file + terraform_path = target / terraform_binary + terraform_provider_path = target / terraform_provider_file + + retrieve(terraform_url, zip_path) + retrieve(terraform_provider_url, terraform_provider_path) + + if not terraform_path.exists(): + print(f"Extracting {zip_path} -> {terraform_path}") + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(target) + + terraform_path.chmod(0o755) + + tfplugins_path = target / "tfplugins" + provider_dir = Path(tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}") + if not provider_dir.exists(): + print(f"Extracting {terraform_provider_path} -> {provider_dir}") + os.makedirs(provider_dir, exist_ok=True) + with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref: + zip_ref.extractall(provider_dir) + + files = list(provider_dir.iterdir()) + assert files, provider_dir + + for f in files: + f.chmod(0o755) + + terraformrc_path = target / ".terraformrc" + if not terraformrc_path.exists(): + path = json.dumps(str(tfplugins_path.absolute())) + text = f"""# Set these env variables before running databricks cli: +# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()} +# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()} + +provider_installation {{ + filesystem_mirror {{ + path = {path} + include = ["registry.terraform.io/databricks/databricks"] + }} +}} +""" + print(f"Writing {terraformrc_path}:\n{text}") + terraformrc_path.write_text(text) + + +if __name__ == "__main__": + main() diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf new file mode 100644 index 000000000..93f665ff4 --- /dev/null +++ b/acceptance/terraform/main.tf @@ -0,0 +1,25 @@ +terraform { + required_providers { + databricks = { + source = "databricks/databricks" + version = "1.64.1" + } + } + + required_version = "= 1.5.5" +} + +provider "databricks" { + # Optionally, specify the Databricks host and token + # host = "https://" + # token = "" +} + +data "databricks_current_user" "me" { + # Retrieves the current user's information +} + +output "username" { + description = "Username" + value = "${data.databricks_current_user.me.user_name}" +} diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt new file mode 100644 index 000000000..c3d453ea5 --- /dev/null +++ b/acceptance/terraform/output.txt @@ -0,0 +1,51 @@ + +>>> $TERRAFORM init -no-color -get=false + +Initializing the backend... + +Initializing provider plugins... +- Finding databricks/databricks versions matching "1.64.1"... +- Installing databricks/databricks v1.64.1... +- Installed databricks/databricks v1.64.1 (unauthenticated) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + + +Warning: Incomplete lock file information for providers + +Due to your customized provider installation methods, Terraform was forced to +calculate lock file checksums locally for the following providers: + - databricks/databricks + + +To calculate additional checksums for another platform, run: + terraform providers lock -platform=linux_amd64 +(where linux_amd64 is the platform to generate) + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. + +>>> $TERRAFORM plan -no-color +data.databricks_current_user.me: Reading... +data.databricks_current_user.me: Read complete after 0s [id=$USER.Id] + +Changes to Outputs: + + username = "$USERNAME" + +You can apply this plan to save these new output values to the Terraform +state, without changing any real infrastructure. + +───────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so Terraform can't +guarantee to take exactly these actions if you run "terraform apply" now. diff --git a/acceptance/terraform/script b/acceptance/terraform/script new file mode 100644 index 000000000..78e35049d --- /dev/null +++ b/acceptance/terraform/script @@ -0,0 +1,14 @@ +# Want to filter out these message: +# Mac: +# The current .terraform.lock.hcl file only includes checksums for +# darwin_arm64, so Terraform running on another platform will fail to install +# these providers. +# +# Linux: +# The current .terraform.lock.hcl file only includes checksums for linux_amd64, +# so Terraform running on another platform will fail to install these +# providers. + +trace $TERRAFORM init -no-color -get=false | grep -v 'includes checksums for' | grep -v 'so Terraform running on another' | grep -v 'providers\.' +trace $TERRAFORM plan -no-color +rm -fr .terraform.lock.hcl .terraform