From d86ad9189902d629cc85e0d79250a7301bb29aeb Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 4 Feb 2025 22:08:11 +0530 Subject: [PATCH 1/7] Allow test servers to return errors responses (#2291) ## Changes The APIs at Databricks when returning a non `200` status code will return a response body of the format: ``` { "error_code": "Error code", "message": "Human-readable error message." } ``` This PR adds the ability to stub non-200 status codes in the test server, allowing us to mock API errors from Databricks. ## Tests New test --- acceptance/acceptance_test.go | 8 +++-- acceptance/cmd_server_test.go | 4 +-- acceptance/config_test.go | 3 +- acceptance/server_test.go | 32 +++++++++---------- .../jobs/create-error/out.requests.txt | 1 + .../workspace/jobs/create-error/output.txt | 5 +++ acceptance/workspace/jobs/create-error/script | 1 + .../workspace/jobs/create-error/test.toml | 12 +++++++ libs/testserver/server.go | 11 +++---- 9 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 acceptance/workspace/jobs/create-error/out.requests.txt create mode 100644 acceptance/workspace/jobs/create-error/output.txt create mode 100644 acceptance/workspace/jobs/create-error/script create mode 100644 acceptance/workspace/jobs/create-error/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 871b8bd62..6e302fa96 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -261,8 +261,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) { - return stub.Response.Body, nil + server.Handle(stub.Pattern, func(req *http.Request) (any, int) { + statusCode := http.StatusOK + if stub.Response.StatusCode != 0 { + statusCode = stub.Response.StatusCode + } + return stub.Response.Body, statusCode }) } cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 9af63d0db..04d56c7d4 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(r *http.Request) (any, error) { + server.Handle("/", func(r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") @@ -40,7 +40,7 @@ func StartCmdServer(t *testing.T) *testserver.Server { exitcode = 1 } result["exitcode"] = exitcode - return result, nil + return result, http.StatusOK }) return server } diff --git a/acceptance/config_test.go b/acceptance/config_test.go index c7be223de..97be457e2 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -57,7 +57,8 @@ type ServerStub struct { // The response body to return. Response struct { - Body string + Body string + StatusCode int } } diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 4957a7668..a7695b21e 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -11,7 +11,7 @@ import ( ) func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -23,10 +23,10 @@ func AddHandlers(server *testserver.Server) { Name: "some-test-cluster-policy", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -34,10 +34,10 @@ func AddHandlers(server *testserver.Server) { InstancePoolId: "1234", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -49,32 +49,32 @@ func AddHandlers(server *testserver.Server) { ClusterId: "9876", }, }, - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) { return iam.User{ Id: "1000012345", UserName: "tester@databricks.com", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) { return workspace.ObjectInfo{ ObjectId: 1001, ObjectType: "DIRECTORY", Path: "", ResourceId: "1001", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", - }, nil + }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, error) { + server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) { return workspace.WorkspaceObjectPermissions{ ObjectId: "1001", ObjectType: "DIRECTORY", @@ -88,10 +88,10 @@ func AddHandlers(server *testserver.Server) { }, }, }, - }, nil + }, http.StatusOK }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, error) { - return "{}", nil + server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) { + return "{}", http.StatusOK }) } diff --git a/acceptance/workspace/jobs/create-error/out.requests.txt b/acceptance/workspace/jobs/create-error/out.requests.txt new file mode 100644 index 000000000..b22876b70 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/out.requests.txt @@ -0,0 +1 @@ +{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create-error/output.txt b/acceptance/workspace/jobs/create-error/output.txt new file mode 100644 index 000000000..0e69eeb4b --- /dev/null +++ b/acceptance/workspace/jobs/create-error/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] jobs create --json {"name":"abc"} +Error: Invalid access token. + +Exit code: 1 diff --git a/acceptance/workspace/jobs/create-error/script b/acceptance/workspace/jobs/create-error/script new file mode 100644 index 000000000..9ff7b5b87 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/script @@ -0,0 +1 @@ +trace $CLI jobs create --json '{"name":"abc"}' diff --git a/acceptance/workspace/jobs/create-error/test.toml b/acceptance/workspace/jobs/create-error/test.toml new file mode 100644 index 000000000..b45bf77e5 --- /dev/null +++ b/acceptance/workspace/jobs/create-error/test.toml @@ -0,0 +1,12 @@ +LocalOnly = true # request recording currently does not work with cloud environment +RecordRequests = true + +[[Server]] +Pattern = "POST /api/2.1/jobs/create" +Response.Body = ''' +{ + "error_code": "PERMISSION_DENIED", + "message": "Invalid access token." +} +''' +Response.StatusCode = 403 diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 2e8dbdfda..a751531ed 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -40,15 +40,11 @@ func New(t testutil.TestingT) *Server { } } -type HandlerFunc func(req *http.Request) (resp any, err error) +type HandlerFunc func(req *http.Request) (resp any, statusCode int) func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, err := handler(r) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + resp, statusCode := handler(r) if s.RecordRequests { body, err := io.ReadAll(r.Body) @@ -63,9 +59,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) var respBytes []byte - + var err error respString, ok := resp.(string) if ok { respBytes = []byte(respString) From 84b694f2a158186fa6d641e55e51af29761fa419 Mon Sep 17 00:00:00 2001 From: Simon Poltier Date: Tue, 4 Feb 2025 19:28:19 +0100 Subject: [PATCH 2/7] accept JSON includes (#2265) #2201 disabled using JSON as part of a bundle definition. I believe this was not intended. ## Changes Accept json files as includes, just as YAML files. ## Tests Covered by the tests in #2201 --- acceptance/bundle/includes/non_yaml_in_include/output.txt | 4 ++-- bundle/config/loader/process_root_includes.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/includes/non_yaml_in_include/output.txt b/acceptance/bundle/includes/non_yaml_in_include/output.txt index 6006ca14e..f5211cc4b 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/output.txt +++ b/acceptance/bundle/includes/non_yaml_in_include/output.txt @@ -1,7 +1,7 @@ -Error: Files in the 'include' configuration section must be YAML files. +Error: Files in the 'include' configuration section must be YAML or JSON files. in databricks.yml:5:4 -The file test.py in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead. +The file test.py in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead. Name: non_yaml_in_includes diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 198095742..69e6dd4e4 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -71,11 +71,11 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. continue } seen[rel] = true - if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" { + if filepath.Ext(rel) != ".yaml" && filepath.Ext(rel) != ".yml" && filepath.Ext(rel) != ".json" { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "Files in the 'include' configuration section must be YAML files.", - Detail: fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML file, and only YAML files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel), + Summary: "Files in the 'include' configuration section must be YAML or JSON files.", + Detail: fmt.Sprintf("The file %s in the 'include' configuration section is not a YAML or JSON file, and only such files are supported. To include files to sync, specify them in the 'sync.include' configuration section instead.", rel), Locations: b.Config.GetLocations(fmt.Sprintf("include[%d]", i)), }) continue From dcc61cd7636b4cdc647256bbd07aac8d4651da9e Mon Sep 17 00:00:00 2001 From: rikjansen-hu Date: Tue, 4 Feb 2025 19:30:02 +0100 Subject: [PATCH 3/7] Fix env variable for AzureCli local config (#2248) ## Changes Solves #1722 (current solution passes wrong variable) ## Tests None, this is a simple find-and-replace on a previous PR. Proof that this is the correct [variable](https://learn.microsoft.com/en-us/cli/azure/azure-cli-configuration#cli-configuration-file). This just passes the variable along to the Terraform environment, which [should](https://github.com/hashicorp/terraform/issues/25416) be picked up by Terraform. Co-authored-by: Rik Jansen --- bundle/deploy/terraform/init.go | 4 ++-- bundle/deploy/terraform/init_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index 5957611a4..a204222d0 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -101,9 +101,9 @@ var envCopy = []string{ // same auxiliary programs (e.g. `az`, or `gcloud`) as the CLI. "PATH", - // Include $AZURE_CONFIG_FILE in set of environment variables to pass along. + // Include $AZURE_CONFIG_DIR in set of environment variables to pass along. // This is set in Azure DevOps by the AzureCLI@2 task. - "AZURE_CONFIG_FILE", + "AZURE_CONFIG_DIR", // Include $TF_CLI_CONFIG_FILE to override terraform provider in development. // See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index c7a4ffe4a..4645ed007 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -292,7 +292,7 @@ func TestInheritEnvVars(t *testing.T) { t.Setenv("HOME", "/home/testuser") t.Setenv("PATH", "/foo:/bar") t.Setenv("TF_CLI_CONFIG_FILE", "/tmp/config.tfrc") - t.Setenv("AZURE_CONFIG_FILE", "/tmp/foo/bar") + t.Setenv("AZURE_CONFIG_DIR", "/tmp/foo/bar") ctx := context.Background() env := map[string]string{} @@ -301,7 +301,7 @@ func TestInheritEnvVars(t *testing.T) { assert.Equal(t, "/home/testuser", env["HOME"]) assert.Equal(t, "/foo:/bar", env["PATH"]) assert.Equal(t, "/tmp/config.tfrc", env["TF_CLI_CONFIG_FILE"]) - assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_FILE"]) + assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_DIR"]) } } From 2e1455841cced1337ad8ff8b456f57f217a9a74d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 4 Feb 2025 22:20:02 +0100 Subject: [PATCH 4/7] Update CODEOWNERS for cmd/labs (#2295) ## Changes The `CODEOWNERS` file must live in one of the directories specified in the [docs][docs], so the existing file under `cmd/labs` didn't work. This change moves the contents to the top-level file and includes @alexott as owner. [docs]: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-file-location --- .github/CODEOWNERS | 1 + cmd/labs/CODEOWNERS | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 cmd/labs/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 76835de7d..3c3895bc1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,2 @@ * @pietern @andrewnester @shreyas-goenka @denik +cmd/labs @alexott @nfx diff --git a/cmd/labs/CODEOWNERS b/cmd/labs/CODEOWNERS deleted file mode 100644 index cc93a75e6..000000000 --- a/cmd/labs/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @nfx From 1678503cb04abfed7fdd1d595d93ed722c75330e Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 5 Feb 2025 10:01:51 +0100 Subject: [PATCH 5/7] Fix docs template (#2283) ## Changes Comment breaks markdown front-matter and description cannot be read ## Tests --- bundle/docsgen/output/reference.md | 3 ++- bundle/docsgen/output/resources.md | 3 ++- bundle/docsgen/templates/reference.md | 3 ++- bundle/docsgen/templates/resources.md | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index a2241d017..8a89d354b 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,8 +1,9 @@ - --- description: Configuration reference for databricks.yml --- + + # Configuration reference This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index ff80ee635..df7578c73 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,8 +1,9 @@ - --- description: Learn about resources supported by Databricks Asset Bundles and how to configure them. --- + + # resources allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md index a17d53315..345afc509 100644 --- a/bundle/docsgen/templates/reference.md +++ b/bundle/docsgen/templates/reference.md @@ -1,8 +1,9 @@ - --- description: Configuration reference for databricks.yml --- + + # Configuration reference This article provides reference for keys supported by configuration (YAML). See [_](/dev-tools/bundles/index.md). diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md index fccfac47d..e9a6c8c5b 100644 --- a/bundle/docsgen/templates/resources.md +++ b/bundle/docsgen/templates/resources.md @@ -1,8 +1,9 @@ - --- description: Learn about resources supported by Databricks Asset Bundles and how to configure them. --- + + # resources allows you to specify information about the resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources). From 57b8d336e03cb2b9aa0c353d9cce7d85d5c72c7f Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:02:15 +0530 Subject: [PATCH 6/7] Add ability to record headers in acceptance tests (#2296) ## Changes HTTP headers like the User-Agent are an important part of our internal ETL pipelines. This PR adds the ability to validate the headers used in an HTTP request as part of our acceptance tests. ## Tests Modifying existing test. --- acceptance/acceptance_test.go | 3 +++ acceptance/config_test.go | 3 +++ .../workspace/jobs/create/out.requests.txt | 2 +- acceptance/workspace/jobs/create/test.toml | 17 ++++++++++++ libs/testdiff/replacement.go | 18 +++++++++++++ libs/testserver/server.go | 26 ++++++++++++++----- 6 files changed, 61 insertions(+), 8 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6e302fa96..f205217ff 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -156,6 +156,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { testdiff.PrepareReplacementsWorkspaceClient(t, &repls, workspaceClient) testdiff.PrepareReplacementsUUID(t, &repls) testdiff.PrepareReplacementsDevVersion(t, &repls) + testdiff.PrepareReplacementSdkVersion(t, &repls) + testdiff.PrepareReplacementsGoVersion(t, &repls) testDirs := getTests(t) require.NotEmpty(t, testDirs) @@ -253,6 +255,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if len(config.Server) > 0 || config.RecordRequests { server = testserver.New(t) server.RecordRequests = config.RecordRequests + server.IncludeRequestHeaders = config.IncludeRequestHeaders // If no custom server stubs are defined, add the default handlers. if len(config.Server) == 0 { diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 97be457e2..e24a683e7 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -47,6 +47,9 @@ type TestConfig struct { // Record the requests made to the server and write them as output to // out.requests.txt RecordRequests bool + + // List of request headers to include when recording requests. + IncludeRequestHeaders []string } type ServerStub struct { diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index b22876b70..4a85c4c43 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index e69569c18..1fd9b3cec 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,5 +1,6 @@ LocalOnly = true # request recording currently does not work with cloud environment RecordRequests = true +IncludeRequestHeaders = ["Authorization", "User-Agent"] [[Server]] Pattern = "POST /api/2.1/jobs/create" @@ -8,3 +9,19 @@ Response.Body = ''' "job_id": 1111 } ''' + +[[Repls]] +Old = "(linux|darwin|windows)" +New = "[OS]" + +[[Repls]] +Old = " upstream/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " upstream-version/[A-Za-z0-9.-]+" +New = "" + +[[Repls]] +Old = " cicd/[A-Za-z0-9.-]+" +New = "" diff --git a/libs/testdiff/replacement.go b/libs/testdiff/replacement.go index 5bbba1be1..d4d5eb27b 100644 --- a/libs/testdiff/replacement.go +++ b/libs/testdiff/replacement.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" + "golang.org/x/mod/semver" ) const ( @@ -208,3 +209,20 @@ func PrepareReplacementsDevVersion(t testutil.TestingT, r *ReplacementsContext) t.Helper() r.append(devVersionRegex, "[DEV_VERSION]") } + +func PrepareReplacementSdkVersion(t testutil.TestingT, r *ReplacementsContext) { + t.Helper() + r.Set(databricks.Version(), "[SDK_VERSION]") +} + +func goVersion() string { + gv := runtime.Version() + ssv := strings.ReplaceAll(gv, "go", "v") + sv := semver.Canonical(ssv) + return strings.TrimPrefix(sv, "v") +} + +func PrepareReplacementsGoVersion(t testutil.TestingT, r *ReplacementsContext) { + t.Helper() + r.Set(goVersion(), "[GO_VERSION]") +} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index a751531ed..5e3efe1c5 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "github.com/stretchr/testify/assert" @@ -17,15 +18,17 @@ type Server struct { t testutil.TestingT - RecordRequests bool + RecordRequests bool + IncludeRequestHeaders []string Requests []Request } type Request struct { - Method string `json:"method"` - Path string `json:"path"` - Body any `json:"body"` + Headers map[string]string `json:"headers,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -50,10 +53,19 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { body, err := io.ReadAll(r.Body) assert.NoError(s.t, err) + headers := make(map[string]string) + for k, v := range r.Header { + if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) { + continue + } + headers[k] = v[0] + } + s.Requests = append(s.Requests, Request{ - Method: r.Method, - Path: r.URL.Path, - Body: json.RawMessage(body), + Headers: headers, + Method: r.Method, + Path: r.URL.Path, + Body: json.RawMessage(body), }) } From 5c90752797d4cd2e49bdda5a05fb35e9a59d5edc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Feb 2025 11:53:36 +0000 Subject: [PATCH 7/7] acc: Added acceptance test for CLI commands inside bundle with and without profile flag (#2270) ## Changes This encodes existing behaviour in CLI as reported here: #1358 --- .../auth/bundle_and_profile/.databrickscfg | 5 +++ .../auth/bundle_and_profile/databricks.yml | 14 ++++++++ acceptance/auth/bundle_and_profile/output.txt | 32 +++++++++++++++++++ acceptance/auth/bundle_and_profile/script | 30 +++++++++++++++++ acceptance/auth/bundle_and_profile/test.toml | 8 +++++ 5 files changed, 89 insertions(+) create mode 100644 acceptance/auth/bundle_and_profile/.databrickscfg create mode 100644 acceptance/auth/bundle_and_profile/databricks.yml create mode 100644 acceptance/auth/bundle_and_profile/output.txt create mode 100644 acceptance/auth/bundle_and_profile/script create mode 100644 acceptance/auth/bundle_and_profile/test.toml diff --git a/acceptance/auth/bundle_and_profile/.databrickscfg b/acceptance/auth/bundle_and_profile/.databrickscfg new file mode 100644 index 000000000..628505286 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/.databrickscfg @@ -0,0 +1,5 @@ +[DEFAULT] +host = $DATABRICKS_HOST + +[profile_name] +host = https://test@non-existing-subdomain.databricks.com diff --git a/acceptance/auth/bundle_and_profile/databricks.yml b/acceptance/auth/bundle_and_profile/databricks.yml new file mode 100644 index 000000000..975661395 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-auth + +workspace: + host: $DATABRICKS_HOST + +targets: + dev: + default: true + workspace: + host: $DATABRICKS_HOST + prod: + workspace: + host: https://bar.com diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt new file mode 100644 index 000000000..022b3148d --- /dev/null +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -0,0 +1,32 @@ + +=== Inside the bundle, no flags +>>> errcode [CLI] current-user me +"[USERNAME]" + +=== Inside the bundle, target flags +>>> errcode [CLI] current-user me -t dev +"[USERNAME]" + +=== Inside the bundle, target and matching profile +>>> errcode [CLI] current-user me -t dev -p DEFAULT +"[USERNAME]" + +=== Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail +>>> errcode [CLI] current-user me -p profile_name +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] + +Exit code: 1 + +=== Inside the bundle, target and not matching profile +>>> errcode [CLI] current-user me -t dev -p profile_name +Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_URL] + +Exit code: 1 + +=== Outside the bundle, no flags +>>> errcode [CLI] current-user me +"[USERNAME]" + +=== Outside the bundle, profile flag +>>> errcode [CLI] current-user me -p profile_name +"[USERNAME]" diff --git a/acceptance/auth/bundle_and_profile/script b/acceptance/auth/bundle_and_profile/script new file mode 100644 index 000000000..b37f5e01d --- /dev/null +++ b/acceptance/auth/bundle_and_profile/script @@ -0,0 +1,30 @@ +# Replace placeholder with an actual host URL +envsubst < databricks.yml > out.yml && mv out.yml databricks.yml +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +host=$DATABRICKS_HOST +unset DATABRICKS_HOST + +title "Inside the bundle, no flags" +trace errcode $CLI current-user me | jq .userName + +title "Inside the bundle, target flags" +trace errcode $CLI current-user me -t dev | jq .userName + +title "Inside the bundle, target and matching profile" +trace errcode $CLI current-user me -t dev -p DEFAULT | jq .userName + +title "Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail" +trace errcode $CLI current-user me -p profile_name | jq .userName + +title "Inside the bundle, target and not matching profile" +trace errcode $CLI current-user me -t dev -p profile_name + +cd .. +export DATABRICKS_HOST=$host +title "Outside the bundle, no flags" +trace errcode $CLI current-user me | jq .userName + +title "Outside the bundle, profile flag" +trace errcode $CLI current-user me -p profile_name | jq .userName diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml new file mode 100644 index 000000000..b20190ca5 --- /dev/null +++ b/acceptance/auth/bundle_and_profile/test.toml @@ -0,0 +1,8 @@ +Badness = "When -p flag is used inside the bundle folder for any CLI commands, CLI use bundle host anyway instead of profile one" + +# Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix +# In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST +# This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST +[[Repls]] +Old='DATABRICKS_HOST' +New='DATABRICKS_URL'