From e088d0d996979ddd56dd9ce10d3d1becbdc771f6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 16:18:57 +0100 Subject: [PATCH 01/27] Add lint.sh to run golanci-lint in 2 stages (#2051) First stage is to run goimports and formatter, second is full suite. This ensures that imports and formatting are fixed even in presence of other issues. Otherwise golanci-lint refuses to fix anything https://github.com/golangci/golangci-lint/issues/5257 This helpful when running aider with config like this - aider will use that to autofix what it can after every update: ``` % cat .aider.conf.yml lint-cmd: - "go: ./lint.sh" ``` --- Makefile | 2 +- lint.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 lint.sh diff --git a/Makefile b/Makefile index f8e7834a5..7dca3b2cf 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ default: build lint: vendor @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." - @golangci-lint run --fix ./... + @./lint.sh ./... lintcheck: vendor @echo "✓ Linting source code with https://golangci-lint.run/ ..." diff --git a/lint.sh b/lint.sh new file mode 100755 index 000000000..c93d04c66 --- /dev/null +++ b/lint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +# With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied. +# https://github.com/golangci/golangci-lint/issues/5257 +# However, running goimports first alone will actually fix some of the compilation issues. +# Fixing formatting is also reasonable thing to do. +# For this reason, this script runs golangci-lint in two stages: +golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@ +exec golangci-lint run --fix $@ From 261b7f4083c4f10c0cc0e6c0ec9d95bfc2927174 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 16:26:21 +0100 Subject: [PATCH 02/27] Move bulk of "golden tests" logic to libs/testdiff (#2054) ## Changes - Detach "golden files" assertions from testcli runner. Now any output can be compared, no matter how it is obtained. - Move those assertion to libs/testdiff package. This allows using "golden files" in non-integration tests. ## Tests Existing tests --- .../bundle/init_default_python_test.go | 7 +- internal/testcli/golden.go | 201 +---------------- libs/testdiff/golden.go | 212 ++++++++++++++++++ .../testcli => libs/testdiff}/golden_test.go | 2 +- 4 files changed, 220 insertions(+), 202 deletions(-) create mode 100644 libs/testdiff/golden.go rename {internal/testcli => libs/testdiff}/golden_test.go (92%) diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go index 9b65636e9..c93e6b50b 100644 --- a/integration/bundle/init_default_python_test.go +++ b/integration/bundle/init_default_python_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/python/pythontest" + "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" ) @@ -50,14 +51,14 @@ func testDefaultPython(t *testing.T, pythonVersion string) { ctx, wt := acc.WorkspaceTest(t) uniqueProjectId := testutil.RandomName("") - ctx, replacements := testcli.WithReplacementsMap(ctx) + ctx, replacements := testdiff.WithReplacementsMap(ctx) replacements.Set(uniqueProjectId, "$UNIQUE_PRJ") user, err := wt.W.CurrentUser.Me(ctx) require.NoError(t, err) require.NotNil(t, user) - testcli.PrepareReplacementsUser(t, replacements, *user) - testcli.PrepareReplacements(t, replacements, wt.W) + testdiff.PrepareReplacementsUser(t, replacements, *user) + testdiff.PrepareReplacements(t, replacements, wt.W) tmpDir := t.TempDir() testutil.Chdir(t, tmpDir) diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go index 34f38f18a..a16f42c72 100644 --- a/internal/testcli/golden.go +++ b/internal/testcli/golden.go @@ -3,222 +3,27 @@ package testcli import ( "context" "fmt" - "os" - "regexp" - "slices" "strings" - "testing" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/testdiff" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) -var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" - -func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { - data, err := os.ReadFile(filename) - if os.IsNotExist(err) { - return "" - } - assert.NoError(t, err) - // On CI, on Windows \n in the file somehow end up as \r\n - return NormalizeNewlines(string(data)) -} - func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string { t.Logf("run args: [%s]", strings.Join(args, ", ")) r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() assert.NoError(t, err) - out := stderr.String() + stdout.String() - return ReplaceOutput(t, ctx, out) -} - -func WriteFile(t testutil.TestingT, filename, data string) { - t.Logf("Overwriting %s", filename) - err := os.WriteFile(filename, []byte(data), 0o644) - assert.NoError(t, err) + return stderr.String() + stdout.String() } func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) { - expected := ReadFile(t, ctx, expectedPath) - out := captureOutput(t, ctx, args) - - if out != expected { - actual := fmt.Sprintf("Output from %v", args) - testdiff.AssertEqualTexts(t, expectedPath, actual, expected, out) - - if OverwriteMode { - WriteFile(t, expectedPath, out) - } - } + testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath) } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) { - expected := ReadFile(t, ctx, expectedPath) - out := captureOutput(t, ctx, args) - - if out != expected { - actual := fmt.Sprintf("Output from %v", args) - testdiff.AssertEqualJQ(t.(*testing.T), expectedPath, actual, expected, out, ignorePaths) - - if OverwriteMode { - WriteFile(t, expectedPath, out) - } - } -} - -var ( - uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) - numIdRegex = regexp.MustCompile(`[0-9]{3,}`) - privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`) -) - -func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { - out = NormalizeNewlines(out) - replacements := GetReplacementsMap(ctx) - if replacements == nil { - t.Fatal("WithReplacementsMap was not called") - } - out = replacements.Replace(out) - out = uuidRegex.ReplaceAllString(out, "") - out = numIdRegex.ReplaceAllString(out, "") - out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3") - - return out -} - -type key int - -const ( - replacementsMapKey = key(1) -) - -type Replacement struct { - Old string - New string -} - -type ReplacementsContext struct { - Repls []Replacement -} - -func (r *ReplacementsContext) Replace(s string) string { - // QQQ Should probably only replace whole words - for _, repl := range r.Repls { - s = strings.ReplaceAll(s, repl.Old, repl.New) - } - return s -} - -func (r *ReplacementsContext) Set(old, new string) { - if old == "" || new == "" { - return - } - r.Repls = append(r.Repls, Replacement{Old: old, New: new}) -} - -func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) { - value := ctx.Value(replacementsMapKey) - if value != nil { - if existingMap, ok := value.(*ReplacementsContext); ok { - return ctx, existingMap - } - } - - newMap := &ReplacementsContext{} - ctx = context.WithValue(ctx, replacementsMapKey, newMap) - return ctx, newMap -} - -func GetReplacementsMap(ctx context.Context) *ReplacementsContext { - value := ctx.Value(replacementsMapKey) - if value != nil { - if existingMap, ok := value.(*ReplacementsContext); ok { - return existingMap - } - } - return nil -} - -func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { - // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) - host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") - r.Set(host, "$DATABRICKS_HOST") - r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") - r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") - r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") - r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL") - r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") - r.Set(w.Config.Token, "$DATABRICKS_TOKEN") - r.Set(w.Config.Username, "$DATABRICKS_USERNAME") - r.Set(w.Config.Password, "$DATABRICKS_PASSWORD") - r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") - r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") - r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") - r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS") - r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") - r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET") - // r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID") - r.Set(w.Config.AzureClientID, "$USERNAME") - r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") - r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL") - r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN") - r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") - r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") - r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET") - r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") - // This is set to words like "path" that happen too frequently - // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") -} - -func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { - // There could be exact matches or overlap between different name fields, so sort them by length - // to ensure we match the largest one first and map them all to the same token - names := []string{ - u.DisplayName, - u.UserName, - iamutil.GetShortUserName(&u), - u.Name.FamilyName, - u.Name.GivenName, - } - if u.Name != nil { - names = append(names, u.Name.FamilyName) - names = append(names, u.Name.GivenName) - } - for _, val := range u.Emails { - names = append(names, val.Value) - } - stableSortReverseLength(names) - - for _, name := range names { - r.Set(name, "$USERNAME") - } - - for ind, val := range u.Groups { - r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind)) - } - - r.Set(u.Id, "$USER.Id") - - for ind, val := range u.Roles { - r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind)) - } -} - -func stableSortReverseLength(strs []string) { - slices.SortStableFunc(strs, func(a, b string) int { - return len(b) - len(a) - }) -} - -func NormalizeNewlines(input string) string { - output := strings.ReplaceAll(input, "\r\n", "\n") - return strings.ReplaceAll(output, "\r", "\n") + testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths) } diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go new file mode 100644 index 000000000..c338ac9e5 --- /dev/null +++ b/libs/testdiff/golden.go @@ -0,0 +1,212 @@ +package testdiff + +import ( + "context" + "fmt" + "os" + "regexp" + "slices" + "strings" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/assert" +) + +var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" + +func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { + data, err := os.ReadFile(filename) + if os.IsNotExist(err) { + return "" + } + assert.NoError(t, err) + // On CI, on Windows \n in the file somehow end up as \r\n + return NormalizeNewlines(string(data)) +} + +func WriteFile(t testutil.TestingT, filename, data string) { + t.Logf("Overwriting %s", filename) + err := os.WriteFile(filename, []byte(data), 0o644) + assert.NoError(t, err) +} + +func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) { + expected := ReadFile(t, ctx, expectedPath) + + out = ReplaceOutput(t, ctx, out) + + if out != expected { + AssertEqualTexts(t, expectedPath, outTitle, expected, out) + + if OverwriteMode { + WriteFile(t, expectedPath, out) + } + } +} + +func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) { + expected := ReadFile(t, ctx, expectedPath) + + out = ReplaceOutput(t, ctx, out) + + if out != expected { + AssertEqualJQ(t.(*testing.T), expectedPath, outTitle, expected, out, ignorePaths) + + if OverwriteMode { + WriteFile(t, expectedPath, out) + } + } +} + +var ( + uuidRegex = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) + numIdRegex = regexp.MustCompile(`[0-9]{3,}`) + privatePathRegex = regexp.MustCompile(`(/tmp|/private)(/.*)/([a-zA-Z0-9]+)`) +) + +func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { + out = NormalizeNewlines(out) + replacements := GetReplacementsMap(ctx) + if replacements == nil { + t.Fatal("WithReplacementsMap was not called") + } + out = replacements.Replace(out) + out = uuidRegex.ReplaceAllString(out, "") + out = numIdRegex.ReplaceAllString(out, "") + out = privatePathRegex.ReplaceAllString(out, "/tmp/.../$3") + + return out +} + +type key int + +const ( + replacementsMapKey = key(1) +) + +type Replacement struct { + Old string + New string +} + +type ReplacementsContext struct { + Repls []Replacement +} + +func (r *ReplacementsContext) Replace(s string) string { + // QQQ Should probably only replace whole words + for _, repl := range r.Repls { + s = strings.ReplaceAll(s, repl.Old, repl.New) + } + return s +} + +func (r *ReplacementsContext) Set(old, new string) { + if old == "" || new == "" { + return + } + r.Repls = append(r.Repls, Replacement{Old: old, New: new}) +} + +func WithReplacementsMap(ctx context.Context) (context.Context, *ReplacementsContext) { + value := ctx.Value(replacementsMapKey) + if value != nil { + if existingMap, ok := value.(*ReplacementsContext); ok { + return ctx, existingMap + } + } + + newMap := &ReplacementsContext{} + ctx = context.WithValue(ctx, replacementsMapKey, newMap) + return ctx, newMap +} + +func GetReplacementsMap(ctx context.Context) *ReplacementsContext { + value := ctx.Value(replacementsMapKey) + if value != nil { + if existingMap, ok := value.(*ReplacementsContext); ok { + return existingMap + } + } + return nil +} + +func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { + // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) + host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") + r.Set(host, "$DATABRICKS_HOST") + r.Set(w.Config.ClusterID, "$DATABRICKS_CLUSTER_ID") + r.Set(w.Config.WarehouseID, "$DATABRICKS_WAREHOUSE_ID") + r.Set(w.Config.ServerlessComputeID, "$DATABRICKS_SERVERLESS_COMPUTE_ID") + r.Set(w.Config.MetadataServiceURL, "$DATABRICKS_METADATA_SERVICE_URL") + r.Set(w.Config.AccountID, "$DATABRICKS_ACCOUNT_ID") + r.Set(w.Config.Token, "$DATABRICKS_TOKEN") + r.Set(w.Config.Username, "$DATABRICKS_USERNAME") + r.Set(w.Config.Password, "$DATABRICKS_PASSWORD") + r.Set(w.Config.Profile, "$DATABRICKS_CONFIG_PROFILE") + r.Set(w.Config.ConfigFile, "$DATABRICKS_CONFIG_FILE") + r.Set(w.Config.GoogleServiceAccount, "$DATABRICKS_GOOGLE_SERVICE_ACCOUNT") + r.Set(w.Config.GoogleCredentials, "$GOOGLE_CREDENTIALS") + r.Set(w.Config.AzureResourceID, "$DATABRICKS_AZURE_RESOURCE_ID") + r.Set(w.Config.AzureClientSecret, "$ARM_CLIENT_SECRET") + // r.Set(w.Config.AzureClientID, "$ARM_CLIENT_ID") + r.Set(w.Config.AzureClientID, "$USERNAME") + r.Set(w.Config.AzureTenantID, "$ARM_TENANT_ID") + r.Set(w.Config.ActionsIDTokenRequestURL, "$ACTIONS_ID_TOKEN_REQUEST_URL") + r.Set(w.Config.ActionsIDTokenRequestToken, "$ACTIONS_ID_TOKEN_REQUEST_TOKEN") + r.Set(w.Config.AzureEnvironment, "$ARM_ENVIRONMENT") + r.Set(w.Config.ClientID, "$DATABRICKS_CLIENT_ID") + r.Set(w.Config.ClientSecret, "$DATABRICKS_CLIENT_SECRET") + r.Set(w.Config.DatabricksCliPath, "$DATABRICKS_CLI_PATH") + // This is set to words like "path" that happen too frequently + // r.Set(w.Config.AuthType, "$DATABRICKS_AUTH_TYPE") +} + +func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { + // There could be exact matches or overlap between different name fields, so sort them by length + // to ensure we match the largest one first and map them all to the same token + names := []string{ + u.DisplayName, + u.UserName, + iamutil.GetShortUserName(&u), + u.Name.FamilyName, + u.Name.GivenName, + } + if u.Name != nil { + names = append(names, u.Name.FamilyName) + names = append(names, u.Name.GivenName) + } + for _, val := range u.Emails { + names = append(names, val.Value) + } + stableSortReverseLength(names) + + for _, name := range names { + r.Set(name, "$USERNAME") + } + + for ind, val := range u.Groups { + r.Set(val.Value, fmt.Sprintf("$USER.Groups[%d]", ind)) + } + + r.Set(u.Id, "$USER.Id") + + for ind, val := range u.Roles { + r.Set(val.Value, fmt.Sprintf("$USER.Roles[%d]", ind)) + } +} + +func stableSortReverseLength(strs []string) { + slices.SortStableFunc(strs, func(a, b string) int { + return len(b) - len(a) + }) +} + +func NormalizeNewlines(input string) string { + output := strings.ReplaceAll(input, "\r\n", "\n") + return strings.ReplaceAll(output, "\r", "\n") +} diff --git a/internal/testcli/golden_test.go b/libs/testdiff/golden_test.go similarity index 92% rename from internal/testcli/golden_test.go rename to libs/testdiff/golden_test.go index 215bf33d3..0fc32be21 100644 --- a/internal/testcli/golden_test.go +++ b/libs/testdiff/golden_test.go @@ -1,4 +1,4 @@ -package testcli +package testdiff import ( "testing" From 1306e5ec6745b36acd7c17c9b3b4aea1f55d6807 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 18:41:45 +0100 Subject: [PATCH 03/27] Add CODEOWNERS (#2055) Goal is to have DABs core team automatically added as reviewers so that you don't have to click manually. Based on this example: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..76835de7d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @pietern @andrewnester @shreyas-goenka @denik From 1ce20a2612031a3d5c7b663203296957671e4f1d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Dec 2024 19:39:33 +0100 Subject: [PATCH 04/27] lint.sh: read config for formatters; include gofmt (#2056) As suggested here: https://github.com/databricks/cli/pull/2051#discussion_r1899641273 --- lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint.sh b/lint.sh index c93d04c66..13efa9855 100755 --- a/lint.sh +++ b/lint.sh @@ -5,5 +5,5 @@ set -euo pipefail # However, running goimports first alone will actually fix some of the compilation issues. # Fixing formatting is also reasonable thing to do. # For this reason, this script runs golangci-lint in two stages: -golangci-lint run --fix --no-config --disable-all --enable gofumpt,goimports $@ +golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@ exec golangci-lint run --fix $@ From 511db52bb7a47270dd883bb490a8844f9b25fe76 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:21:13 +0530 Subject: [PATCH 05/27] Remove unnecessary GET call in pipeline runner (#1850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This GET API call is unnecessary and serves no purpose. Let's remove it. Noticed this when I was adding a unit test for the pipeline runner here: https://github.com/databricks/cli/pull/1849 ## Tests Manually. ### Case 1: The pipeline does not exist Before: ``` ➜ my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood Error: User shreyas.goenka@databricks.com does not have View permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg. ``` After: ``` ➜ my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood Error: User shreyas.goenka@databricks.com does not have Run permissions on pipeline 9941901a-e48b-4d04-b6ba-e0072ad126bg. ``` ### Case 2: Pipeline exists Before: ``` ➜ my_project git:(master) ✗ databricks bundle run my_project_pipeline -p dogfood --restart Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/0f988d62-9ec7-49f1-b429-5572ece3a9aa 2024-11-18T15:30:36.054Z update_progress INFO "Update 0f988d is WAITING_FOR_RESOURCES." ``` After: ``` ➜ my_project git:(master) ✗ cli bundle run my_project_pipeline -p dogfood --restart Update URL: https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/9941901a-e48b-4d04-b6ba-e0072ad126bf/updates/87b43350-6186-4a9b-9d0e-38da2ecf33ae 2024-11-18T15:28:27.144Z update_progress INFO "Update 87b433 is WAITING_FOR_RESOURCES." ``` --- bundle/run/pipeline.go | 5 ----- bundle/run/pipeline_test.go | 2 -- 2 files changed, 7 deletions(-) diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index a0e7d1e1e..d84015d76 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -90,11 +90,6 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp // Include resource key in logger. ctx = log.NewContext(ctx, log.GetLogger(ctx).With("resource", r.Key())) w := r.bundle.WorkspaceClient() - _, err := w.Pipelines.GetByPipelineId(ctx, pipelineID) - if err != nil { - log.Warnf(ctx, "Cannot get pipeline: %s", err) - return nil, err - } req, err := opts.Pipeline.toPayload(r.pipeline, pipelineID) if err != nil { diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index 66f9d86be..bfa0c5846 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -90,8 +90,6 @@ func TestPipelineRunnerRestart(t *testing.T) { PipelineId: "123", }).Return(mockWait, nil) - pipelineApi.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{}, nil) - // Mock runner starting a new update pipelineApi.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{ PipelineId: "123", From 3f523b45cc59aa2cb4fbbffdf267e8ea094b3ad6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 31 Dec 2024 15:01:45 +0100 Subject: [PATCH 06/27] Fix lost diags across different mutators (#2057) ## Changes Fix cases where accumulated diagnostics are lost instead of being propagated further. In some cases it's not possible, add a comment there. ## Tests Existing tests --- bundle/artifacts/expand_globs.go | 2 +- bundle/config/mutator/resolve_resource_references.go | 1 + bundle/config/root.go | 3 ++- bundle/config/validate/folder_permissions.go | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 7d44db0be..c0af7c69e 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -97,7 +97,7 @@ func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return dyn.SetByPath(v, base, dyn.V(output)) }) if err != nil { - return diag.FromErr(err) + diags = diags.Extend(diag.FromErr(err)) } return diags diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index bf902f928..20a5b6585 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -40,6 +40,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) }) } + // Note, diags are lost from all goroutines except the first one to return diag return diag.FromErr(errs.Wait()) } diff --git a/bundle/config/root.go b/bundle/config/root.go index 4b1467456..91c15fd9d 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -102,7 +102,8 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { // Convert normalized configuration tree to typed configuration. err = r.updateWithDynamicValue(v) if err != nil { - return nil, diag.Errorf("failed to load %s: %v", path, err) + diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) + return nil, diags } return &r, diags } diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go index aa89a0551..5b28d599e 100644 --- a/bundle/config/validate/folder_permissions.go +++ b/bundle/config/validate/folder_permissions.go @@ -36,7 +36,8 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) } if err := g.Wait(); err != nil { - return diag.FromErr(err) + // Note, only diag from first coroutine is captured, others are lost + diags = diags.Extend(diag.FromErr(err)) } for _, r := range results { From 3f75240a562d88a9411cdd50a2ba24139ca094b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 10:49:21 +0100 Subject: [PATCH 07/27] Improve test output to include correct location (#2058) ## Changes - Add t.Helper() in testcli-related helpers, this ensures that output is attributed correctly to test case and not to the helper. - Modify testlcli.Run() to run process in foreground. This is needed for t.Helper to work. - Extend a few assertions with message to help attribute it to proper helper where needed. ## Tests Manually reviewed test output. Before: ``` + go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/ === RUN TestDefaultPython === RUN TestDefaultPython/3.9 workspace.go:26: aws golden.go:14: run args: [bundle, init, default-python, --config-file, config.json] runner.go:206: [databricks stderr]: runner.go:206: [databricks stderr]: Welcome to the default Python template for Databricks Asset Bundles! ... testdiff.go:23: Error Trace: /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:23 /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:43 /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45 ... ``` After: ``` + go test --timeout 3h -v -run TestDefaultPython/3.9 ./integration/bundle/ === RUN TestDefaultPython === RUN TestDefaultPython/3.9 init_default_python_test.go:51: CLOUD_ENV=aws init_default_python_test.go:92: args: bundle, init, default-python, --config-file, config.json init_default_python_test.go:92: stderr: init_default_python_test.go:92: stderr: Welcome to the default Python template for Databricks Asset Bundles! ... init_default_python_test.go:92: Error Trace: /Users/denis.bilenko/work/cli/libs/testdiff/testdiff.go:24 /Users/denis.bilenko/work/cli/libs/testdiff/golden.go:46 /Users/denis.bilenko/work/cli/internal/testcli/golden.go:23 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:92 /Users/denis.bilenko/work/cli/integration/bundle/init_default_python_test.go:45 ... ``` --- integration/internal/acc/workspace.go | 14 +++++--- internal/testcli/golden.go | 5 +-- internal/testcli/runner.go | 47 ++++++++++++++++++++++++--- internal/testutil/interface.go | 2 ++ libs/testdiff/golden.go | 11 +++++-- libs/testdiff/testdiff.go | 4 ++- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/integration/internal/acc/workspace.go b/integration/internal/acc/workspace.go index 2f8a5b8e7..64deda7c1 100644 --- a/integration/internal/acc/workspace.go +++ b/integration/internal/acc/workspace.go @@ -21,9 +21,10 @@ type WorkspaceT struct { } func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { + t.Helper() loadDebugEnvIfRunFromIDE(t, "workspace") - t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) + t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) w, err := databricks.NewWorkspaceClient() require.NoError(t, err) @@ -41,9 +42,10 @@ func WorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { // Run the workspace test only on UC workspaces. func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { + t.Helper() loadDebugEnvIfRunFromIDE(t, "workspace") - t.Log(testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) + t.Logf("CLOUD_ENV=%s", testutil.GetEnvOrSkipTest(t, "CLOUD_ENV")) if os.Getenv("TEST_METASTORE_ID") == "" { t.Skipf("Skipping on non-UC workspaces") @@ -67,19 +69,21 @@ func UcWorkspaceTest(t testutil.TestingT) (context.Context, *WorkspaceT) { } func (t *WorkspaceT) TestClusterID() string { + t.Helper() clusterID := testutil.GetEnvOrSkipTest(t, "TEST_BRICKS_CLUSTER_ID") err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from EnsureClusterIsRunning for clusterID=%s", clusterID) return clusterID } func (t *WorkspaceT) RunPython(code string) (string, error) { + t.Helper() var err error // Create command executor only once per test. if t.exec == nil { t.exec, err = t.W.CommandExecution.Start(t.ctx, t.TestClusterID(), compute.LanguagePython) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from CommandExecution.Start(clusterID=%v)", t.TestClusterID()) t.Cleanup(func() { err := t.exec.Destroy(t.ctx) @@ -88,7 +92,7 @@ func (t *WorkspaceT) RunPython(code string) (string, error) { } results, err := t.exec.Execute(t.ctx, code) - require.NoError(t, err) + require.NoError(t, err, "Unexpected error from Execute(%v)", code) require.NotEqual(t, compute.ResultTypeError, results.ResultType, results.Cause) output, ok := results.Data.(string) require.True(t, ok, "unexpected type %T", results.Data) diff --git a/internal/testcli/golden.go b/internal/testcli/golden.go index a16f42c72..669cc2f9b 100644 --- a/internal/testcli/golden.go +++ b/internal/testcli/golden.go @@ -3,7 +3,6 @@ package testcli import ( "context" "fmt" - "strings" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/testdiff" @@ -11,7 +10,7 @@ import ( ) func captureOutput(t testutil.TestingT, ctx context.Context, args []string) string { - t.Logf("run args: [%s]", strings.Join(args, ", ")) + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() assert.NoError(t, err) @@ -19,11 +18,13 @@ func captureOutput(t testutil.TestingT, ctx context.Context, args []string) stri } func AssertOutput(t testutil.TestingT, ctx context.Context, args []string, expectedPath string) { + t.Helper() out := captureOutput(t, ctx, args) testdiff.AssertOutput(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath) } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, args []string, expectedPath string, ignorePaths []string) { + t.Helper() out := captureOutput(t, ctx, args) testdiff.AssertOutputJQ(t, ctx, out, fmt.Sprintf("Output from %v", args), expectedPath, ignorePaths) } diff --git a/internal/testcli/runner.go b/internal/testcli/runner.go index 95073b57c..52decad2c 100644 --- a/internal/testcli/runner.go +++ b/internal/testcli/runner.go @@ -69,6 +69,7 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s } func (r *Runner) registerFlagCleanup(c *cobra.Command) { + r.Helper() // Find target command that will be run. Example: if the command run is `databricks fs cp`, // target command corresponds to `cp` targetCmd, _, err := c.Find(r.args) @@ -230,13 +231,48 @@ func (r *Runner) RunBackground() { } func (r *Runner) Run() (bytes.Buffer, bytes.Buffer, error) { - r.RunBackground() - err := <-r.errch - return r.stdout, r.stderr, err + r.Helper() + var stdout, stderr bytes.Buffer + ctx := cmdio.NewContext(r.ctx, &cmdio.Logger{ + Mode: flags.ModeAppend, + Reader: bufio.Reader{}, + Writer: &stderr, + }) + + cli := cmd.New(ctx) + cli.SetOut(&stdout) + cli.SetErr(&stderr) + cli.SetArgs(r.args) + + r.Logf(" args: %s", strings.Join(r.args, ", ")) + + err := root.Execute(ctx, cli) + if err != nil { + r.Logf(" error: %s", err) + } + + if stdout.Len() > 0 { + // Make a copy of the buffer such that it remains "unread". + scanner := bufio.NewScanner(bytes.NewBuffer(stdout.Bytes())) + for scanner.Scan() { + r.Logf("stdout: %s", scanner.Text()) + } + } + + if stderr.Len() > 0 { + // Make a copy of the buffer such that it remains "unread". + scanner := bufio.NewScanner(bytes.NewBuffer(stderr.Bytes())) + for scanner.Scan() { + r.Logf("stderr: %s", scanner.Text()) + } + } + + return stdout, stderr, err } // Like [require.Eventually] but errors if the underlying command has failed. func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, msgAndArgs ...any) { + r.Helper() ch := make(chan bool, 1) timer := time.NewTimer(waitFor) @@ -269,12 +305,14 @@ func (r *Runner) Eventually(condition func() bool, waitFor, tick time.Duration, } func (r *Runner) RunAndExpectOutput(heredoc string) { + r.Helper() stdout, _, err := r.Run() require.NoError(r, err) require.Equal(r, cmdio.Heredoc(heredoc), strings.TrimSpace(stdout.String())) } func (r *Runner) RunAndParseJSON(v any) { + r.Helper() stdout, _, err := r.Run() require.NoError(r, err) err = json.Unmarshal(stdout.Bytes(), &v) @@ -291,7 +329,7 @@ func NewRunner(t testutil.TestingT, ctx context.Context, args ...string) *Runner } func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer) { - t.Logf("run args: [%s]", strings.Join(args, ", ")) + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() require.NoError(t, err) @@ -299,6 +337,7 @@ func RequireSuccessfulRun(t testutil.TestingT, ctx context.Context, args ...stri } func RequireErrorRun(t testutil.TestingT, ctx context.Context, args ...string) (bytes.Buffer, bytes.Buffer, error) { + t.Helper() r := NewRunner(t, ctx, args...) stdout, stderr, err := r.Run() require.Error(t, err) diff --git a/internal/testutil/interface.go b/internal/testutil/interface.go index 2c3004800..97441212d 100644 --- a/internal/testutil/interface.go +++ b/internal/testutil/interface.go @@ -24,4 +24,6 @@ type TestingT interface { Setenv(key, value string) TempDir() string + + Helper() } diff --git a/libs/testdiff/golden.go b/libs/testdiff/golden.go index c338ac9e5..b67eb50a9 100644 --- a/libs/testdiff/golden.go +++ b/libs/testdiff/golden.go @@ -19,22 +19,25 @@ import ( var OverwriteMode = os.Getenv("TESTS_OUTPUT") == "OVERWRITE" func ReadFile(t testutil.TestingT, ctx context.Context, filename string) string { + t.Helper() data, err := os.ReadFile(filename) if os.IsNotExist(err) { return "" } - assert.NoError(t, err) + assert.NoError(t, err, "Failed to read %s", filename) // On CI, on Windows \n in the file somehow end up as \r\n return NormalizeNewlines(string(data)) } func WriteFile(t testutil.TestingT, filename, data string) { + t.Helper() t.Logf("Overwriting %s", filename) err := os.WriteFile(filename, []byte(data), 0o644) - assert.NoError(t, err) + assert.NoError(t, err, "Failed to write %s", filename) } func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string) { + t.Helper() expected := ReadFile(t, ctx, expectedPath) out = ReplaceOutput(t, ctx, out) @@ -49,6 +52,7 @@ func AssertOutput(t testutil.TestingT, ctx context.Context, out, outTitle, expec } func AssertOutputJQ(t testutil.TestingT, ctx context.Context, out, outTitle, expectedPath string, ignorePaths []string) { + t.Helper() expected := ReadFile(t, ctx, expectedPath) out = ReplaceOutput(t, ctx, out) @@ -69,6 +73,7 @@ var ( ) func ReplaceOutput(t testutil.TestingT, ctx context.Context, out string) string { + t.Helper() out = NormalizeNewlines(out) replacements := GetReplacementsMap(ctx) if replacements == nil { @@ -136,6 +141,7 @@ func GetReplacementsMap(ctx context.Context) *ReplacementsContext { } func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databricks.WorkspaceClient) { + t.Helper() // in some clouds (gcp) w.Config.Host includes "https://" prefix in others it's really just a host (azure) host := strings.TrimPrefix(strings.TrimPrefix(w.Config.Host, "http://"), "https://") r.Set(host, "$DATABRICKS_HOST") @@ -167,6 +173,7 @@ func PrepareReplacements(t testutil.TestingT, r *ReplacementsContext, w *databri } func PrepareReplacementsUser(t testutil.TestingT, r *ReplacementsContext, u iam.User) { + t.Helper() // There could be exact matches or overlap between different name fields, so sort them by length // to ensure we match the largest one first and map them all to the same token names := []string{ diff --git a/libs/testdiff/testdiff.go b/libs/testdiff/testdiff.go index 1e1df727a..71b781362 100644 --- a/libs/testdiff/testdiff.go +++ b/libs/testdiff/testdiff.go @@ -18,9 +18,10 @@ func UnifiedDiff(filename1, filename2, s1, s2 string) string { } func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out string) { + t.Helper() if len(out) < 1000 && len(expected) < 1000 { // This shows full strings + diff which could be useful when debugging newlines - assert.Equal(t, expected, out) + assert.Equal(t, expected, out, "%s vs %s", filename1, filename2) } else { // only show diff for large texts diff := UnifiedDiff(filename1, filename2, expected, out) @@ -29,6 +30,7 @@ func AssertEqualTexts(t testutil.TestingT, filename1, filename2, expected, out s } func AssertEqualJQ(t testutil.TestingT, expectedName, outName, expected, out string, ignorePaths []string) { + t.Helper() patch, err := jsondiff.CompareJSON([]byte(expected), []byte(out)) if err != nil { t.Logf("CompareJSON error for %s vs %s: %s (fallback to textual comparison)", outName, expectedName, err) From d7f69f6a5dc80450498119a7e2a60eecfc3f0cab Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 11:31:35 +0100 Subject: [PATCH 08/27] Upgrade golangci-lint to v1.63.1 (#2064) Upgrade your laptops with: brew install golangci-lint This has a lot more autofixes, which makes it easier to adopt those linters. https://golangci-lint.run/product/changelog/#v1630 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a51927594..c737993aa 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -75,7 +75,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.62.2 + version: v1.63.1 args: --timeout=15m validate-bundle-schema: From c262b75c3d0e48839fe327697e2758d3120d780e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 11:33:06 +0100 Subject: [PATCH 09/27] Make lint.sh to run golangci-lint only once in the best case (#2062) Follow up to #2051 and #2056. Running golangci-lint twice always is measurably slower (1.5s vs 2.5s), so only run it twice in case it is necessary. --- lint.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lint.sh b/lint.sh index 13efa9855..1f881eaf7 100755 --- a/lint.sh +++ b/lint.sh @@ -1,9 +1,14 @@ #!/bin/bash -set -euo pipefail +set -uo pipefail # With golangci-lint, if there are any compliation issues, then formatters' autofix won't be applied. # https://github.com/golangci/golangci-lint/issues/5257 -# However, running goimports first alone will actually fix some of the compilation issues. -# Fixing formatting is also reasonable thing to do. -# For this reason, this script runs golangci-lint in two stages: -golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix $@ -exec golangci-lint run --fix $@ + +golangci-lint run --fix "$@" +lint_exit_code=$? + +if [ $lint_exit_code -ne 0 ]; then + # These linters work in presence of compilation issues when run alone, so let's get these fixes at least. + golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix "$@" +fi + +exit $lint_exit_code From b053bfb4de577d8ee2d4058672a694af69d80891 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 2 Jan 2025 11:41:55 +0100 Subject: [PATCH 10/27] Verify that the bundle schema is up to date in CI (#2061) ## Changes I noticed a diff in the schema in #2052. This check should be performed automatically. ## Tests This PR includes a commit that changes the schema to check that the workflow actually fails. --- .github/workflows/push.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c737993aa..25bd5f4d6 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -90,6 +90,13 @@ jobs: with: go-version: 1.23.4 + - name: Verify that the schema is up to date + run: | + if ! ( make schema && git diff --exit-code ); then + echo "The schema is not up to date. Please run 'make schema' and commit the changes." + exit 1 + fi + # Github repo: https://github.com/ajv-validator/ajv-cli - name: Install ajv-cli run: npm install -g ajv-cli@5.0.0 From 0b80784df759673d0ccc3874d19d6662ff11ac54 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:03:41 +0100 Subject: [PATCH 11/27] Enable testifylint and fix the issues (#2065) ## Changes - Enable new linter: testifylint. - Apply fixes with --fix. - Fix remaining issues (mostly with aider). There were 2 cases we --fix did the wrong thing - this seems to a be a bug in linter: https://github.com/Antonboom/testifylint/issues/210 Nonetheless, I kept that check enabled, it seems useful, just need to be fixed manually after autofix. ## Tests Existing tests --- .golangci.yaml | 8 +++- bundle/bundle_test.go | 3 +- .../configure_dashboard_defaults_test.go | 6 +-- .../config/mutator/default_queueing_test.go | 10 ++-- .../mutator/environments_compat_test.go | 4 +- bundle/config/mutator/merge_job_tasks_test.go | 4 +- .../mutator/process_target_mode_test.go | 12 ++--- .../resolve_variable_references_test.go | 6 +-- .../mutator/rewrite_workspace_prefix_test.go | 2 +- bundle/config/mutator/set_variables_test.go | 8 ++-- bundle/config/resources_test.go | 6 +-- bundle/config/validate/files_to_sync_test.go | 4 +- .../validate/job_cluster_key_defined_test.go | 10 ++-- bundle/context_test.go | 2 +- bundle/deploy/state_pull_test.go | 3 +- bundle/deploy/state_update_test.go | 12 ++--- bundle/deploy/terraform/convert_test.go | 10 ++-- bundle/deploy/terraform/init_test.go | 4 +- bundle/internal/schema/main_test.go | 3 +- bundle/libraries/filer_volume_test.go | 4 +- bundle/libraries/helpers_test.go | 12 ++--- bundle/permissions/filter_test.go | 28 +++++------ bundle/permissions/mutator_test.go | 2 +- .../permission_diagnostics_test.go | 2 +- bundle/run/job_test.go | 2 +- bundle/tests/complex_variables_test.go | 2 +- bundle/tests/environment_git_test.go | 5 +- bundle/tests/environment_overrides_test.go | 8 ++-- .../environments_job_and_pipeline_test.go | 6 +-- bundle/tests/git_test.go | 5 +- bundle/tests/issue_1828_test.go | 4 +- bundle/tests/job_and_pipeline_test.go | 6 +-- bundle/tests/job_cluster_key_test.go | 6 +-- bundle/tests/model_serving_endpoint_test.go | 2 +- bundle/tests/override_job_tasks_test.go | 28 +++++------ bundle/tests/presets_test.go | 4 +- bundle/tests/python_wheel_test.go | 10 ++-- bundle/tests/quality_monitor_test.go | 2 +- bundle/tests/registered_model_test.go | 2 +- .../sync_include_exclude_no_matches_test.go | 20 ++++---- bundle/tests/sync_test.go | 4 +- cmd/bundle/generate/dashboard_test.go | 2 +- cmd/bundle/generate/generate_test.go | 3 +- cmd/configure/configure_test.go | 4 +- cmd/labs/github/ref_test.go | 9 ++-- cmd/labs/github/releases_test.go | 5 +- cmd/labs/github/repositories_test.go | 5 +- cmd/labs/localcache/jsonfile_test.go | 2 +- cmd/labs/project/installer_test.go | 17 +++---- integration/bundle/artifacts_test.go | 12 ++--- integration/bundle/deploy_test.go | 3 +- integration/bundle/destroy_test.go | 5 +- integration/cmd/fs/ls_test.go | 15 +++--- integration/cmd/fs/mkdir_test.go | 8 ++-- integration/cmd/sync/sync_test.go | 2 +- integration/libs/filer/filer_test.go | 46 +++++++++---------- libs/auth/cache/file_test.go | 2 +- libs/databrickscfg/ops_test.go | 2 +- libs/databrickscfg/profile/file_test.go | 8 ++-- libs/errs/aggregate_test.go | 19 ++++---- libs/filer/completer/completer_test.go | 12 ++--- libs/filer/fs_test.go | 4 +- libs/fileset/glob_test.go | 2 +- libs/flags/json_flag_test.go | 8 ++-- libs/folders/folders_test.go | 2 +- libs/jsonschema/utils_test.go | 4 +- libs/notebook/detect_test.go | 5 +- libs/process/background_test.go | 2 +- libs/process/forwarded_test.go | 4 +- libs/process/opts_test.go | 2 +- libs/python/interpreters_unix_test.go | 2 +- libs/sync/snapshot_test.go | 24 +++++----- libs/sync/sync_test.go | 12 ++--- libs/template/config_test.go | 8 ++-- libs/template/helpers_test.go | 2 +- libs/template/renderer_test.go | 8 ++-- libs/vfs/filer_test.go | 9 ++-- 77 files changed, 277 insertions(+), 283 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 9e69e5146..fd83e0882 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -11,6 +11,7 @@ linters: - gofmt - gofumpt - goimports + - testifylint linters-settings: govet: enable-all: true @@ -32,7 +33,10 @@ linters-settings: gofumpt: module-path: github.com/databricks/cli extra-rules: true - #goimports: - # local-prefixes: github.com/databricks/cli + testifylint: + enable-all: true + disable: + # good check, but we have too many assert.(No)?Errorf? so excluding for now + - require-error issues: exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/ diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 1c3102357..d52088e34 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -2,7 +2,6 @@ package bundle import ( "context" - "errors" "io/fs" "os" "path/filepath" @@ -16,7 +15,7 @@ import ( func TestLoadNotExists(t *testing.T) { b, err := Load(context.Background(), "/doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) assert.Nil(t, b) } diff --git a/bundle/config/mutator/configure_dashboard_defaults_test.go b/bundle/config/mutator/configure_dashboard_defaults_test.go index 2234f9a73..9794d355c 100644 --- a/bundle/config/mutator/configure_dashboard_defaults_test.go +++ b/bundle/config/mutator/configure_dashboard_defaults_test.go @@ -109,19 +109,19 @@ func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) { // Set to true; still true. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, true, v.MustBool()) + assert.True(t, v.MustBool()) } // Set to false; still false. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, false, v.MustBool()) + assert.False(t, v.MustBool()) } // Not set; now false. v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials") if assert.NoError(t, err) { - assert.Equal(t, false, v.MustBool()) + assert.False(t, v.MustBool()) } // No valid dashboard; no change. diff --git a/bundle/config/mutator/default_queueing_test.go b/bundle/config/mutator/default_queueing_test.go index d3621663b..4c521812e 100644 --- a/bundle/config/mutator/default_queueing_test.go +++ b/bundle/config/mutator/default_queueing_test.go @@ -28,8 +28,8 @@ func TestDefaultQueueingApplyNoJobs(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, d) + assert.Empty(t, b.Config.Resources.Jobs) } func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) { @@ -47,7 +47,7 @@ func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled) } @@ -66,7 +66,7 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.NotNil(t, b.Config.Resources.Jobs["job"].Queue) assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled) } @@ -96,7 +96,7 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) { }, } d := bundle.Apply(context.Background(), b, DefaultQueueing()) - assert.Len(t, d, 0) + assert.Empty(t, d) assert.False(t, b.Config.Resources.Jobs["job1"].Queue.Enabled) assert.True(t, b.Config.Resources.Jobs["job2"].Queue.Enabled) assert.True(t, b.Config.Resources.Jobs["job3"].Queue.Enabled) diff --git a/bundle/config/mutator/environments_compat_test.go b/bundle/config/mutator/environments_compat_test.go index 8a2129847..11facf9fb 100644 --- a/bundle/config/mutator/environments_compat_test.go +++ b/bundle/config/mutator/environments_compat_test.go @@ -44,7 +44,7 @@ func TestEnvironmentsToTargetsWithEnvironmentsDefined(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) - assert.Len(t, b.Config.Environments, 0) + assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) } @@ -61,6 +61,6 @@ func TestEnvironmentsToTargetsWithTargetsDefined(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.EnvironmentsToTargets()) require.NoError(t, diags.Error()) - assert.Len(t, b.Config.Environments, 0) + assert.Empty(t, b.Config.Environments) assert.Len(t, b.Config.Targets, 1) } diff --git a/bundle/config/mutator/merge_job_tasks_test.go b/bundle/config/mutator/merge_job_tasks_test.go index a9dae1e10..e6675eefb 100644 --- a/bundle/config/mutator/merge_job_tasks_test.go +++ b/bundle/config/mutator/merge_job_tasks_test.go @@ -74,8 +74,8 @@ func TestMergeJobTasks(t *testing.T) { assert.Equal(t, "i3.2xlarge", cluster.NodeTypeId) assert.Equal(t, 4, cluster.NumWorkers) assert.Len(t, task0.Libraries, 2) - assert.Equal(t, task0.Libraries[0].Whl, "package1") - assert.Equal(t, task0.Libraries[1].Pypi.Package, "package2") + assert.Equal(t, "package1", task0.Libraries[0].Whl) + assert.Equal(t, "package2", task0.Libraries[1].Pypi.Package) // This task was left untouched. task1 := j.Tasks[1].NewCluster diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 14d524416..e1aa9e59b 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -163,18 +163,18 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Job 1 assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) + assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["existing"]) + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job1"].Tags["dev"]) + assert.Equal(t, jobs.PauseStatusPaused, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus) // Job 2 assert.Equal(t, "[dev lennart] job2", b.Config.Resources.Jobs["job2"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job2"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus, jobs.PauseStatusUnpaused) + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) + assert.Equal(t, jobs.PauseStatusUnpaused, b.Config.Resources.Jobs["job2"].Schedule.PauseStatus) // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous) assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) // Experiment 1 diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 7bb6f11a0..07972ecf4 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -185,11 +185,11 @@ func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) { // Apply for the variable prefix. This should resolve the variables to their values. diags = bundle.Apply(context.Background(), b, ResolveVariableReferences("variables")) require.NoError(t, diags.Error()) - assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns) - assert.Equal(t, true, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.True(t, b.Config.Resources.Jobs["job1"].JobSettings.NotificationSettings.NoAlertForSkippedRuns) assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MinWorkers) assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers) - assert.Equal(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice) + assert.InDelta(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice, 0.0001) } func TestResolveComplexVariable(t *testing.T) { diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go index 48973a4cf..099738c02 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix_test.go +++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go @@ -71,7 +71,7 @@ func TestNoWorkspacePrefixUsed(t *testing.T) { } for _, d := range diags { - require.Equal(t, d.Severity, diag.Warning) + require.Equal(t, diag.Warning, d.Severity) require.Contains(t, expectedErrors, d.Summary) delete(expectedErrors, d.Summary) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index d9719793f..07a5c8214 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -30,7 +30,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "process-env") + assert.Equal(t, "process-env", variable.Value) } func TestSetVariableUsingDefaultValue(t *testing.T) { @@ -48,7 +48,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "default") + assert.Equal(t, "default", variable.Value) } func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { @@ -70,7 +70,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "assigned-value") + assert.Equal(t, "assigned-value", variable.Value) } func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { @@ -95,7 +95,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { err = convert.ToTyped(&variable, v) require.NoError(t, err) - assert.Equal(t, variable.Value, "assigned-value") + assert.Equal(t, "assigned-value", variable.Value) } func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 2d05acf3e..3da645585 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -37,11 +37,11 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { field := rt.Field(i) // Fields in Resources are expected be of the form map[string]*resourceStruct - assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name) + assert.Equal(t, reflect.Map, field.Type.Kind(), "Resource %s is not a map", field.Name) kt := field.Type.Key() - assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name) + assert.Equal(t, reflect.String, kt.Kind(), "Resource %s is not a map with string keys", field.Name) vt := field.Type.Elem() - assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name) + assert.Equal(t, reflect.Ptr, vt.Kind(), "Resource %s is not a map with pointer values", field.Name) // Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller // This is because resourceStruct embeds a Go SDK struct that implements diff --git a/bundle/config/validate/files_to_sync_test.go b/bundle/config/validate/files_to_sync_test.go index d6a1ed59a..dd40295c3 100644 --- a/bundle/config/validate/files_to_sync_test.go +++ b/bundle/config/validate/files_to_sync_test.go @@ -87,7 +87,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) { ctx := context.Background() rb := bundle.ReadOnly(b) diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) - require.Equal(t, 1, len(diags)) + require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary) } @@ -101,7 +101,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) { ctx := context.Background() rb := bundle.ReadOnly(b) diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) - require.Equal(t, 1, len(diags)) + require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary) } diff --git a/bundle/config/validate/job_cluster_key_defined_test.go b/bundle/config/validate/job_cluster_key_defined_test.go index 176b0fedc..2cbdb7c6a 100644 --- a/bundle/config/validate/job_cluster_key_defined_test.go +++ b/bundle/config/validate/job_cluster_key_defined_test.go @@ -34,7 +34,7 @@ func TestJobClusterKeyDefined(t *testing.T) { } diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) - require.Len(t, diags, 0) + require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -59,8 +59,8 @@ func TestJobClusterKeyNotDefined(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary) } func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { @@ -92,6 +92,6 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key do-not-exist is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key do-not-exist is not defined", diags[0].Summary) } diff --git a/bundle/context_test.go b/bundle/context_test.go index 3a0f159d9..89a6df052 100644 --- a/bundle/context_test.go +++ b/bundle/context_test.go @@ -12,7 +12,7 @@ func TestGetPanics(t *testing.T) { defer func() { r := recover() require.NotNil(t, r, "The function did not panic") - assert.Equal(t, r, "context not configured with bundle") + assert.Equal(t, "context not configured with bundle", r) }() Get(context.Background()) diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index 36c49fb01..f38b71f6b 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "io" "io/fs" "os" @@ -279,7 +278,7 @@ func TestStatePullNoState(t *testing.T) { require.NoError(t, err) _, err = os.Stat(statePath) - require.True(t, errors.Is(err, fs.ErrNotExist)) + require.ErrorIs(t, err, fs.ErrNotExist) } func TestStatePullOlderState(t *testing.T) { diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index e561f534e..04c5579a8 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -60,7 +60,7 @@ func TestStateUpdate(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(1), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -68,7 +68,7 @@ func TestStateUpdate(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) diags = bundle.Apply(ctx, b, s) @@ -79,7 +79,7 @@ func TestStateUpdate(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(2), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -87,7 +87,7 @@ func TestStateUpdate(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) // Valid non-empty UUID is generated. @@ -130,7 +130,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(11), state.Seq) - require.Equal(t, state.Files, Filelist{ + require.Equal(t, Filelist{ { LocalPath: "test1.py", }, @@ -138,7 +138,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { LocalPath: "test2.py", IsNotebook: true, }, - }) + }, state.Files) require.Equal(t, build.GetInfo().Version, state.CliVersion) // Existing UUID is not overwritten. diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 61f26c088..84b3c5788 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -254,10 +254,10 @@ func TestBundleToTerraformPipeline(t *testing.T) { assert.Equal(t, "my pipeline", resource.Name) assert.Len(t, resource.Library, 2) assert.Len(t, resource.Notification, 2) - assert.Equal(t, resource.Notification[0].Alerts, []string{"on-update-fatal-failure"}) - assert.Equal(t, resource.Notification[0].EmailRecipients, []string{"jane@doe.com"}) - assert.Equal(t, resource.Notification[1].Alerts, []string{"on-update-failure", "on-flow-failure"}) - assert.Equal(t, resource.Notification[1].EmailRecipients, []string{"jane@doe.com", "john@doe.com"}) + assert.Equal(t, []string{"on-update-fatal-failure"}, resource.Notification[0].Alerts) + assert.Equal(t, []string{"jane@doe.com"}, resource.Notification[0].EmailRecipients) + assert.Equal(t, []string{"on-update-failure", "on-flow-failure"}, resource.Notification[1].Alerts) + assert.Equal(t, []string{"jane@doe.com", "john@doe.com"}, resource.Notification[1].EmailRecipients) assert.Nil(t, out.Data) } @@ -454,7 +454,7 @@ func TestBundleToTerraformModelServing(t *testing.T) { assert.Equal(t, "name", resource.Name) assert.Equal(t, "model_name", resource.Config.ServedModels[0].ModelName) assert.Equal(t, "1", resource.Config.ServedModels[0].ModelVersion) - assert.Equal(t, true, resource.Config.ServedModels[0].ScaleToZeroEnabled) + assert.True(t, resource.Config.ServedModels[0].ScaleToZeroEnabled) assert.Equal(t, "Small", resource.Config.ServedModels[0].WorkloadSize) assert.Equal(t, "model_name-1", resource.Config.TrafficConfig.Routes[0].ServedModelName) assert.Equal(t, 100, resource.Config.TrafficConfig.Routes[0].TrafficPercentage) diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index a1ffc5a1a..30ac9e301 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -225,7 +225,7 @@ func TestSetProxyEnvVars(t *testing.T) { env := make(map[string]string, 0) err := setProxyEnvVars(context.Background(), env, b) require.NoError(t, err) - assert.Len(t, env, 0) + assert.Empty(t, env) // Lower case set. clearEnv() @@ -293,7 +293,7 @@ func TestSetUserProfileFromInheritEnvVars(t *testing.T) { require.NoError(t, err) assert.Contains(t, env, "USERPROFILE") - assert.Equal(t, env["USERPROFILE"], "c:\\foo\\c") + assert.Equal(t, "c:\\foo\\c", env["USERPROFILE"]) } func TestInheritEnvVarsWithAbsentTFConfigFile(t *testing.T) { diff --git a/bundle/internal/schema/main_test.go b/bundle/internal/schema/main_test.go index 4eeb41d47..06e89c856 100644 --- a/bundle/internal/schema/main_test.go +++ b/bundle/internal/schema/main_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "fmt" "io" "os" "path" @@ -80,7 +79,7 @@ func TestRequiredAnnotationsForNewFields(t *testing.T) { }, }) assert.NoError(t, err) - assert.Empty(t, updatedFieldPaths, fmt.Sprintf("Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n"))) + assert.Empty(t, updatedFieldPaths, "Missing JSON-schema descriptions for new config fields in bundle/internal/schema/annotations.yml:\n%s", strings.Join(updatedFieldPaths, "\n")) } // Checks whether types in annotation files are still present in Config type diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go index 7b2f5c5ba..2af54b0cb 100644 --- a/bundle/libraries/filer_volume_test.go +++ b/bundle/libraries/filer_volume_test.go @@ -212,12 +212,12 @@ func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) { bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) _, _, diags := GetFilerForLibraries(context.Background(), b) - require.Equal(t, diags, diag.Diagnostics{{ + require.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }}) + }}, diags) } } diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go index 9d7e12ee5..754aa8f95 100644 --- a/bundle/libraries/helpers_test.go +++ b/bundle/libraries/helpers_test.go @@ -12,25 +12,25 @@ func TestLibraryPath(t *testing.T) { p, err := libraryPath(&compute.Library{Whl: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Jar: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Egg: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{Requirements: path}) assert.Equal(t, path, p) - assert.Nil(t, err) + assert.NoError(t, err) p, err = libraryPath(&compute.Library{}) assert.Equal(t, "", p) - assert.NotNil(t, err) + assert.Error(t, err) p, err = libraryPath(&compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: "pypipackage"}}) assert.Equal(t, "", p) - assert.NotNil(t, err) + assert.Error(t, err) } diff --git a/bundle/permissions/filter_test.go b/bundle/permissions/filter_test.go index e6e5a3799..ef7167d75 100644 --- a/bundle/permissions/filter_test.go +++ b/bundle/permissions/filter_test.go @@ -99,32 +99,32 @@ func TestFilterCurrentUser(t *testing.T) { assert.NoError(t, diags.Error()) // Assert current user is filtered out. - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, robot) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions)) + assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions)) + assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions)) + assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, robot) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions)) + assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, robot) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob) // Assert there's no change to the grant. - assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants)) + assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1) } func TestFilterCurrentServicePrincipal(t *testing.T) { @@ -134,32 +134,32 @@ func TestFilterCurrentServicePrincipal(t *testing.T) { assert.NoError(t, diags.Error()) // Assert current user is filtered out. - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job1"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Jobs["job1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Jobs["job2"].Permissions)) + assert.Len(t, b.Config.Resources.Jobs["job2"].Permissions, 2) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, alice) assert.Contains(t, b.Config.Resources.Jobs["job2"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Pipelines["pipeline1"].Permissions)) + assert.Len(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Pipelines["pipeline1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Experiments["experiment1"].Permissions)) + assert.Len(t, b.Config.Resources.Experiments["experiment1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Experiments["experiment1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.Models["model1"].Permissions)) + assert.Len(t, b.Config.Resources.Models["model1"].Permissions, 2) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, alice) assert.Contains(t, b.Config.Resources.Models["model1"].Permissions, bob) - assert.Equal(t, 2, len(b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions)) + assert.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, 2) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, alice) assert.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint1"].Permissions, bob) // Assert there's no change to the grant. - assert.Equal(t, 1, len(b.Config.Resources.RegisteredModels["registered_model1"].Grants)) + assert.Len(t, b.Config.Resources.RegisteredModels["registered_model1"].Grants, 1) } func TestFilterCurrentUserDoesNotErrorWhenNoResources(t *testing.T) { diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go index 78703e90f..15586e979 100644 --- a/bundle/permissions/mutator_test.go +++ b/bundle/permissions/mutator_test.go @@ -164,7 +164,7 @@ func TestAllResourcesExplicitlyDefinedForPermissionsSupport(t *testing.T) { for _, resource := range unsupportedResources { _, ok := levelsMap[resource] - assert.False(t, ok, fmt.Sprintf("Resource %s is defined in both levelsMap and unsupportedResources", resource)) + assert.False(t, ok, "Resource %s is defined in both levelsMap and unsupportedResources", resource) } for _, resource := range r.AllResources() { diff --git a/bundle/permissions/permission_diagnostics_test.go b/bundle/permissions/permission_diagnostics_test.go index 7b0afefa0..6c55ab594 100644 --- a/bundle/permissions/permission_diagnostics_test.go +++ b/bundle/permissions/permission_diagnostics_test.go @@ -28,7 +28,7 @@ func TestPermissionDiagnosticsApplyFail(t *testing.T) { }) diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) - require.Equal(t, diags[0].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[0].Severity) require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions") } diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go index 5d19ca4ff..72aecc887 100644 --- a/bundle/run/job_test.go +++ b/bundle/run/job_test.go @@ -54,7 +54,7 @@ func TestConvertPythonParams(t *testing.T) { err = runner.convertPythonParams(opts) require.NoError(t, err) require.Contains(t, opts.Job.notebookParams, "__python_params") - require.Equal(t, opts.Job.notebookParams["__python_params"], `["param1","param2","param3"]`) + require.Equal(t, `["param1","param2","param3"]`, opts.Job.notebookParams["__python_params"]) } func TestJobRunnerCancel(t *testing.T) { diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index e68823c33..d72b5f157 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -30,7 +30,7 @@ func TestComplexVariables(t *testing.T) { require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"]) require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"]) - require.Equal(t, 3, len(b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries)) + require.Len(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, 3) require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{ Jar: "/path/to/jar", }) diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go index d4695c78d..848b972b1 100644 --- a/bundle/tests/environment_git_test.go +++ b/bundle/tests/environment_git_test.go @@ -2,7 +2,6 @@ package config_tests import ( "context" - "fmt" "strings" "testing" @@ -16,7 +15,7 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) { bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitManuallySetBranchWithEnvironment(t *testing.T) { @@ -25,5 +24,5 @@ func TestGitManuallySetBranchWithEnvironment(t *testing.T) { assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } diff --git a/bundle/tests/environment_overrides_test.go b/bundle/tests/environment_overrides_test.go index 4a1115048..b68b083ff 100644 --- a/bundle/tests/environment_overrides_test.go +++ b/bundle/tests/environment_overrides_test.go @@ -21,8 +21,8 @@ func TestEnvironmentOverridesResourcesDev(t *testing.T) { assert.Equal(t, "base job", b.Config.Resources.Jobs["job1"].Name) // Base values are preserved in the development environment. - assert.Equal(t, true, b.Config.Resources.Pipelines["boolean1"].Photon) - assert.Equal(t, false, b.Config.Resources.Pipelines["boolean2"].Photon) + assert.True(t, b.Config.Resources.Pipelines["boolean1"].Photon) + assert.False(t, b.Config.Resources.Pipelines["boolean2"].Photon) } func TestEnvironmentOverridesResourcesStaging(t *testing.T) { @@ -30,6 +30,6 @@ func TestEnvironmentOverridesResourcesStaging(t *testing.T) { assert.Equal(t, "staging job", b.Config.Resources.Jobs["job1"].Name) // Override values are applied in the staging environment. - assert.Equal(t, false, b.Config.Resources.Pipelines["boolean1"].Photon) - assert.Equal(t, true, b.Config.Resources.Pipelines["boolean2"].Photon) + assert.False(t, b.Config.Resources.Pipelines["boolean1"].Photon) + assert.True(t, b.Config.Resources.Pipelines["boolean2"].Photon) } diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index 218d2e470..423b14c07 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -10,11 +10,11 @@ import ( func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_job_and_pipeline", "development") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_job_and_pipeline", "staging") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go index dec6c268a..41293e450 100644 --- a/bundle/tests/git_test.go +++ b/bundle/tests/git_test.go @@ -2,7 +2,6 @@ package config_tests import ( "context" - "fmt" "strings" "testing" @@ -17,7 +16,7 @@ func TestGitAutoLoad(t *testing.T) { bundle.Apply(context.Background(), b, mutator.LoadGitDetails()) assert.True(t, b.Config.Bundle.Git.Inferred) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitManuallySetBranch(t *testing.T) { @@ -26,7 +25,7 @@ func TestGitManuallySetBranch(t *testing.T) { assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") - assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) + assert.True(t, validUrl, "Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL) } func TestGitBundleBranchValidation(t *testing.T) { diff --git a/bundle/tests/issue_1828_test.go b/bundle/tests/issue_1828_test.go index 5f2becce5..31fcfeb8e 100644 --- a/bundle/tests/issue_1828_test.go +++ b/bundle/tests/issue_1828_test.go @@ -35,7 +35,7 @@ func TestIssue1828(t *testing.T) { } if assert.Contains(t, b.Config.Variables, "float") { - assert.Equal(t, 3.14, b.Config.Variables["float"].Default) + assert.InDelta(t, 3.14, b.Config.Variables["float"].Default, 0.0001) } if assert.Contains(t, b.Config.Variables, "time") { @@ -43,6 +43,6 @@ func TestIssue1828(t *testing.T) { } if assert.Contains(t, b.Config.Variables, "nil") { - assert.Equal(t, nil, b.Config.Variables["nil"].Default) + assert.Nil(t, b.Config.Variables["nil"].Default) } } diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 65aa5bdc4..408e3e3ef 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -10,11 +10,11 @@ import ( func TestJobAndPipelineDevelopment(t *testing.T) { b := loadTarget(t, "./job_and_pipeline", "development") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -23,7 +23,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) { func TestJobAndPipelineStaging(t *testing.T) { b := loadTarget(t, "./job_and_pipeline", "staging") - assert.Len(t, b.Config.Resources.Jobs, 0) + assert.Empty(t, b.Config.Resources.Jobs) assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] diff --git a/bundle/tests/job_cluster_key_test.go b/bundle/tests/job_cluster_key_test.go index 5a8b368e5..6a08da89c 100644 --- a/bundle/tests/job_cluster_key_test.go +++ b/bundle/tests/job_cluster_key_test.go @@ -16,13 +16,13 @@ func TestJobClusterKeyNotDefinedTest(t *testing.T) { diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "job_cluster_key key is not defined") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "job_cluster_key key is not defined", diags[0].Summary) } func TestJobClusterKeyDefinedTest(t *testing.T) { b := loadTarget(t, "./job_cluster_key", "development") diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) - require.Len(t, diags, 0) + require.Empty(t, diags) } diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go index b8b800863..f779a07e6 100644 --- a/bundle/tests/model_serving_endpoint_test.go +++ b/bundle/tests/model_serving_endpoint_test.go @@ -20,7 +20,7 @@ func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) { func TestModelServingEndpointDevelopment(t *testing.T) { b := loadTarget(t, "./model_serving_endpoint", "development") assert.Len(t, b.Config.Resources.ModelServingEndpoints, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.ModelServingEndpoints["my_model_serving_endpoint"] assert.Equal(t, "my-dev-endpoint", p.Name) diff --git a/bundle/tests/override_job_tasks_test.go b/bundle/tests/override_job_tasks_test.go index 82da04da2..85463e17c 100644 --- a/bundle/tests/override_job_tasks_test.go +++ b/bundle/tests/override_job_tasks_test.go @@ -12,14 +12,14 @@ func TestOverrideTasksDev(t *testing.T) { assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2) tasks := b.Config.Resources.Jobs["foo"].Tasks - assert.Equal(t, tasks[0].TaskKey, "key1") - assert.Equal(t, tasks[0].NewCluster.NodeTypeId, "i3.xlarge") - assert.Equal(t, tasks[0].NewCluster.NumWorkers, 1) - assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py") + assert.Equal(t, "key1", tasks[0].TaskKey) + assert.Equal(t, "i3.xlarge", tasks[0].NewCluster.NodeTypeId) + assert.Equal(t, 1, tasks[0].NewCluster.NumWorkers) + assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile) - assert.Equal(t, tasks[1].TaskKey, "key2") - assert.Equal(t, tasks[1].NewCluster.SparkVersion, "13.3.x-scala2.12") - assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test2.py") + assert.Equal(t, "key2", tasks[1].TaskKey) + assert.Equal(t, "13.3.x-scala2.12", tasks[1].NewCluster.SparkVersion) + assert.Equal(t, "./test2.py", tasks[1].SparkPythonTask.PythonFile) } func TestOverrideTasksStaging(t *testing.T) { @@ -28,12 +28,12 @@ func TestOverrideTasksStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Jobs["foo"].Tasks, 2) tasks := b.Config.Resources.Jobs["foo"].Tasks - assert.Equal(t, tasks[0].TaskKey, "key1") - assert.Equal(t, tasks[0].NewCluster.SparkVersion, "13.3.x-scala2.12") - assert.Equal(t, tasks[0].SparkPythonTask.PythonFile, "./test1.py") + assert.Equal(t, "key1", tasks[0].TaskKey) + assert.Equal(t, "13.3.x-scala2.12", tasks[0].NewCluster.SparkVersion) + assert.Equal(t, "./test1.py", tasks[0].SparkPythonTask.PythonFile) - assert.Equal(t, tasks[1].TaskKey, "key2") - assert.Equal(t, tasks[1].NewCluster.NodeTypeId, "i3.2xlarge") - assert.Equal(t, tasks[1].NewCluster.NumWorkers, 4) - assert.Equal(t, tasks[1].SparkPythonTask.PythonFile, "./test3.py") + assert.Equal(t, "key2", tasks[1].TaskKey) + assert.Equal(t, "i3.2xlarge", tasks[1].NewCluster.NodeTypeId) + assert.Equal(t, 4, tasks[1].NewCluster.NumWorkers) + assert.Equal(t, "./test3.py", tasks[1].SparkPythonTask.PythonFile) } diff --git a/bundle/tests/presets_test.go b/bundle/tests/presets_test.go index 5fcb5d95b..c2cbe497b 100644 --- a/bundle/tests/presets_test.go +++ b/bundle/tests/presets_test.go @@ -13,7 +13,7 @@ func TestPresetsDev(t *testing.T) { assert.Equal(t, "myprefix", b.Config.Presets.NamePrefix) assert.Equal(t, config.Paused, b.Config.Presets.TriggerPauseStatus) assert.Equal(t, 10, b.Config.Presets.JobsMaxConcurrentRuns) - assert.Equal(t, true, *b.Config.Presets.PipelinesDevelopment) + assert.True(t, *b.Config.Presets.PipelinesDevelopment) assert.Equal(t, "true", b.Config.Presets.Tags["dev"]) assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) assert.Equal(t, "false", b.Config.Presets.Tags["prod"]) @@ -22,7 +22,7 @@ func TestPresetsDev(t *testing.T) { func TestPresetsProd(t *testing.T) { b := loadTarget(t, "./presets", "prod") - assert.Equal(t, false, *b.Config.Presets.PipelinesDevelopment) + assert.False(t, *b.Config.Presets.PipelinesDevelopment) assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) assert.Equal(t, "true", b.Config.Presets.Tags["prod"]) } diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index c982c09d6..06cb05270 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -23,7 +23,7 @@ func TestPythonWheelBuild(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -39,7 +39,7 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -55,7 +55,7 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -108,7 +108,7 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") require.NoError(t, err) - require.Equal(t, 1, len(matches)) + require.Len(t, matches, 1) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) @@ -124,7 +124,7 @@ func TestPythonWheelBuildMultiple(t *testing.T) { matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") require.NoError(t, err) - require.Equal(t, 2, len(matches)) + require.Len(t, matches, 2) match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go index 9b91052f5..e95c7b7c1 100644 --- a/bundle/tests/quality_monitor_test.go +++ b/bundle/tests/quality_monitor_test.go @@ -19,7 +19,7 @@ func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) { func TestMonitorTableNames(t *testing.T) { b := loadTarget(t, "./quality_monitor", "development") assert.Len(t, b.Config.Resources.QualityMonitors, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.QualityMonitors["my_monitor"] assert.Equal(t, "main.test.dev", p.TableName) diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go index 008db8bdd..e9d572a3a 100644 --- a/bundle/tests/registered_model_test.go +++ b/bundle/tests/registered_model_test.go @@ -19,7 +19,7 @@ func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) { func TestRegisteredModelDevelopment(t *testing.T) { b := loadTarget(t, "./registered_model", "development") assert.Len(t, b.Config.Resources.RegisteredModels, 1) - assert.Equal(t, b.Config.Bundle.Mode, config.Development) + assert.Equal(t, config.Development, b.Config.Bundle.Mode) p := b.Config.Resources.RegisteredModels["my_registered_model"] assert.Equal(t, "my-dev-model", p.Name) diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 0192b61e6..c206e7471 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -20,26 +20,26 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Len(t, diags, 3) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "Pattern dist does not match any files", diags[0].Summary) require.Len(t, diags[0].Paths, 1) - require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]") + require.Equal(t, "sync.exclude[0]", diags[0].Paths[0].String()) assert.Len(t, diags[0].Locations, 1) require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) - require.Equal(t, diags[0].Locations[0].Line, 17) - require.Equal(t, diags[0].Locations[0].Column, 11) + require.Equal(t, 17, diags[0].Locations[0].Line) + require.Equal(t, 11, diags[0].Locations[0].Column) summaries := []string{ fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")), fmt.Sprintf("Pattern %s does not match any files", filepath.Join("tests", "*")), } - require.Equal(t, diags[1].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[1].Severity) require.Contains(t, summaries, diags[1].Summary) - require.Equal(t, diags[2].Severity, diag.Warning) + require.Equal(t, diag.Warning, diags[2].Severity) require.Contains(t, summaries, diags[2].Summary) } @@ -47,7 +47,7 @@ func TestSyncIncludeWithNegate(t *testing.T) { b := loadTarget(t, "./sync/negate", "default") diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) - require.Len(t, diags, 0) + require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -58,6 +58,6 @@ func TestSyncIncludeWithNegateNoMatches(t *testing.T) { require.Len(t, diags, 1) require.NoError(t, diags.Error()) - require.Equal(t, diags[0].Severity, diag.Warning) - require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files") + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "Pattern !*.txt2 does not match any files", diags[0].Summary) } diff --git a/bundle/tests/sync_test.go b/bundle/tests/sync_test.go index 15644b67e..f5a0296a9 100644 --- a/bundle/tests/sync_test.go +++ b/bundle/tests/sync_test.go @@ -115,12 +115,12 @@ func TestSyncPathsNoRoot(t *testing.T) { // If set to nil, it won't sync anything. b = loadTarget(t, "./sync/paths_no_root", "nil") assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) - assert.Len(t, b.Config.Sync.Paths, 0) + assert.Empty(t, b.Config.Sync.Paths) // If set to an empty sequence, it won't sync anything. b = loadTarget(t, "./sync/paths_no_root", "empty") assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) - assert.Len(t, b.Config.Sync.Paths, 0) + assert.Empty(t, b.Config.Sync.Paths) } func TestSyncSharedCode(t *testing.T) { diff --git a/cmd/bundle/generate/dashboard_test.go b/cmd/bundle/generate/dashboard_test.go index f1161950b..33a463ea0 100644 --- a/cmd/bundle/generate/dashboard_test.go +++ b/cmd/bundle/generate/dashboard_test.go @@ -44,7 +44,7 @@ func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) { _, diags := d.resolveID(ctx, b) require.Len(t, diags, 1) - assert.Equal(t, diags[0].Summary, "dashboard \"legacy dashboard\" is a legacy dashboard") + assert.Equal(t, "dashboard \"legacy dashboard\" is a legacy dashboard", diags[0].Summary) } func TestDashboard_ExistingID_Nominal(t *testing.T) { diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index 896b7de51..395d4ebd4 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -3,7 +3,6 @@ package generate import ( "bytes" "context" - "errors" "fmt" "io" "io/fs" @@ -302,7 +301,7 @@ func TestGenerateJobCommandOldFileRename(t *testing.T) { // Make sure file do not exists after the run _, err = os.Stat(oldFilename) - require.True(t, errors.Is(err, fs.ErrNotExist)) + require.ErrorIs(t, err, fs.ErrNotExist) data, err := os.ReadFile(filepath.Join(configDir, "test_job.job.yml")) require.NoError(t, err) diff --git a/cmd/configure/configure_test.go b/cmd/configure/configure_test.go index e2f6c1e29..14eb0674a 100644 --- a/cmd/configure/configure_test.go +++ b/cmd/configure/configure_test.go @@ -148,9 +148,9 @@ func TestEnvVarsConfigureNoInteractive(t *testing.T) { // We should only save host and token for a profile, other env variables should not be saved _, err = defaultSection.GetKey("auth_type") - assert.NotNil(t, err) + assert.Error(t, err) _, err = defaultSection.GetKey("metadata_service_url") - assert.NotNil(t, err) + assert.Error(t, err) } func TestEnvVarsConfigureNoArgsNoInteractive(t *testing.T) { diff --git a/cmd/labs/github/ref_test.go b/cmd/labs/github/ref_test.go index cc27d1e81..9668cd7ec 100644 --- a/cmd/labs/github/ref_test.go +++ b/cmd/labs/github/ref_test.go @@ -7,14 +7,15 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestFileFromRef(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/ucx/main/README.md" { _, err := w.Write([]byte(`abc`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) @@ -34,7 +35,9 @@ func TestDownloadZipball(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/repos/databrickslabs/ucx/zipball/main" { _, err := w.Write([]byte(`abc`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) diff --git a/cmd/labs/github/releases_test.go b/cmd/labs/github/releases_test.go index 9c3d7a959..93ac33aee 100644 --- a/cmd/labs/github/releases_test.go +++ b/cmd/labs/github/releases_test.go @@ -7,14 +7,15 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestLoadsReleasesForCLI(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/repos/databricks/cli/releases" { _, err := w.Write([]byte(`[{"tag_name": "v1.2.3"}, {"tag_name": "v1.2.2"}]`)) - require.NoError(t, err) + if !assert.NoError(t, err) { + return + } return } t.Logf("Requested: %s", r.URL.Path) diff --git a/cmd/labs/github/repositories_test.go b/cmd/labs/github/repositories_test.go index 412b440bc..29ec2ce03 100644 --- a/cmd/labs/github/repositories_test.go +++ b/cmd/labs/github/repositories_test.go @@ -7,14 +7,13 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestRepositories(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/users/databrickslabs/repos" { _, err := w.Write([]byte(`[{"name": "x"}]`)) - require.NoError(t, err) + assert.NoError(t, err) return } t.Logf("Requested: %s", r.URL.Path) @@ -28,5 +27,5 @@ func TestRepositories(t *testing.T) { r := NewRepositoryCache("databrickslabs", t.TempDir()) all, err := r.Load(ctx) assert.NoError(t, err) - assert.True(t, len(all) > 0) + assert.NotEmpty(t, all) } diff --git a/cmd/labs/localcache/jsonfile_test.go b/cmd/labs/localcache/jsonfile_test.go index 0d852174c..9d42c6179 100644 --- a/cmd/labs/localcache/jsonfile_test.go +++ b/cmd/labs/localcache/jsonfile_test.go @@ -22,7 +22,7 @@ func TestCreatesDirectoryIfNeeded(t *testing.T) { } first, err := c.Load(ctx, tick) assert.NoError(t, err) - assert.Equal(t, first, int64(1)) + assert.Equal(t, int64(1), first) } func TestImpossibleToCreateDir(t *testing.T) { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index a69389b31..a01ba864a 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -26,6 +26,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -169,17 +170,17 @@ func TestInstallerWorksForReleases(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/blueprint/v0.3.15/labs.yml" { raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml") - require.NoError(t, err) + assert.NoError(t, err) _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.3.15" { raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib") - require.NoError(t, err) + assert.NoError(t, err) w.Header().Add("Content-Type", "application/octet-stream") _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/api/2.1/clusters/get" { @@ -376,17 +377,17 @@ func TestUpgraderWorksForReleases(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/databrickslabs/blueprint/v0.4.0/labs.yml" { raw, err := os.ReadFile("testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml") - require.NoError(t, err) + assert.NoError(t, err) _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/repos/databrickslabs/blueprint/zipball/v0.4.0" { raw, err := zipballFromFolder("testdata/installed-in-home/.databricks/labs/blueprint/lib") - require.NoError(t, err) + assert.NoError(t, err) w.Header().Add("Content-Type", "application/octet-stream") _, err = w.Write(raw) - require.NoError(t, err) + assert.NoError(t, err) return } if r.URL.Path == "/api/2.1/clusters/get" { diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go index 1b71a1c3d..9dafe2f83 100644 --- a/integration/bundle/artifacts_test.go +++ b/integration/bundle/artifacts_test.go @@ -85,13 +85,13 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The task library path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, ) } @@ -149,13 +149,13 @@ func TestUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The job environment deps path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`)), + path.Join("/Workspace", regexp.QuoteMeta(wsDir), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Environments[0].Spec.Dependencies[0], ) } @@ -218,13 +218,13 @@ func TestUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { // The remote path attribute on the artifact file should have been set. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`), b.Config.Artifacts["test"].Files[0].RemotePath, ) // The task library path should have been updated to the remote path. require.Regexp(t, - regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`), b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, ) } diff --git a/integration/bundle/deploy_test.go b/integration/bundle/deploy_test.go index 0b37e5630..309b82917 100644 --- a/integration/bundle/deploy_test.go +++ b/integration/bundle/deploy_test.go @@ -2,7 +2,6 @@ package bundle_test import ( "context" - "errors" "fmt" "io" "os" @@ -99,7 +98,7 @@ func TestBundleDeployUcSchema(t *testing.T) { // Assert the schema is deleted _, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) apiErr := &apierr.APIError{} - assert.True(t, errors.As(err, &apiErr)) + assert.ErrorAs(t, err, &apiErr) assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode) } diff --git a/integration/bundle/destroy_test.go b/integration/bundle/destroy_test.go index f18138ce5..b69382a58 100644 --- a/integration/bundle/destroy_test.go +++ b/integration/bundle/destroy_test.go @@ -1,7 +1,6 @@ package bundle_test import ( - "errors" "os" "path/filepath" "testing" @@ -71,11 +70,11 @@ func TestBundleDestroy(t *testing.T) { // Assert snapshot file is deleted entries, err = os.ReadDir(snapshotsDir) require.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Assert bundle deployment path is deleted _, err = w.Workspace.GetStatusByPath(ctx, remoteRoot) apiErr := &apierr.APIError{} - assert.True(t, errors.As(err, &apiErr)) + assert.ErrorAs(t, err, &apiErr) assert.Equal(t, "RESOURCE_DOES_NOT_EXIST", apiErr.ErrorCode) } diff --git a/integration/cmd/fs/ls_test.go b/integration/cmd/fs/ls_test.go index 58e776d8a..25929fdf3 100644 --- a/integration/cmd/fs/ls_test.go +++ b/integration/cmd/fs/ls_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "io/fs" "path" - "regexp" "strings" "testing" @@ -65,11 +64,11 @@ func TestFsLs(t *testing.T) { assert.Equal(t, "a", parsedStdout[0]["name"]) assert.Equal(t, true, parsedStdout[0]["is_directory"]) - assert.Equal(t, float64(0), parsedStdout[0]["size"]) + assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001) assert.Equal(t, "bye.txt", parsedStdout[1]["name"]) assert.Equal(t, false, parsedStdout[1]["is_directory"]) - assert.Equal(t, float64(3), parsedStdout[1]["size"]) + assert.InDelta(t, float64(3), parsedStdout[1]["size"], 0.0001) }) } } @@ -99,11 +98,11 @@ func TestFsLsWithAbsolutePaths(t *testing.T) { assert.Equal(t, path.Join(tmpDir, "a"), parsedStdout[0]["name"]) assert.Equal(t, true, parsedStdout[0]["is_directory"]) - assert.Equal(t, float64(0), parsedStdout[0]["size"]) + assert.InDelta(t, float64(0), parsedStdout[0]["size"], 0.0001) assert.Equal(t, path.Join(tmpDir, "bye.txt"), parsedStdout[1]["name"]) assert.Equal(t, false, parsedStdout[1]["is_directory"]) - assert.Equal(t, float64(3), parsedStdout[1]["size"]) + assert.InDelta(t, float64(3), parsedStdout[1]["size"].(float64), 0.0001) }) } } @@ -122,7 +121,7 @@ func TestFsLsOnFile(t *testing.T) { setupLsFiles(t, f) _, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "a", "hello.txt"), "--output=json") - assert.Regexp(t, regexp.MustCompile("not a directory: .*/a/hello.txt"), err.Error()) + assert.Regexp(t, "not a directory: .*/a/hello.txt", err.Error()) assert.ErrorAs(t, err, &filer.NotADirectory{}) }) } @@ -147,7 +146,7 @@ func TestFsLsOnEmptyDir(t *testing.T) { require.NoError(t, err) // assert on ls output - assert.Equal(t, 0, len(parsedStdout)) + assert.Empty(t, parsedStdout) }) } } @@ -166,7 +165,7 @@ func TestFsLsForNonexistingDir(t *testing.T) { _, _, err := testcli.RequireErrorRun(t, ctx, "fs", "ls", path.Join(tmpDir, "nonexistent"), "--output=json") assert.ErrorIs(t, err, fs.ErrNotExist) - assert.Regexp(t, regexp.MustCompile("no such directory: .*/nonexistent"), err.Error()) + assert.Regexp(t, "no such directory: .*/nonexistent", err.Error()) }) } } diff --git a/integration/cmd/fs/mkdir_test.go b/integration/cmd/fs/mkdir_test.go index f332bb526..eff0599a7 100644 --- a/integration/cmd/fs/mkdir_test.go +++ b/integration/cmd/fs/mkdir_test.go @@ -34,7 +34,7 @@ func TestFsMkdir(t *testing.T) { info, err := f.Stat(context.Background(), "a") require.NoError(t, err) assert.Equal(t, "a", info.Name()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) }) } } @@ -60,19 +60,19 @@ func TestFsMkdirCreatesIntermediateDirectories(t *testing.T) { infoA, err := f.Stat(context.Background(), "a") require.NoError(t, err) assert.Equal(t, "a", infoA.Name()) - assert.Equal(t, true, infoA.IsDir()) + assert.True(t, infoA.IsDir()) // assert directory "b" is created infoB, err := f.Stat(context.Background(), "a/b") require.NoError(t, err) assert.Equal(t, "b", infoB.Name()) - assert.Equal(t, true, infoB.IsDir()) + assert.True(t, infoB.IsDir()) // assert directory "c" is created infoC, err := f.Stat(context.Background(), "a/b/c") require.NoError(t, err) assert.Equal(t, "c", infoC.Name()) - assert.Equal(t, true, infoC.IsDir()) + assert.True(t, infoC.IsDir()) }) } } diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index 984f6ac49..6f58b7e42 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -225,7 +225,7 @@ func (a *syncTest) snapshotContains(files []string) { assert.Equal(a.t, s.RemotePath, a.remoteRoot) for _, filePath := range files { _, ok := s.LastModifiedTimes[filePath] - assert.True(a.t, ok, fmt.Sprintf("%s not in snapshot file: %v", filePath, s.LastModifiedTimes)) + assert.True(a.t, ok, "%s not in snapshot file: %v", filePath, s.LastModifiedTimes) } assert.Equal(a.t, len(files), len(s.LastModifiedTimes)) } diff --git a/integration/libs/filer/filer_test.go b/integration/libs/filer/filer_test.go index 766f9817b..21c839e1b 100644 --- a/integration/libs/filer/filer_test.go +++ b/integration/libs/filer/filer_test.go @@ -4,11 +4,9 @@ import ( "bytes" "context" "encoding/json" - "errors" "io" "io/fs" "path" - "regexp" "strings" "testing" @@ -106,7 +104,7 @@ func commonFilerRecursiveDeleteTest(t *testing.T, ctx context.Context, f filer.F for _, e := range entriesBeforeDelete { names = append(names, e.Name()) } - assert.Equal(t, names, []string{"file1", "file2", "subdir1", "subdir2"}) + assert.Equal(t, []string{"file1", "file2", "subdir1", "subdir2"}, names) err = f.Delete(ctx, "dir") assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{}) @@ -149,13 +147,13 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Write should fail because the intermediate directory doesn't exist. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello world`)) - assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{})) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{}) + assert.ErrorIs(t, err, fs.ErrNotExist) // Read should fail because the intermediate directory doesn't yet exist. _, err = f.Read(ctx, "/foo/bar") - assert.True(t, errors.As(err, &filer.FileDoesNotExistError{})) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) + assert.ErrorIs(t, err, fs.ErrNotExist) // Read should fail because the path points to a directory err = f.Mkdir(ctx, "/dir") @@ -170,8 +168,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Write should fail because there is an existing file at the specified path. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`)) - assert.True(t, errors.As(err, &filer.FileAlreadyExistsError{})) - assert.True(t, errors.Is(err, fs.ErrExist)) + assert.ErrorAs(t, err, &filer.FileAlreadyExistsError{}) + assert.ErrorIs(t, err, fs.ErrExist) // Write with OverwriteIfExists should succeed. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello universe`), filer.OverwriteIfExists) @@ -188,7 +186,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) require.NoError(t, err) assert.Equal(t, "foo", info.Name()) assert.True(t, info.Mode().IsDir()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) // Stat on a file should succeed. // Note: size and modification time behave differently between backends. @@ -196,17 +194,17 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) require.NoError(t, err) assert.Equal(t, "bar", info.Name()) assert.True(t, info.Mode().IsRegular()) - assert.Equal(t, false, info.IsDir()) + assert.False(t, info.IsDir()) // Delete should fail if the file doesn't exist. err = f.Delete(ctx, "/doesnt_exist") assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Stat should fail if the file doesn't exist. _, err = f.Stat(ctx, "/doesnt_exist") assert.ErrorAs(t, err, &filer.FileDoesNotExistError{}) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Delete should succeed for file that does exist. err = f.Delete(ctx, "/foo/bar") @@ -215,7 +213,7 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Delete should fail for a non-empty directory. err = f.Delete(ctx, "/foo") assert.ErrorAs(t, err, &filer.DirectoryNotEmptyError{}) - assert.True(t, errors.Is(err, fs.ErrInvalid)) + assert.ErrorIs(t, err, fs.ErrInvalid) // Delete should succeed for a non-empty directory if the DeleteRecursively flag is set. err = f.Delete(ctx, "/foo", filer.DeleteRecursively) @@ -224,8 +222,8 @@ func commonFilerReadWriteTests(t *testing.T, ctx context.Context, f filer.Filer) // Delete of the filer root should ALWAYS fail, otherwise subsequent writes would fail. // It is not in the filer's purview to delete its root directory. err = f.Delete(ctx, "/") - assert.True(t, errors.As(err, &filer.CannotDeleteRootError{})) - assert.True(t, errors.Is(err, fs.ErrInvalid)) + assert.ErrorAs(t, err, &filer.CannotDeleteRootError{}) + assert.ErrorIs(t, err, fs.ErrInvalid) } func TestFilerReadWrite(t *testing.T) { @@ -262,7 +260,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { // We start with an empty directory. entries, err := f.ReadDir(ctx, ".") require.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Write a file. err = f.Write(ctx, "/hello.txt", strings.NewReader(`hello world`)) @@ -282,8 +280,8 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { // Expect an error if the path doesn't exist. _, err = f.ReadDir(ctx, "/dir/a/b/c/d/e") - assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}), err) - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorAs(t, err, &filer.NoSuchDirectoryError{}, err) + assert.ErrorIs(t, err, fs.ErrNotExist) // Expect two entries in the root. entries, err = f.ReadDir(ctx, ".") @@ -295,7 +293,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { assert.False(t, entries[1].IsDir()) info, err = entries[1].Info() require.NoError(t, err) - assert.Greater(t, info.ModTime().Unix(), int64(0)) + assert.Positive(t, info.ModTime().Unix()) // Expect two entries in the directory. entries, err = f.ReadDir(ctx, "/dir") @@ -307,7 +305,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { assert.False(t, entries[1].IsDir()) info, err = entries[1].Info() require.NoError(t, err) - assert.Greater(t, info.ModTime().Unix(), int64(0)) + assert.Positive(t, info.ModTime().Unix()) // Expect a single entry in the nested path. entries, err = f.ReadDir(ctx, "/dir/a/b") @@ -325,7 +323,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { require.NoError(t, err) entries, err = f.ReadDir(ctx, "empty-dir") assert.NoError(t, err) - assert.Len(t, entries, 0) + assert.Empty(t, entries) // Expect one entry for a directory with a file in it err = f.Write(ctx, "dir-with-one-file/my-file.txt", strings.NewReader("abc"), filer.CreateParentDirectories) @@ -333,7 +331,7 @@ func commonFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) { entries, err = f.ReadDir(ctx, "dir-with-one-file") assert.NoError(t, err) assert.Len(t, entries, 1) - assert.Equal(t, entries[0].Name(), "my-file.txt") + assert.Equal(t, "my-file.txt", entries[0].Name()) assert.False(t, entries[0].IsDir()) } @@ -459,7 +457,7 @@ func TestFilerWorkspaceNotebook(t *testing.T) { // Assert uploading a second time fails due to overwrite mode missing err = f.Write(ctx, tc.name, strings.NewReader(tc.content2)) require.ErrorIs(t, err, fs.ErrExist) - assert.Regexp(t, regexp.MustCompile(`file already exists: .*/`+tc.nameWithoutExt+`$`), err.Error()) + assert.Regexp(t, `file already exists: .*/`+tc.nameWithoutExt+`$`, err.Error()) // Try uploading the notebook again with overwrite flag. This time it should succeed. err = f.Write(ctx, tc.name, strings.NewReader(tc.content2), filer.OverwriteIfExists) diff --git a/libs/auth/cache/file_test.go b/libs/auth/cache/file_test.go index 3e4aae36f..54964bed3 100644 --- a/libs/auth/cache/file_test.go +++ b/libs/auth/cache/file_test.go @@ -42,7 +42,7 @@ func TestStoreAndLookup(t *testing.T) { tok, err := l.Lookup("x") require.NoError(t, err) assert.Equal(t, "abc", tok.AccessToken) - assert.Equal(t, 2, len(l.Tokens)) + assert.Len(t, l.Tokens, 2) _, err = l.Lookup("z") assert.Equal(t, ErrNotConfigured, err) diff --git a/libs/databrickscfg/ops_test.go b/libs/databrickscfg/ops_test.go index 3ea92024c..dd8484fb7 100644 --- a/libs/databrickscfg/ops_test.go +++ b/libs/databrickscfg/ops_test.go @@ -216,7 +216,7 @@ func TestSaveToProfile_ClearingPreviousProfile(t *testing.T) { dlft, err := file.GetSection("DEFAULT") assert.NoError(t, err) - assert.Len(t, dlft.KeysHash(), 0) + assert.Empty(t, dlft.KeysHash()) abc, err := file.GetSection("abc") assert.NoError(t, err) diff --git a/libs/databrickscfg/profile/file_test.go b/libs/databrickscfg/profile/file_test.go index 8e5cfefc0..6bcaec4b7 100644 --- a/libs/databrickscfg/profile/file_test.go +++ b/libs/databrickscfg/profile/file_test.go @@ -11,10 +11,10 @@ import ( ) func TestProfileCloud(t *testing.T) { - assert.Equal(t, Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud(), "AWS") - assert.Equal(t, Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud(), "Azure") - assert.Equal(t, Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud(), "GCP") - assert.Equal(t, Profile{Host: "https://some.invalid.host.com/"}.Cloud(), "AWS") + assert.Equal(t, "AWS", Profile{Host: "https://dbc-XXXXXXXX-YYYY.cloud.databricks.com"}.Cloud()) + assert.Equal(t, "Azure", Profile{Host: "https://adb-xxx.y.azuredatabricks.net/"}.Cloud()) + assert.Equal(t, "GCP", Profile{Host: "https://workspace.gcp.databricks.com/"}.Cloud()) + assert.Equal(t, "AWS", Profile{Host: "https://some.invalid.host.com/"}.Cloud()) } func TestProfilesSearchCaseInsensitive(t *testing.T) { diff --git a/libs/errs/aggregate_test.go b/libs/errs/aggregate_test.go index a307e956f..1af57e099 100644 --- a/libs/errs/aggregate_test.go +++ b/libs/errs/aggregate_test.go @@ -1,7 +1,6 @@ package errs import ( - "errors" "fmt" "testing" @@ -14,13 +13,13 @@ func TestFromManyErrors(t *testing.T) { e3 := fmt.Errorf("Error 3") err := FromMany(e1, e2, e3) - assert.True(t, errors.Is(err, e1)) - assert.True(t, errors.Is(err, e2)) - assert.True(t, errors.Is(err, e3)) + assert.ErrorIs(t, err, e1) + assert.ErrorIs(t, err, e2) + assert.ErrorIs(t, err, e3) - assert.Equal(t, err.Error(), `Error 1 + assert.Equal(t, `Error 1 Error 2 -Error 3`) +Error 3`, err.Error()) } func TestFromManyErrorsWihtNil(t *testing.T) { @@ -29,9 +28,9 @@ func TestFromManyErrorsWihtNil(t *testing.T) { e3 := fmt.Errorf("Error 3") err := FromMany(e1, e2, e3) - assert.True(t, errors.Is(err, e1)) - assert.True(t, errors.Is(err, e3)) + assert.ErrorIs(t, err, e1) + assert.ErrorIs(t, err, e3) - assert.Equal(t, err.Error(), `Error 1 -Error 3`) + assert.Equal(t, `Error 1 +Error 3`, err.Error()) } diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go index d284447b9..865d34c2f 100644 --- a/libs/filer/completer/completer_test.go +++ b/libs/filer/completer/completer_test.go @@ -37,7 +37,7 @@ func TestFilerCompleterSetsPrefix(t *testing.T) { assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsNestedDirs(t *testing.T) { @@ -46,7 +46,7 @@ func TestFilerCompleterReturnsNestedDirs(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { @@ -55,7 +55,7 @@ func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { @@ -64,7 +64,7 @@ func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterAddsDbfsPath(t *testing.T) { @@ -78,7 +78,7 @@ func TestFilerCompleterAddsDbfsPath(t *testing.T) { assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterWindowsSeparator(t *testing.T) { @@ -92,7 +92,7 @@ func TestFilerCompleterWindowsSeparator(t *testing.T) { assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions) assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) - assert.Nil(t, err) + assert.NoError(t, err) } func TestFilerCompleterNoCompletions(t *testing.T) { diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 849cf6f7c..08d7a9428 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -63,7 +63,7 @@ func TestFsOpenFile(t *testing.T) { assert.Equal(t, "fileA", info.Name()) assert.Equal(t, int64(3), info.Size()) assert.Equal(t, fs.FileMode(0), info.Mode()) - assert.Equal(t, false, info.IsDir()) + assert.False(t, info.IsDir()) // Read until closed. b := make([]byte, 3) @@ -91,7 +91,7 @@ func TestFsOpenDir(t *testing.T) { info, err := fakeFile.Stat() require.NoError(t, err) assert.Equal(t, "root", info.Name()) - assert.Equal(t, true, info.IsDir()) + assert.True(t, info.IsDir()) de, ok := fakeFile.(fs.ReadDirFile) require.True(t, ok) diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 9eb786db9..0224b2547 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -52,7 +52,7 @@ func TestGlobFileset(t *testing.T) { files, err = g.Files() require.NoError(t, err) - require.Equal(t, len(files), 0) + require.Empty(t, files) } func TestGlobFilesetWithRelativeRoot(t *testing.T) { diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index b31324011..956a3541c 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -123,12 +123,12 @@ func TestJsonUnmarshalForRequest(t *testing.T) { assert.Equal(t, "new job", r.NewSettings.Name) assert.Equal(t, 0, r.NewSettings.TimeoutSeconds) assert.Equal(t, 1, r.NewSettings.MaxConcurrentRuns) - assert.Equal(t, 1, len(r.NewSettings.Tasks)) + assert.Len(t, r.NewSettings.Tasks, 1) assert.Equal(t, "new task", r.NewSettings.Tasks[0].TaskKey) assert.Equal(t, 0, r.NewSettings.Tasks[0].TimeoutSeconds) assert.Equal(t, 0, r.NewSettings.Tasks[0].MaxRetries) assert.Equal(t, 0, r.NewSettings.Tasks[0].MinRetryIntervalMillis) - assert.Equal(t, true, r.NewSettings.Tasks[0].RetryOnTimeout) + assert.True(t, r.NewSettings.Tasks[0].RetryOnTimeout) } const incorrectJsonData = `{ @@ -280,8 +280,8 @@ func TestJsonUnmarshalForRequestWithForceSendFields(t *testing.T) { assert.NoError(t, diags.Error()) assert.Empty(t, diags) - assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) - assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.False(t, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.False(t, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) assert.NotContains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForSkippedRuns") assert.Contains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForCanceledRuns") } diff --git a/libs/folders/folders_test.go b/libs/folders/folders_test.go index 17afc4022..d2afc4f2d 100644 --- a/libs/folders/folders_test.go +++ b/libs/folders/folders_test.go @@ -33,6 +33,6 @@ func TestFindDirWithLeaf(t *testing.T) { { out, err := FindDirWithLeaf(root, "this-leaf-doesnt-exist-anywhere") assert.ErrorIs(t, err, os.ErrNotExist) - assert.Equal(t, out, "") + assert.Equal(t, "", out) } } diff --git a/libs/jsonschema/utils_test.go b/libs/jsonschema/utils_test.go index 89200dae3..954c723d3 100644 --- a/libs/jsonschema/utils_test.go +++ b/libs/jsonschema/utils_test.go @@ -96,7 +96,7 @@ func TestTemplateFromString(t *testing.T) { v, err = fromString("1.1", NumberType) assert.NoError(t, err) // Floating point conversions are not perfect - assert.True(t, (v.(float64)-1.1) < 0.000001) + assert.Less(t, (v.(float64) - 1.1), 0.000001) v, err = fromString("12345", IntegerType) assert.NoError(t, err) @@ -104,7 +104,7 @@ func TestTemplateFromString(t *testing.T) { v, err = fromString("123", NumberType) assert.NoError(t, err) - assert.Equal(t, float64(123), v) + assert.InDelta(t, float64(123), v.(float64), 0.0001) _, err = fromString("qrt", ArrayType) assert.EqualError(t, err, "cannot parse string as object of type array. Value of string: \"qrt\"") diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index 4ede7bf9b..49a67d2d3 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -1,7 +1,6 @@ package notebook import ( - "errors" "io/fs" "os" "path/filepath" @@ -53,7 +52,7 @@ func TestDetectCallsDetectJupyter(t *testing.T) { func TestDetectUnknownExtension(t *testing.T) { _, _, err := Detect("./testdata/doesntexist.foobar") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) nb, _, err := Detect("./testdata/unknown_extension.foobar") require.NoError(t, err) @@ -62,7 +61,7 @@ func TestDetectUnknownExtension(t *testing.T) { func TestDetectNoExtension(t *testing.T) { _, _, err := Detect("./testdata/doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) nb, _, err := Detect("./testdata/no_extension") require.NoError(t, err) diff --git a/libs/process/background_test.go b/libs/process/background_test.go index 2e47e814b..7843375cf 100644 --- a/libs/process/background_test.go +++ b/libs/process/background_test.go @@ -95,7 +95,7 @@ func TestBackgroundNoStdin(t *testing.T) { func TestBackgroundFails(t *testing.T) { ctx := context.Background() _, err := Background(ctx, []string{"ls", "/dev/null/x"}) - assert.NotNil(t, err) + assert.Error(t, err) } func TestBackgroundFailsOnOption(t *testing.T) { diff --git a/libs/process/forwarded_test.go b/libs/process/forwarded_test.go index ddb79818f..71f0a6a63 100644 --- a/libs/process/forwarded_test.go +++ b/libs/process/forwarded_test.go @@ -27,7 +27,7 @@ func TestForwardedFails(t *testing.T) { err := Forwarded(ctx, []string{ "_non_existent_", }, strings.NewReader("abc\n"), &buf, &buf) - assert.NotNil(t, err) + assert.Error(t, err) } func TestForwardedFailsOnStdinPipe(t *testing.T) { @@ -39,5 +39,5 @@ func TestForwardedFailsOnStdinPipe(t *testing.T) { c.Stdin = strings.NewReader("x") return nil }) - assert.NotNil(t, err) + assert.Error(t, err) } diff --git a/libs/process/opts_test.go b/libs/process/opts_test.go index 3a819fbb9..8b5d51928 100644 --- a/libs/process/opts_test.go +++ b/libs/process/opts_test.go @@ -41,7 +41,7 @@ func TestWorksWithLibsEnv(t *testing.T) { vars := cmd.Environ() sort.Strings(vars) - assert.True(t, len(vars) >= 2) + assert.GreaterOrEqual(t, len(vars), 2) assert.Equal(t, "CCC=DDD", vars[0]) assert.Equal(t, "EEE=FFF", vars[1]) } diff --git a/libs/python/interpreters_unix_test.go b/libs/python/interpreters_unix_test.go index 8471644a1..57adc9279 100644 --- a/libs/python/interpreters_unix_test.go +++ b/libs/python/interpreters_unix_test.go @@ -18,7 +18,7 @@ func TestAtLeastOnePythonInstalled(t *testing.T) { assert.NoError(t, err) a := all.Latest() t.Logf("latest is: %s", a) - assert.True(t, len(all) > 0) + assert.NotEmpty(t, all) } func TestNoInterpretersFound(t *testing.T) { diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go index eef526e58..4ba3874ae 100644 --- a/libs/sync/snapshot_test.go +++ b/libs/sync/snapshot_test.go @@ -51,7 +51,7 @@ func TestDiff(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 2) assert.Contains(t, change.put, "hello.txt") assert.Contains(t, change.put, "world.txt") @@ -67,7 +67,7 @@ func TestDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "world.txt") assertKeysOfMap(t, state.LastModifiedTimes, []string{"hello.txt", "world.txt"}) @@ -82,7 +82,7 @@ func TestDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) assert.Len(t, change.delete, 1) - assert.Len(t, change.put, 0) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "hello.txt") assertKeysOfMap(t, state.LastModifiedTimes, []string{"world.txt"}) assert.Equal(t, map[string]string{"world.txt": "world.txt"}, state.LocalToRemoteNames) @@ -145,8 +145,8 @@ func TestFolderDiff(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) - assert.Len(t, change.rmdir, 0) + assert.Empty(t, change.delete) + assert.Empty(t, change.rmdir) assert.Len(t, change.mkdir, 1) assert.Len(t, change.put, 1) assert.Contains(t, change.mkdir, "foo") @@ -159,8 +159,8 @@ func TestFolderDiff(t *testing.T) { assert.NoError(t, err) assert.Len(t, change.delete, 1) assert.Len(t, change.rmdir, 1) - assert.Len(t, change.mkdir, 0) - assert.Len(t, change.put, 0) + assert.Empty(t, change.mkdir) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "foo/bar") assert.Contains(t, change.rmdir, "foo") } @@ -189,7 +189,7 @@ func TestPythonNotebookDiff(t *testing.T) { foo.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "foo.py") assertKeysOfMap(t, state.LastModifiedTimes, []string{"foo.py"}) @@ -233,9 +233,9 @@ func TestPythonNotebookDiff(t *testing.T) { change, err = state.diff(ctx, files) assert.NoError(t, err) assert.Len(t, change.delete, 1) - assert.Len(t, change.put, 0) + assert.Empty(t, change.put) assert.Contains(t, change.delete, "foo") - assert.Len(t, state.LastModifiedTimes, 0) + assert.Empty(t, state.LastModifiedTimes) assert.Equal(t, map[string]string{}, state.LocalToRemoteNames) assert.Equal(t, map[string]string{}, state.RemoteToLocalNames) } @@ -264,7 +264,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 2) assert.Contains(t, change.put, "foo.py") assert.Contains(t, change.put, "foo") @@ -300,7 +300,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) - assert.Len(t, change.delete, 0) + assert.Empty(t, change.delete) assert.Len(t, change.put, 1) assert.Contains(t, change.put, "foo.py") diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index 6168dc217..f30431770 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -59,7 +59,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 10) + require.Len(t, fileList, 10) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -77,7 +77,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 2) + require.Len(t, fileList, 2) inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) @@ -95,7 +95,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 11) + require.Len(t, fileList, 11) } func TestRecursiveExclude(t *testing.T) { @@ -125,7 +125,7 @@ func TestRecursiveExclude(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 7) + require.Len(t, fileList, 7) } func TestNegateExclude(t *testing.T) { @@ -155,6 +155,6 @@ func TestNegateExclude(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 1) - require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt") + require.Len(t, fileList, 1) + require.Equal(t, "test/sub1/sub2/h.txt", fileList[0].Relative) } diff --git a/libs/template/config_test.go b/libs/template/config_test.go index a855019b6..515a0b9f5 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -24,7 +24,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) { err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) if assert.NoError(t, err) { assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) + assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "hello", c.values["string_val"]) } @@ -44,7 +44,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) if assert.NoError(t, err) { assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) + assert.InDelta(t, float64(2), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) } @@ -89,7 +89,7 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { err = c.assignDefaultValues(r) if assert.NoError(t, err) { assert.Equal(t, int64(123), c.values["int_val"]) - assert.Equal(t, float64(123), c.values["float_val"]) + assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "hello", c.values["string_val"]) } @@ -110,7 +110,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { err = c.assignDefaultValues(r) if assert.NoError(t, err) { assert.Equal(t, int64(123), c.values["int_val"]) - assert.Equal(t, float64(123), c.values["float_val"]) + assert.InDelta(t, float64(123), c.values["float_val"].(float64), 0.0001) assert.Equal(t, true, c.values["bool_val"]) assert.Equal(t, "world", c.values["string_val"]) } diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go index d98f40b24..f8bc1f3da 100644 --- a/libs/template/helpers_test.go +++ b/libs/template/helpers_test.go @@ -86,7 +86,7 @@ func TestTemplateRandIntFunction(t *testing.T) { assert.Len(t, r.files, 1) randInt, err := strconv.Atoi(strings.TrimSpace(string(r.files[0].(*inMemoryFile).content))) assert.Less(t, randInt, 10) - assert.Empty(t, err) + assert.NoError(t, err) } func TestTemplateUuidFunction(t *testing.T) { diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index eeb308732..2c14009ff 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -434,7 +434,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) { entries, err := os.ReadDir(tmpDir) require.NoError(t, err) // Assert none of the files are persisted to disk, because of {{skip "*"}} - assert.Len(t, entries, 0) + assert.Empty(t, entries) } func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) { @@ -588,8 +588,8 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) { assert.NoError(t, err) assert.Len(t, r.files, 1) - assert.Equal(t, r.files[0].(*copyFile).srcPath, "not-a-template") - assert.Equal(t, r.files[0].RelPath(), "not-a-template") + assert.Equal(t, "not-a-template", r.files[0].(*copyFile).srcPath) + assert.Equal(t, "not-a-template", r.files[0].RelPath()) } func TestRendererFileTreeRendering(t *testing.T) { @@ -609,7 +609,7 @@ func TestRendererFileTreeRendering(t *testing.T) { // Assert in memory representation is created. assert.Len(t, r.files, 1) - assert.Equal(t, r.files[0].RelPath(), "my_directory/my_file") + assert.Equal(t, "my_directory/my_file", r.files[0].RelPath()) out, err := filer.NewLocalClient(tmpDir) require.NoError(t, err) diff --git a/libs/vfs/filer_test.go b/libs/vfs/filer_test.go index ee1397521..6987c288e 100644 --- a/libs/vfs/filer_test.go +++ b/libs/vfs/filer_test.go @@ -2,7 +2,6 @@ package vfs import ( "context" - "errors" "io/fs" "os" "path/filepath" @@ -42,7 +41,7 @@ func TestFilerPath(t *testing.T) { // Open non-existent file. _, err = p.Open("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Stat self. s, err = p.Stat("filer_test.go") @@ -52,7 +51,7 @@ func TestFilerPath(t *testing.T) { // Stat non-existent file. _, err = p.Stat("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // ReadDir self. entries, err := p.ReadDir(".") @@ -61,7 +60,7 @@ func TestFilerPath(t *testing.T) { // ReadDir non-existent directory. _, err = p.ReadDir("doesntexist") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // ReadFile self. buf, err = p.ReadFile("filer_test.go") @@ -70,7 +69,7 @@ func TestFilerPath(t *testing.T) { // ReadFile non-existent file. _, err = p.ReadFile("doesntexist_test.go") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) // Parent self. pp := p.Parent() From ef86d2bcaeff4d0e0f1445bf0e834299a63eadd7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:06:01 +0100 Subject: [PATCH 12/27] Speed up best case for "make test" 12x (#2060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On main branch: ‘make test’ takes about 33s On this branch: ‘make test’ takes about 2.7s (all measurements are for hot cache) What’s done (from highest impact to lowest): - Remove -coverprofile= option - this option was disabling "go test"'s built-in cache and also it took extra time to calculate the coverage (extra 21s). - Exclude ./integration/ folder, there are no unit tests there, but having it included adds significant time. "go test"'s caching also does not work there for me, due to TestMain() presence (extra 7.2s). - Remove dependency on "make lint" - nice to have, but slow to re-check the whole repo and should already be done by IDE (extra 2.5s). - Remove dependency on "make vendor" — rarely needed; on CI it is already executed separately (extra 1.1s). The coverage option is still available under "make cover". Use "make showcover" to show it. I’ve also removed separate "make testonly". If you only want tests, run "make test". If you want lint+test run "make lint test" etc. I've also modified the test command, removed unnecessary -short, -v, --raw-command. --- .github/workflows/push.yml | 2 +- Makefile | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 25bd5f4d6..f7c4d5456 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -55,7 +55,7 @@ jobs: pip3 install wheel - name: Run tests - run: make testonly + run: make test golangci: name: lint diff --git a/Makefile b/Makefile index 7dca3b2cf..2e7e34299 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,24 @@ default: build -lint: vendor +PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... . + +lint: @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." @./lint.sh ./... -lintcheck: vendor +lintcheck: @echo "✓ Linting source code with https://golangci-lint.run/ ..." @golangci-lint run ./... -test: lint testonly - -testonly: +test: @echo "✓ Running tests ..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped --raw-command go test -v -json -short -coverprofile=coverage.txt ./... + @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} -coverage: test +cover: + @echo "✓ Running tests with coverage..." + @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} + +showcover: @echo "✓ Opening coverage for unit tests ..." @go tool cover -html=coverage.txt @@ -42,4 +46,4 @@ integration: integration-short: $(INTEGRATION) -short -.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short +.PHONY: lint lintcheck test cover showcover build snapshot vendor schema integration integration-short From ea8445af9eea3a6416c5c3705f664a5f8b66d085 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:18:38 +0100 Subject: [PATCH 13/27] Make "make" output the commands it runs (#2066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is useful on CI and locally for debugging and being able to copy-paste command to tweak the options. Removed redundant and imprecise messages like "✓ Running tests ...". --- Makefile | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 2e7e34299..b6a62765f 100644 --- a/Makefile +++ b/Makefile @@ -3,40 +3,31 @@ default: build PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... . lint: - @echo "✓ Linting source code with https://golangci-lint.run/ (with --fix)..." - @./lint.sh ./... + ./lint.sh ./... lintcheck: - @echo "✓ Linting source code with https://golangci-lint.run/ ..." - @golangci-lint run ./... + golangci-lint run ./... test: - @echo "✓ Running tests ..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} + gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} cover: - @echo "✓ Running tests with coverage..." - @gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} + gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} showcover: - @echo "✓ Opening coverage for unit tests ..." - @go tool cover -html=coverage.txt + go tool cover -html=coverage.txt build: vendor - @echo "✓ Building source code with go build ..." - @go build -mod vendor + go build -mod vendor snapshot: - @echo "✓ Building dev snapshot" - @go build -o .databricks/databricks + go build -o .databricks/databricks vendor: - @echo "✓ Filling vendor folder with library code ..." - @go mod vendor + go mod vendor schema: - @echo "✓ Generating json-schema ..." - @go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json + go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h From 890c57eabeb6252a98bbdfe7fe5a061a6284d48f Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:52:33 +0530 Subject: [PATCH 14/27] Enable debugging integration tests in VS Code (#2053) ## Changes This PR adds back debugging functionality that was lost during migration to `internal.Main` as an entry point for integration tests. The PR that caused the regression: https://github.com/databricks/cli/pull/2009. Specifically the addition of internal.Main as the entrypoint for all integration tests. ## Tests Manually, by trying to debug a test. --- integration/internal/acc/debug.go | 4 ++-- integration/internal/main.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/integration/internal/acc/debug.go b/integration/internal/acc/debug.go index b4939881e..08e385b09 100644 --- a/integration/internal/acc/debug.go +++ b/integration/internal/acc/debug.go @@ -11,14 +11,14 @@ import ( ) // Detects if test is run from "debug test" feature in VS Code. -func isInDebug() bool { +func IsInDebug() bool { ex, _ := os.Executable() return strings.HasPrefix(path.Base(ex), "__debug_bin") } // Loads debug environment from ~/.databricks/debug-env.json. func loadDebugEnvIfRunFromIDE(t testutil.TestingT, key string) { - if !isInDebug() { + if !IsInDebug() { return } home, err := os.UserHomeDir() diff --git a/integration/internal/main.go b/integration/internal/main.go index 6d69dcf70..6aa2a4c93 100644 --- a/integration/internal/main.go +++ b/integration/internal/main.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/databricks/cli/integration/internal/acc" ) // Main is the entry point for integration tests. @@ -11,7 +13,7 @@ import ( // they are not inadvertently executed when calling `go test ./...`. func Main(m *testing.M) { value := os.Getenv("CLOUD_ENV") - if value == "" { + if value == "" && !acc.IsInDebug() { fmt.Println("CLOUD_ENV is not set, skipping integration tests") return } From cae21693bbac8b74204a1f36b4c54ec66553de39 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jan 2025 12:23:48 +0100 Subject: [PATCH 15/27] lint: Raise max issues output (#2067) By default it stops after 3 issues of a given type, which gives false impression and also unhelpful if you fixing it with aider. 1000 is almost like unlimited but not unlimited in case there is a bug in a linter. --- .golangci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yaml b/.golangci.yaml index fd83e0882..3e9a88957 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -40,3 +40,5 @@ linters-settings: - require-error issues: exclude-dirs-use-default: false # recommended by docs https://golangci-lint.run/usage/false-positives/ + max-issues-per-linter: 1000 + max-same-issues: 1000 From 509f5aba6ad74765e97e87adf34694967da119f6 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:09:11 +0530 Subject: [PATCH 16/27] Snooze mlops-stacks integration test (#2063) ## Changes https://github.com/databricks/mlops-stacks/pull/187 broke mlops-stacks deployments for non-UC projects. Snoozing the test until upstream is fixed. ## Tests The test is skipped on my local machine. CI run will verify it's also skipped on Github Actions runners. --- integration/bundle/init_test.go | 2 ++ internal/testutil/helpers.go | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go index f5c263ca3..bc81cd3a8 100644 --- a/integration/bundle/init_test.go +++ b/integration/bundle/init_test.go @@ -39,6 +39,8 @@ func TestBundleInitErrorOnUnknownFields(t *testing.T) { // make changes that can break the MLOps Stacks DAB. In which case we should // skip this test until the MLOps Stacks DAB is updated to work again. func TestBundleInitOnMlopsStacks(t *testing.T) { + testutil.SkipUntil(t, "2025-01-09") + ctx, wt := acc.WorkspaceTest(t) w := wt.W diff --git a/internal/testutil/helpers.go b/internal/testutil/helpers.go index 019a8e618..44c2c9375 100644 --- a/internal/testutil/helpers.go +++ b/internal/testutil/helpers.go @@ -5,6 +5,9 @@ import ( "math/rand" "os" "strings" + "time" + + "github.com/stretchr/testify/require" ) // GetEnvOrSkipTest proceeds with test only with that env variable. @@ -30,3 +33,12 @@ func RandomName(prefix ...string) string { } return string(b) } + +func SkipUntil(t TestingT, date string) { + deadline, err := time.Parse(time.DateOnly, date) + require.NoError(t, err) + + if time.Now().Before(deadline) { + t.Skipf("Skipping test until %s. Time right now: %s", deadline.Format(time.DateOnly), time.Now()) + } +} From 7beb0fb8b5bf6f5eb7f11df7b958f713356ca33d Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:23:15 +0530 Subject: [PATCH 17/27] Add validation mutator for volume `artifact_path` (#2050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This PR: 1. Incrementally improves the error messages shown to the user when the volume they are referring to in `workspace.artifact_path` does not exist. 2. Performs this validation in both `bundle validate` and `bundle deploy` compared to before on just deployments. 3. It runs "fast" validations on `bundle deploy`, which earlier were only run on `bundle validate`. ## Tests Unit tests and manually. Also, existing integration tests provide coverage (`TestUploadArtifactToVolumeNotYetDeployed`, `TestUploadArtifactFileToVolumeThatDoesNotExist`) Examples: ``` .venv➜ bundle-playground git:(master) ✗ cli bundle validate Error: cannot access volume capital.whatever.my_volume: User does not have READ VOLUME on Volume 'capital.whatever.my_volume'. at workspace.artifact_path in databricks.yml:7:18 ``` and ``` .venv➜ bundle-playground git:(master) ✗ cli bundle validate Error: volume capital.whatever.foobar does not exist at workspace.artifact_path resources.volumes.foo in databricks.yml:7:18 databricks.yml:12:7 You are using a volume in your artifact_path that is managed by this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path. ``` --- bundle/config/validate/fast_validate.go | 51 ++++ bundle/config/validate/validate.go | 9 +- .../config/validate/validate_artifact_path.go | 129 +++++++++ .../validate/validate_artifact_path_test.go | 244 ++++++++++++++++ bundle/libraries/filer.go | 2 +- bundle/libraries/filer_test.go | 8 - bundle/libraries/filer_volume.go | 124 +-------- bundle/libraries/filer_volume_test.go | 262 +----------------- bundle/libraries/upload_test.go | 7 - cmd/bundle/deploy.go | 2 + integration/bundle/artifacts_test.go | 4 +- 11 files changed, 444 insertions(+), 398 deletions(-) create mode 100644 bundle/config/validate/fast_validate.go create mode 100644 bundle/config/validate/validate_artifact_path.go create mode 100644 bundle/config/validate/validate_artifact_path_test.go diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go new file mode 100644 index 000000000..47d83036d --- /dev/null +++ b/bundle/config/validate/fast_validate.go @@ -0,0 +1,51 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +// FastValidate runs a subset of fast validation checks. This is a subset of the full +// suite of validation mutators that satisfy ANY ONE of the following criteria: +// +// 1. No file i/o or network requests are made in the mutator. +// 2. The validation is blocking for bundle deployments. +// +// The full suite of validation mutators is available in the [Validate] mutator. +type fastValidateReadonly struct{} + +func FastValidateReadonly() bundle.ReadOnlyMutator { + return &fastValidateReadonly{} +} + +func (f *fastValidateReadonly) Name() string { + return "fast_validate(readonly)" +} + +func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + return bundle.ApplyReadOnly(ctx, rb, bundle.Parallel( + // Fast mutators with only in-memory checks + JobClusterKeyDefined(), + JobTaskClusterSpec(), + SingleNodeCluster(), + + // Blocking mutators. Deployments will fail if these checks fail. + ValidateArtifactPath(), + )) +} + +type fastValidate struct{} + +func FastValidate() bundle.Mutator { + return &fastValidate{} +} + +func (f *fastValidate) Name() string { + return "fast_validate" +} + +func (f *fastValidate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), FastValidateReadonly()) +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index 131566fc9..8fdd704ab 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -30,12 +30,13 @@ func (l location) Path() dyn.Path { // Apply implements bundle.Mutator. func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), bundle.Parallel( - JobClusterKeyDefined(), + FastValidateReadonly(), + + // Slow mutators that require network or file i/o. These are only + // run in the `bundle validate` command. FilesToSync(), - ValidateSyncPatterns(), - JobTaskClusterSpec(), ValidateFolderPermissions(), - SingleNodeCluster(), + ValidateSyncPatterns(), )) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go new file mode 100644 index 000000000..5bab99ccf --- /dev/null +++ b/bundle/config/validate/validate_artifact_path.go @@ -0,0 +1,129 @@ +package validate + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/databricks-sdk-go/apierr" +) + +type validateArtifactPath struct{} + +func ValidateArtifactPath() bundle.ReadOnlyMutator { + return &validateArtifactPath{} +} + +func (v *validateArtifactPath) Name() string { + return "validate:artifact_paths" +} + +func extractVolumeFromPath(artifactPath string) (string, string, string, error) { + if !libraries.IsVolumesPath(artifactPath) { + return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath) + } + + parts := strings.Split(artifactPath, "/") + volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes////..., got %s", artifactPath) + + // Incorrect format. + if len(parts) < 5 { + return "", "", "", volumeFormatErr + } + + catalogName := parts[2] + schemaName := parts[3] + volumeName := parts[4] + + // Incorrect format. + if catalogName == "" || schemaName == "" || volumeName == "" { + return "", "", "", volumeFormatErr + } + + return catalogName, schemaName, volumeName, nil +} + +func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) { + volumes := r.Resources.Volumes + for k, v := range volumes { + if v.CatalogName != catalogName || v.Name != volumeName { + continue + } + // UC schemas can be defined in the bundle itself, and thus might be interpolated + // at runtime via the ${resources.schemas.} syntax. Thus we match the volume + // definition if the schema name is the same as the one in the bundle, or if the + // schema name is interpolated. + // We only have to check for ${resources.schemas...} references because any + // other valid reference (like ${var.foo}) would have been interpolated by this point. + p, ok := dynvar.PureReferenceToPath(v.SchemaName) + isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) + if v.SchemaName != schemaName && !isSchemaDefinedInBundle { + continue + } + pathString := fmt.Sprintf("resources.volumes.%s", k) + return dyn.MustPathFromString(pathString), r.GetLocations(pathString), true + } + return nil, nil, false +} + +func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + // We only validate UC Volumes paths right now. + if !libraries.IsVolumesPath(rb.Config().Workspace.ArtifactPath) { + return nil + } + + wrapErrorMsg := func(s string) diag.Diagnostics { + return diag.Diagnostics{ + { + Summary: s, + Severity: diag.Error, + Locations: rb.Config().GetLocations("workspace.artifact_path"), + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }, + } + } + + catalogName, schemaName, volumeName, err := extractVolumeFromPath(rb.Config().Workspace.ArtifactPath) + if err != nil { + return wrapErrorMsg(err.Error()) + } + volumeFullName := fmt.Sprintf("%s.%s.%s", catalogName, schemaName, volumeName) + w := rb.WorkspaceClient() + _, err = w.Volumes.ReadByName(ctx, volumeFullName) + + if errors.Is(err, apierr.ErrPermissionDenied) { + return wrapErrorMsg(fmt.Sprintf("cannot access volume %s: %s", volumeFullName, err)) + } + if errors.Is(err, apierr.ErrNotFound) { + path, locations, ok := findVolumeInBundle(rb.Config(), catalogName, schemaName, volumeName) + if !ok { + return wrapErrorMsg(fmt.Sprintf("volume %s does not exist", volumeFullName)) + } + + // If the volume is defined in the bundle, provide a more helpful error diagnostic, + // with more details and location information. + return diag.Diagnostics{{ + Summary: fmt.Sprintf("volume %s does not exist", volumeFullName), + Severity: diag.Error, + Detail: `You are using a volume in your artifact_path that is managed by +this bundle but which has not been deployed yet. Please first deploy +the volume using 'bundle deploy' and then switch over to using it in +the artifact_path.`, + Locations: slices.Concat(rb.Config().GetLocations("workspace.artifact_path"), locations), + Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), + }} + + } + if err != nil { + return wrapErrorMsg(fmt.Sprintf("cannot read volume %s: %s", volumeFullName, err)) + } + return nil +} diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go new file mode 100644 index 000000000..8fb5c9618 --- /dev/null +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -0,0 +1,244 @@ +package validate + +import ( + "context" + "fmt" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc", + }, + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ + CatalogName: "catalogN", + Name: "volumeN", + SchemaName: "schemaN", + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "file", Line: 2, Column: 2}}) + + ctx := context.Background() + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockVolumesAPI() + api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), ValidateArtifactPath()) + assert.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: "volume catalogN.schemaN.volumeN does not exist", + Locations: []dyn.Location{ + {File: "file", Line: 1, Column: 1}, + {File: "file", Line: 2, Column: 2}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("workspace.artifact_path"), + dyn.MustPathFromString("resources.volumes.foo"), + }, + Detail: `You are using a volume in your artifact_path that is managed by +this bundle but which has not been deployed yet. Please first deploy +the volume using 'bundle deploy' and then switch over to using it in +the artifact_path.`, + }}, diags) +} + +func TestValidateArtifactPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/catalogN/schemaN/volumeN/abc", + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + assertDiags := func(t *testing.T, diags diag.Diagnostics, expected string) { + assert.Len(t, diags, 1) + assert.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: expected, + Locations: []dyn.Location{{File: "file", Line: 1, Column: 1}}, + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }}, diags) + } + + rb := bundle.ReadOnly(b) + ctx := context.Background() + + tcases := []struct { + err error + expectedSummary string + }{ + { + err: &apierr.APIError{ + StatusCode: 403, + Message: "User does not have USE SCHEMA on Schema 'catalogN.schemaN'", + }, + expectedSummary: "cannot access volume catalogN.schemaN.volumeN: User does not have USE SCHEMA on Schema 'catalogN.schemaN'", + }, + { + err: &apierr.APIError{ + StatusCode: 404, + }, + expectedSummary: "volume catalogN.schemaN.volumeN does not exist", + }, + { + err: &apierr.APIError{ + StatusCode: 500, + Message: "Internal Server Error", + }, + expectedSummary: "cannot read volume catalogN.schemaN.volumeN: Internal Server Error", + }, + } + + for _, tc := range tcases { + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockVolumesAPI() + api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.ApplyReadOnly(ctx, rb, ValidateArtifactPath()) + assertDiags(t, diags, tc.expectedSummary) + } +} + +func invalidVolumePaths() []string { + return []string{ + "/Volumes/", + "/Volumes/main", + "/Volumes/main/", + "/Volumes/main//", + "/Volumes/main//my_schema", + "/Volumes/main/my_schema", + "/Volumes/main/my_schema/", + "/Volumes/main/my_schema//", + "/Volumes//my_schema/my_volume", + } +} + +func TestExtractVolumeFromPath(t *testing.T) { + catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume") + require.NoError(t, err) + assert.Equal(t, "main", catalogName) + assert.Equal(t, "my_schema", schemaName) + assert.Equal(t, "my_volume", volumeName) + + for _, p := range invalidVolumePaths() { + _, _, _, err := extractVolumeFromPath(p) + assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p)) + } +} + +func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { + for _, p := range invalidVolumePaths() { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: p, + }, + }, + } + + bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath()) + require.Equal(t, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), + Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, + }}, diags) + } +} + +func TestFindVolumeInBundle(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Volumes: map[string]*resources.Volume{ + "foo": { + CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ + CatalogName: "main", + Name: "my_volume", + SchemaName: "my_schema", + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{ + { + File: "volume.yml", + Line: 1, + Column: 2, + }, + }) + + // volume is in DAB. + path, locations, ok := findVolumeInBundle(b.Config, "main", "my_schema", "my_volume") + assert.True(t, ok) + assert.Equal(t, []dyn.Location{{ + File: "volume.yml", + Line: 1, + Column: 2, + }}, locations) + assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) + + // wrong volume name + _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "doesnotexist") + assert.False(t, ok) + + // wrong schema name + _, _, ok = findVolumeInBundle(b.Config, "main", "doesnotexist", "my_volume") + assert.False(t, ok) + + // wrong catalog name + _, _, ok = findVolumeInBundle(b.Config, "doesnotexist", "my_schema", "my_volume") + assert.False(t, ok) + + // schema name is interpolated but does not have the right prefix. In this case + // we should not match the volume. + b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}" + _, _, ok = findVolumeInBundle(b.Config, "main", "my_schema", "my_volume") + assert.False(t, ok) + + // schema name is interpolated. + b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}" + path, locations, ok = findVolumeInBundle(b.Config, "main", "valuedoesnotmatter", "my_volume") + assert.True(t, ok) + assert.Equal(t, []dyn.Location{{ + File: "volume.yml", + Line: 1, + Column: 2, + }}, locations) + assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) +} diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 4448ed325..e09c75e0e 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -24,7 +24,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s switch { case IsVolumesPath(artifactPath): - return filerForVolume(ctx, b) + return filerForVolume(b) default: return filerForWorkspace(b) diff --git a/bundle/libraries/filer_test.go b/bundle/libraries/filer_test.go index 88ba152fc..c18da9726 100644 --- a/bundle/libraries/filer_test.go +++ b/bundle/libraries/filer_test.go @@ -7,10 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/filer" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -39,11 +36,6 @@ func TestGetFilerForLibrariesValidUcVolume(t *testing.T) { }, } - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - client, uploadPath, diags := GetFilerForLibraries(context.Background(), b) require.NoError(t, diags.Error()) assert.Equal(t, "/Volumes/main/my_schema/my_volume/.internal", uploadPath) diff --git a/bundle/libraries/filer_volume.go b/bundle/libraries/filer_volume.go index aecf68db1..176f475c6 100644 --- a/bundle/libraries/filer_volume.go +++ b/bundle/libraries/filer_volume.go @@ -1,132 +1,16 @@ package libraries import ( - "context" - "errors" - "fmt" "path" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/apierr" ) -func extractVolumeFromPath(artifactPath string) (string, string, string, error) { - if !IsVolumesPath(artifactPath) { - return "", "", "", fmt.Errorf("expected artifact_path to start with /Volumes/, got %s", artifactPath) - } - - parts := strings.Split(artifactPath, "/") - volumeFormatErr := fmt.Errorf("expected UC volume path to be in the format /Volumes////..., got %s", artifactPath) - - // Incorrect format. - if len(parts) < 5 { - return "", "", "", volumeFormatErr - } - - catalogName := parts[2] - schemaName := parts[3] - volumeName := parts[4] - - // Incorrect format. - if catalogName == "" || schemaName == "" || volumeName == "" { - return "", "", "", volumeFormatErr - } - - return catalogName, schemaName, volumeName, nil -} - -// This function returns a filer for ".internal" folder inside the directory configured -// at `workspace.artifact_path`. -// This function also checks if the UC volume exists in the workspace and then: -// 1. If the UC volume exists in the workspace: -// Returns a filer for the UC volume. -// 2. If the UC volume does not exist in the workspace but is (with high confidence) defined in -// the bundle configuration: -// Returns an error and a warning that instructs the user to deploy the -// UC volume before using it in the artifact path. -// 3. If the UC volume does not exist in the workspace and is not defined in the bundle configuration: -// Returns an error. -func filerForVolume(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { - artifactPath := b.Config.Workspace.ArtifactPath +func filerForVolume(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { w := b.WorkspaceClient() - - catalogName, schemaName, volumeName, err := extractVolumeFromPath(artifactPath) - if err != nil { - return nil, "", diag.Diagnostics{ - { - Severity: diag.Error, - Summary: err.Error(), - Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - } - } - - // Check if the UC volume exists in the workspace. - volumePath := fmt.Sprintf("/Volumes/%s/%s/%s", catalogName, schemaName, volumeName) - err = w.Files.GetDirectoryMetadataByDirectoryPath(ctx, volumePath) - - // If the volume exists already, directly return the filer for the path to - // upload the artifacts to. - if err == nil { - uploadPath := path.Join(artifactPath, InternalDirName) - f, err := filer.NewFilesClient(w, uploadPath) - return f, uploadPath, diag.FromErr(err) - } - - baseErr := diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("unable to determine if volume at %s exists: %s", volumePath, err), - Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - } - - if errors.Is(err, apierr.ErrNotFound) { - // Since the API returned a 404, the volume does not exist. - // Modify the error message to provide more context. - baseErr.Summary = fmt.Sprintf("volume %s does not exist: %s", volumePath, err) - - // If the volume is defined in the bundle, provide a more helpful error diagnostic, - // with more details and location information. - path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName) - if !ok { - return nil, "", diag.Diagnostics{baseErr} - } - baseErr.Detail = `You are using a volume in your artifact_path that is managed by -this bundle but which has not been deployed yet. Please first deploy -the volume using 'bundle deploy' and then switch over to using it in -the artifact_path.` - baseErr.Paths = append(baseErr.Paths, path) - baseErr.Locations = append(baseErr.Locations, locations...) - } - - return nil, "", diag.Diagnostics{baseErr} -} - -func findVolumeInBundle(b *bundle.Bundle, catalogName, schemaName, volumeName string) (dyn.Path, []dyn.Location, bool) { - volumes := b.Config.Resources.Volumes - for k, v := range volumes { - if v.CatalogName != catalogName || v.Name != volumeName { - continue - } - // UC schemas can be defined in the bundle itself, and thus might be interpolated - // at runtime via the ${resources.schemas.} syntax. Thus we match the volume - // definition if the schema name is the same as the one in the bundle, or if the - // schema name is interpolated. - // We only have to check for ${resources.schemas...} references because any - // other valid reference (like ${var.foo}) would have been interpolated by this point. - p, ok := dynvar.PureReferenceToPath(v.SchemaName) - isSchemaDefinedInBundle := ok && p.HasPrefix(dyn.Path{dyn.Key("resources"), dyn.Key("schemas")}) - if v.SchemaName != schemaName && !isSchemaDefinedInBundle { - continue - } - pathString := fmt.Sprintf("resources.volumes.%s", k) - return dyn.MustPathFromString(pathString), b.Config.GetLocations(pathString), true - } - return nil, nil, false + uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) + f, err := filer.NewFilesClient(w, uploadPath) + return f, uploadPath, diag.FromErr(err) } diff --git a/bundle/libraries/filer_volume_test.go b/bundle/libraries/filer_volume_test.go index 2af54b0cb..39bdc4135 100644 --- a/bundle/libraries/filer_volume_test.go +++ b/bundle/libraries/filer_volume_test.go @@ -1,277 +1,27 @@ package libraries import ( - "context" - "fmt" "path" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/apierr" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func TestFindVolumeInBundle(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Volumes: map[string]*resources.Volume{ - "foo": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "main", - Name: "my_volume", - SchemaName: "my_schema", - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{ - { - File: "volume.yml", - Line: 1, - Column: 2, - }, - }) - - // volume is in DAB. - path, locations, ok := findVolumeInBundle(b, "main", "my_schema", "my_volume") - assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ - File: "volume.yml", - Line: 1, - Column: 2, - }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) - - // wrong volume name - _, _, ok = findVolumeInBundle(b, "main", "my_schema", "doesnotexist") - assert.False(t, ok) - - // wrong schema name - _, _, ok = findVolumeInBundle(b, "main", "doesnotexist", "my_volume") - assert.False(t, ok) - - // wrong catalog name - _, _, ok = findVolumeInBundle(b, "doesnotexist", "my_schema", "my_volume") - assert.False(t, ok) - - // schema name is interpolated but does not have the right prefix. In this case - // we should not match the volume. - b.Config.Resources.Volumes["foo"].SchemaName = "${foo.bar.baz}" - _, _, ok = findVolumeInBundle(b, "main", "my_schema", "my_volume") - assert.False(t, ok) - - // schema name is interpolated. - b.Config.Resources.Volumes["foo"].SchemaName = "${resources.schemas.my_schema.name}" - path, locations, ok = findVolumeInBundle(b, "main", "valuedoesnotmatter", "my_volume") - assert.True(t, ok) - assert.Equal(t, []dyn.Location{{ - File: "volume.yml", - Line: 1, - Column: 2, - }}, locations) - assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path) -} - -func TestFilerForVolumeForErrorFromAPI(t *testing.T) { +func TestFilerForVolume(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/my_volume", + ArtifactPath: "/Volumes/main/my_schema/my_volume/abc", }, }, } - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(fmt.Errorf("error from API")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := filerForVolume(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "unable to determine if volume at /Volumes/main/my_schema/my_volume exists: error from API", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - }, diags) -} - -func TestFilerForVolumeWithVolumeNotFound(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/doesnotexist", - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(apierr.NotFound("some error message")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := filerForVolume(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "volume /Volumes/main/my_schema/doesnotexist does not exist: some error message", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, - }, diags) -} - -func TestFilerForVolumeNotFoundAndInBundle(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/main/my_schema/my_volume", - }, - Resources: config.Resources{ - Volumes: map[string]*resources.Volume{ - "foo": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "main", - Name: "my_volume", - VolumeType: "MANAGED", - SchemaName: "my_schema", - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - bundletest.SetLocation(b, "resources.volumes.foo", []dyn.Location{{File: "volume.yml", Line: 1, Column: 2}}) - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(apierr.NotFound("error from API")) - b.SetWorkpaceClient(m.WorkspaceClient) - - _, _, diags := GetFilerForLibraries(context.Background(), b) - assert.Equal(t, diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "volume /Volumes/main/my_schema/my_volume does not exist: error from API", - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}, {File: "volume.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")}, - Detail: `You are using a volume in your artifact_path that is managed by -this bundle but which has not been deployed yet. Please first deploy -the volume using 'bundle deploy' and then switch over to using it in -the artifact_path.`, - }, - }, diags) -} - -func invalidVolumePaths() []string { - return []string{ - "/Volumes/", - "/Volumes/main", - "/Volumes/main/", - "/Volumes/main//", - "/Volumes/main//my_schema", - "/Volumes/main/my_schema", - "/Volumes/main/my_schema/", - "/Volumes/main/my_schema//", - "/Volumes//my_schema/my_volume", - } -} - -func TestFilerForVolumeWithInvalidVolumePaths(t *testing.T) { - for _, p := range invalidVolumePaths() { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: p, - }, - }, - } - - bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - - _, _, diags := GetFilerForLibraries(context.Background(), b) - require.Equal(t, diag.Diagnostics{{ - Severity: diag.Error, - Summary: fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p), - Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}, - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }}, diags) - } -} - -func TestFilerForVolumeWithInvalidPrefix(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volume/main/my_schema/my_volume", - }, - }, - } - - _, _, diags := filerForVolume(context.Background(), b) - require.EqualError(t, diags.Error(), "expected artifact_path to start with /Volumes/, got /Volume/main/my_schema/my_volume") -} - -func TestFilerForVolumeWithValidVolumePaths(t *testing.T) { - validPaths := []string{ - "/Volumes/main/my_schema/my_volume", - "/Volumes/main/my_schema/my_volume/", - "/Volumes/main/my_schema/my_volume/a/b/c", - "/Volumes/main/my_schema/my_volume/a/a/a", - } - - for _, p := range validPaths { - b := &bundle.Bundle{ - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: p, - }, - }, - } - - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - client, uploadPath, diags := filerForVolume(context.Background(), b) - require.NoError(t, diags.Error()) - assert.Equal(t, path.Join(p, ".internal"), uploadPath) - assert.IsType(t, &filer.FilesClient{}, client) - } -} - -func TestExtractVolumeFromPath(t *testing.T) { - catalogName, schemaName, volumeName, err := extractVolumeFromPath("/Volumes/main/my_schema/my_volume") - require.NoError(t, err) - assert.Equal(t, "main", catalogName) - assert.Equal(t, "my_schema", schemaName) - assert.Equal(t, "my_volume", volumeName) - - for _, p := range invalidVolumePaths() { - _, _, _, err := extractVolumeFromPath(p) - assert.EqualError(t, err, fmt.Sprintf("expected UC volume path to be in the format /Volumes////..., got %s", p)) - } + client, uploadPath, diags := filerForVolume(b) + require.NoError(t, diags.Error()) + assert.Equal(t, path.Join("/Volumes/main/my_schema/my_volume/abc/.internal"), uploadPath) + assert.IsType(t, &filer.FilesClient{}, client) } diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go index 493785bf5..44b194c56 100644 --- a/bundle/libraries/upload_test.go +++ b/bundle/libraries/upload_test.go @@ -11,8 +11,6 @@ import ( mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/filer" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/mock" @@ -183,11 +181,6 @@ func TestArtifactUploadForVolumes(t *testing.T) { filer.CreateParentDirectories, ).Return(nil) - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &sdkconfig.Config{} - m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/foo/bar/artifacts").Return(nil) - b.SetWorkpaceClient(m.WorkspaceClient) - diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) require.NoError(t, diags.Error()) diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index a25e02f6c..560b07e39 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" @@ -71,6 +72,7 @@ func newDeployCommand() *cobra.Command { diags = diags.Extend( bundle.Apply(ctx, b, bundle.Seq( phases.Initialize(), + validate.FastValidate(), phases.Build(), phases.Deploy(outputHandler), )), diff --git a/integration/bundle/artifacts_test.go b/integration/bundle/artifacts_test.go index 9dafe2f83..94b96899e 100644 --- a/integration/bundle/artifacts_test.go +++ b/integration/bundle/artifacts_test.go @@ -257,7 +257,7 @@ func TestUploadArtifactFileToVolumeThatDoesNotExist(t *testing.T) { stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy") assert.Error(t, err) - assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/doesnotexist does not exist: Not Found + assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.doesnotexist does not exist at workspace.artifact_path in databricks.yml:6:18 @@ -293,7 +293,7 @@ func TestUploadArtifactToVolumeNotYetDeployed(t *testing.T) { stdout, stderr, err := testcli.RequireErrorRun(t, ctx, "bundle", "deploy") assert.Error(t, err) - assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/my_volume does not exist: Not Found + assert.Equal(t, fmt.Sprintf(`Error: volume main.%s.my_volume does not exist at workspace.artifact_path resources.volumes.foo in databricks.yml:6:18 From 60782b57bdbe09af7cfe1232b827b4668ca92bb9 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Jan 2025 14:23:00 +0100 Subject: [PATCH 18/27] Added close stale issues workflow (#634) ## Changes Added workflows for closing stale issues. It adds a Github Action that warns and then auto closes stale issues. --- .github/workflows/close-stale-issues.yml | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/close-stale-issues.yml diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml new file mode 100644 index 000000000..ffe550132 --- /dev/null +++ b/.github/workflows/close-stale-issues.yml @@ -0,0 +1,36 @@ +name: "Close Stale Issues" + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # Run at midnight every day + +jobs: + cleanup: + permissions: + issues: write + contents: read + pull-requests: write + runs-on: ubuntu-latest + name: Stale issue job + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: This issue has not received a response in a while. If you want to keep this issue open, please leave a comment below and auto-close will be canceled. + stale-pr-message: This PR has not received an update in a while. If you want to keep this PR open, please leave a comment below or push a new commit and auto-close will be canceled. + + # These labels are required + stale-issue-label: Stale + stale-pr-label: Stale + + exempt-issue-labels: No Autoclose + exempt-pr-labels: No Autoclose + + # Issue timing + days-before-stale: 30 + days-before-close: 7 + + repo-token: ${{ secrets.GITHUB_TOKEN }} + loglevel: DEBUG + # TODO: Remove dry-run after merge when confirmed it works correctly + dry-run: true From 39d1e8093fcaf359c617c4f5580123a43f7f7ec9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 3 Jan 2025 10:25:07 +0100 Subject: [PATCH 19/27] Enable intrange linter and apply autofix (#2069) New construct in Go1.22+ for integer iteration: https://github.com/ckaznocha/intrange?tab=readme-ov-file#intrange --- .golangci.yaml | 1 + bundle/config/mutator/process_target_mode_test.go | 4 ++-- bundle/config/resources_test.go | 6 +++--- bundle/config/variable/lookup_test.go | 2 +- bundle/deploy/terraform/convert_test.go | 2 +- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/internal/schema/parser.go | 2 +- bundle/run/pipeline.go | 4 ++-- integration/libs/locker/locker_test.go | 12 ++++++------ integration/python/python_tasks_test.go | 6 +++--- libs/cmdio/render.go | 2 +- libs/cmdio/render_test.go | 2 +- libs/dyn/convert/from_typed.go | 2 +- libs/dyn/convert/struct_info.go | 2 +- libs/dyn/mapping_test.go | 2 +- libs/dyn/merge/override.go | 2 +- libs/dyn/path.go | 4 ++-- libs/dyn/visit_map_test.go | 8 ++++---- libs/exec/exec_test.go | 2 +- libs/filer/fs_test.go | 2 +- libs/jsonschema/from_type.go | 4 ++-- 21 files changed, 37 insertions(+), 36 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 3e9a88957..6558c502d 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -12,6 +12,7 @@ linters: - gofumpt - goimports - testifylint + - intrange linters-settings: govet: enable-all: true diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index e1aa9e59b..c299a7636 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -382,7 +382,7 @@ func TestAllResourcesMocked(t *testing.T) { b := mockBundle(config.Development) resources := reflect.ValueOf(b.Config.Resources) - for i := 0; i < resources.NumField(); i++ { + for i := range resources.NumField() { field := resources.Field(i) if field.Kind() == reflect.Map { assert.True( @@ -411,7 +411,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) { require.NoError(t, diags.Error()) resources := reflect.ValueOf(b.Config.Resources) - for i := 0; i < resources.NumField(); i++ { + for i := range resources.NumField() { field := resources.Field(i) if field.Kind() == reflect.Map { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 3da645585..cbbcf5e27 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -33,7 +33,7 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { r := Resources{} rt := reflect.TypeOf(r) - for i := 0; i < rt.NumField(); i++ { + for i := range rt.NumField() { field := rt.Field(i) // Fields in Resources are expected be of the form map[string]*resourceStruct @@ -75,7 +75,7 @@ func TestResourcesAllResourcesCompleteness(t *testing.T) { types = append(types, group.Description.PluralName) } - for i := 0; i < rt.NumField(); i++ { + for i := range rt.NumField() { field := rt.Field(i) jsonTag := field.Tag.Get("json") @@ -92,7 +92,7 @@ func TestSupportedResources(t *testing.T) { actual := SupportedResources() typ := reflect.TypeOf(Resources{}) - for i := 0; i < typ.NumField(); i++ { + for i := range typ.NumField() { field := typ.Field(i) jsonTags := strings.Split(field.Tag.Get("json"), ",") pluralName := jsonTags[0] diff --git a/bundle/config/variable/lookup_test.go b/bundle/config/variable/lookup_test.go index bd54d89fc..bcfcb4626 100644 --- a/bundle/config/variable/lookup_test.go +++ b/bundle/config/variable/lookup_test.go @@ -13,7 +13,7 @@ func TestLookup_Coverage(t *testing.T) { val := reflect.ValueOf(lookup) typ := val.Type() - for i := 0; i < val.NumField(); i++ { + for i := range val.NumField() { field := val.Field(i) if field.Kind() != reflect.String { t.Fatalf("Field %s is not a string", typ.Field(i).Name) diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 84b3c5788..ccfdcece3 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -1261,7 +1261,7 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { func AssertFullResourceCoverage(t *testing.T, config *config.Root) { resources := reflect.ValueOf(config.Resources) - for i := 0; i < resources.NumField(); i++ { + for i := range resources.NumField() { field := resources.Field(i) if field.Kind() == reflect.Map { assert.True( diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index 4cc52b7a7..54e7f621c 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -71,7 +71,7 @@ func TestStatePushLargeState(t *testing.T) { b := statePushTestBundle(t) largeState := map[string]any{} - for i := 0; i < 1000000; i++ { + for i := range 1000000 { largeState[fmt.Sprintf("field_%d", i)] = i } diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index e1d1a13dc..919908429 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -54,7 +54,7 @@ func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { // Check for embedded Databricks Go SDK types. if typ.Kind() == reflect.Struct { - for i := 0; i < typ.NumField(); i++ { + for i := range typ.NumField() { if !typ.Field(i).Anonymous { continue } diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index d84015d76..c447f044a 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -17,7 +17,7 @@ import ( func filterEventsByUpdateId(events []pipelines.PipelineEvent, updateId string) []pipelines.PipelineEvent { result := []pipelines.PipelineEvent{} - for i := 0; i < len(events); i++ { + for i := range events { if events[i].Origin.UpdateId == updateId { result = append(result, events[i]) } @@ -32,7 +32,7 @@ func (r *pipelineRunner) logEvent(ctx context.Context, event pipelines.PipelineE } if event.Error != nil && len(event.Error.Exceptions) > 0 { logString += "trace for most recent exception: \n" - for i := 0; i < len(event.Error.Exceptions); i++ { + for i := range len(event.Error.Exceptions) { logString += fmt.Sprintf("%s\n", event.Error.Exceptions[i].Message) } } diff --git a/integration/libs/locker/locker_test.go b/integration/libs/locker/locker_test.go index c51972b90..524996465 100644 --- a/integration/libs/locker/locker_test.go +++ b/integration/libs/locker/locker_test.go @@ -60,13 +60,13 @@ func TestLock(t *testing.T) { lockerErrs := make([]error, numConcurrentLocks) lockers := make([]*lockpkg.Locker, numConcurrentLocks) - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { lockers[i], err = lockpkg.CreateLocker("humpty.dumpty@databricks.com", remoteProjectRoot, wsc) require.NoError(t, err) } var wg sync.WaitGroup - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { wg.Add(1) currentIndex := i go func() { @@ -80,7 +80,7 @@ func TestLock(t *testing.T) { countActive := 0 indexOfActiveLocker := 0 indexOfAnInactiveLocker := -1 - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { if lockers[i].Active { countActive += 1 assert.NoError(t, lockerErrs[i]) @@ -102,7 +102,7 @@ func TestLock(t *testing.T) { assert.True(t, remoteLocker.AcquisitionTime.Equal(lockers[indexOfActiveLocker].State.AcquisitionTime), "remote locker acquisition time does not match active locker") // test all other locks (inactive ones) do not match the remote lock and Unlock fails - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { if i == indexOfActiveLocker { continue } @@ -112,7 +112,7 @@ func TestLock(t *testing.T) { } // test inactive locks fail to write a file - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { if i == indexOfActiveLocker { continue } @@ -140,7 +140,7 @@ func TestLock(t *testing.T) { assert.Equal(t, "Shah Rukh", res["name"]) // inactive locker file reads fail - for i := 0; i < numConcurrentLocks; i++ { + for i := range numConcurrentLocks { if i == indexOfActiveLocker { continue } diff --git a/integration/python/python_tasks_test.go b/integration/python/python_tasks_test.go index 9ad3ed5de..9411afb13 100644 --- a/integration/python/python_tasks_test.go +++ b/integration/python/python_tasks_test.go @@ -266,7 +266,7 @@ func prepareRepoFiles(t *testing.T) *testFiles { func GenerateNotebookTasks(notebookPath string, versions []string, nodeTypeId string) []jobs.SubmitTask { tasks := make([]jobs.SubmitTask, 0) - for i := 0; i < len(versions); i++ { + for i := range versions { task := jobs.SubmitTask{ TaskKey: fmt.Sprintf("notebook_%s", strings.ReplaceAll(versions[i], ".", "_")), NotebookTask: &jobs.NotebookTask{ @@ -287,7 +287,7 @@ func GenerateNotebookTasks(notebookPath string, versions []string, nodeTypeId st func GenerateSparkPythonTasks(notebookPath string, versions []string, nodeTypeId string) []jobs.SubmitTask { tasks := make([]jobs.SubmitTask, 0) - for i := 0; i < len(versions); i++ { + for i := range versions { task := jobs.SubmitTask{ TaskKey: fmt.Sprintf("spark_%s", strings.ReplaceAll(versions[i], ".", "_")), SparkPythonTask: &jobs.SparkPythonTask{ @@ -308,7 +308,7 @@ func GenerateSparkPythonTasks(notebookPath string, versions []string, nodeTypeId func GenerateWheelTasks(wheelPath string, versions []string, nodeTypeId string) []jobs.SubmitTask { tasks := make([]jobs.SubmitTask, 0) - for i := 0; i < len(versions); i++ { + for i := range versions { task := jobs.SubmitTask{ TaskKey: fmt.Sprintf("whl_%s", strings.ReplaceAll(versions[i], ".", "_")), PythonWheelTask: &jobs.PythonWheelTask{ diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index 1529274a3..1a6aadcfa 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -39,7 +39,7 @@ func Heredoc(tmpl string) (trimmed string) { break } } - for i := 0; i < len(lines); i++ { + for i := range lines { if lines[i] == "" || strings.TrimSpace(lines[i]) == "" { continue } diff --git a/libs/cmdio/render_test.go b/libs/cmdio/render_test.go index f26190a23..1c9d1c93b 100644 --- a/libs/cmdio/render_test.go +++ b/libs/cmdio/render_test.go @@ -55,7 +55,7 @@ func (d *dummyIterator) Next(ctx context.Context) (*provisioning.Workspace, erro func makeWorkspaces(count int) []*provisioning.Workspace { res := make([]*provisioning.Workspace, 0, count) next := []*provisioning.Workspace{&dummyWorkspace1, &dummyWorkspace2} - for i := 0; i < count; i++ { + for range count { n := next[0] next = append(next[1:], n) res = append(res, n) diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index ed1b85a36..3de017b75 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -209,7 +209,7 @@ func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { } out := make([]dyn.Value, src.Len()) - for i := 0; i < src.Len(); i++ { + for i := range src.Len() { v := src.Index(i) refv := ref.Index(i) diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go index f5fd29cb9..1e34008e2 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/dyn/convert/struct_info.go @@ -65,7 +65,7 @@ func buildStructInfo(typ reflect.Type) structInfo { } nf := styp.NumField() - for j := 0; j < nf; j++ { + for j := range nf { sf := styp.Field(j) // Recurse into anonymous fields. diff --git a/libs/dyn/mapping_test.go b/libs/dyn/mapping_test.go index 43b24b0c5..67144ae55 100644 --- a/libs/dyn/mapping_test.go +++ b/libs/dyn/mapping_test.go @@ -185,7 +185,7 @@ func TestMappingClone(t *testing.T) { func TestMappingMerge(t *testing.T) { var m1 dyn.Mapping - for i := 0; i < 10; i++ { + for i := range 10 { err := m1.Set(dyn.V(fmt.Sprintf("%d", i)), dyn.V(i)) require.NoError(t, err) } diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go index ca62c7305..1e49d5544 100644 --- a/libs/dyn/merge/override.go +++ b/libs/dyn/merge/override.go @@ -165,7 +165,7 @@ func overrideSequence(basePath dyn.Path, left, right []dyn.Value, visitor Overri minLen := min(len(left), len(right)) var values []dyn.Value - for i := 0; i < minLen; i++ { + for i := range minLen { path := basePath.Append(dyn.Index(i)) merged, err := override(path, left[i], right[i], visitor) if err != nil { diff --git a/libs/dyn/path.go b/libs/dyn/path.go index 76377e2dc..25bff5070 100644 --- a/libs/dyn/path.go +++ b/libs/dyn/path.go @@ -65,7 +65,7 @@ func (p Path) Equal(q Path) bool { if pl != ql { return false } - for i := 0; i < pl; i++ { + for i := range pl { if p[i] != q[i] { return false } @@ -81,7 +81,7 @@ func (p Path) HasPrefix(q Path) bool { if pl < ql { return false } - for i := 0; i < ql; i++ { + for i := range ql { if p[i] != q[i] { return false } diff --git a/libs/dyn/visit_map_test.go b/libs/dyn/visit_map_test.go index d62327d6f..3c2908c4b 100644 --- a/libs/dyn/visit_map_test.go +++ b/libs/dyn/visit_map_test.go @@ -87,12 +87,12 @@ func TestMapFuncOnMapWithEmptySequence(t *testing.T) { dyn.V([]dyn.Value{dyn.V(42)}), } - for i := 0; i < len(variants); i++ { + for i := range variants { vin := dyn.V(map[string]dyn.Value{ "key": variants[i], }) - for j := 0; j < len(variants); j++ { + for j := range variants { vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Key("key")), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { return variants[j], nil }) @@ -153,12 +153,12 @@ func TestMapFuncOnSequenceWithEmptySequence(t *testing.T) { dyn.V([]dyn.Value{dyn.V(42)}), } - for i := 0; i < len(variants); i++ { + for i := range variants { vin := dyn.V([]dyn.Value{ variants[i], }) - for j := 0; j < len(variants); j++ { + for j := range variants { vout, err := dyn.MapByPath(vin, dyn.NewPath(dyn.Index(0)), func(_ dyn.Path, v dyn.Value) (dyn.Value, error) { return variants[j], nil }) diff --git a/libs/exec/exec_test.go b/libs/exec/exec_test.go index e75c158bd..c363c1f7c 100644 --- a/libs/exec/exec_test.go +++ b/libs/exec/exec_test.go @@ -141,7 +141,7 @@ func TestMultipleCommandsRunInParrallel(t *testing.T) { const count = 5 var wg sync.WaitGroup - for i := 0; i < count; i++ { + for i := range count { wg.Add(1) cmd, err := executor.StartCommand(context.Background(), fmt.Sprintf("echo 'Hello %d'", i)) go func(cmd Command, i int) { diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 08d7a9428..6168af39a 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -107,7 +107,7 @@ func TestFsOpenDir(t *testing.T) { de.Close() - for i := 0; i < 3; i++ { + for range 3 { tmp, err = de.ReadDir(1) require.NoError(t, err) entries = append(entries, tmp...) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 18a2b3ba5..6f8f39d96 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -211,7 +211,7 @@ func getStructFields(typ reflect.Type) []reflect.StructField { fields := []reflect.StructField{} bfsQueue := list.New() - for i := 0; i < typ.NumField(); i++ { + for i := range typ.NumField() { bfsQueue.PushBack(typ.Field(i)) } for bfsQueue.Len() > 0 { @@ -233,7 +233,7 @@ func getStructFields(typ reflect.Type) []reflect.StructField { fieldType = fieldType.Elem() } - for i := 0; i < fieldType.NumField(); i++ { + for i := range fieldType.NumField() { bfsQueue.PushBack(fieldType.Field(i)) } } From 8e8399da8312c127bcfc3760efafc60d1a501214 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 3 Jan 2025 11:13:12 +0100 Subject: [PATCH 20/27] Enable linter 'mirror' and autofix existing issues (#2070) https://github.com/butuzov/mirror --- .golangci.yaml | 1 + bundle/deploy/terraform/import.go | 3 ++- libs/cmdio/render_test.go | 2 +- libs/filer/workspace_files_client.go | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 6558c502d..06d8152e5 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -13,6 +13,7 @@ linters: - goimports - testifylint - intrange + - mirror linters-settings: govet: enable-all: true diff --git a/bundle/deploy/terraform/import.go b/bundle/deploy/terraform/import.go index 0a1d1b9ce..a0604e71d 100644 --- a/bundle/deploy/terraform/import.go +++ b/bundle/deploy/terraform/import.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cmdio" @@ -67,7 +68,7 @@ func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn if changed && !m.opts.AutoApprove { output := buf.String() // Remove output starting from Warning until end of output - output = output[:bytes.Index([]byte(output), []byte("Warning:"))] + output = output[:strings.Index(output, "Warning:")] cmdio.LogString(ctx, output) if !cmdio.IsPromptSupported(ctx) { diff --git a/libs/cmdio/render_test.go b/libs/cmdio/render_test.go index 1c9d1c93b..51b385b1d 100644 --- a/libs/cmdio/render_test.go +++ b/libs/cmdio/render_test.go @@ -74,7 +74,7 @@ func makeIterator(count int) listing.Iterator[*provisioning.Workspace] { func makeBigOutput(count int) string { res := bytes.Buffer{} for _, ws := range makeWorkspaces(count) { - res.Write([]byte(fmt.Sprintf("%d %s\n", ws.WorkspaceId, ws.WorkspaceName))) + res.WriteString(fmt.Sprintf("%d %s\n", ws.WorkspaceId, ws.WorkspaceName)) } return res.String() } diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 9e0a7ce50..8d5148edd 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -195,7 +195,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io // This API returns 400 if the file already exists, when the object type is notebook regex := regexp.MustCompile(`Path \((.*)\) already exists.`) - if aerr.StatusCode == http.StatusBadRequest && regex.Match([]byte(aerr.Message)) { + if aerr.StatusCode == http.StatusBadRequest && regex.MatchString(aerr.Message) { // Parse file path from regex capture group matches := regex.FindStringSubmatch(aerr.Message) if len(matches) == 2 { From 9abb11decbc9e2293d5e4b3c6a4eabb7ffeca21c Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:05:20 +0530 Subject: [PATCH 21/27] Enable TestBundleInitOnMlopsStacks (#2072) ## Changes This test was broken due to upstream change: https://github.com/databricks/mlops-stacks/pull/187 Fixed in upstream change: https://github.com/databricks/mlops-stacks/pull/188 ## Tests Test passes now --- integration/bundle/init_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go index bc81cd3a8..f5c263ca3 100644 --- a/integration/bundle/init_test.go +++ b/integration/bundle/init_test.go @@ -39,8 +39,6 @@ func TestBundleInitErrorOnUnknownFields(t *testing.T) { // make changes that can break the MLOps Stacks DAB. In which case we should // skip this test until the MLOps Stacks DAB is updated to work again. func TestBundleInitOnMlopsStacks(t *testing.T) { - testutil.SkipUntil(t, "2025-01-09") - ctx, wt := acc.WorkspaceTest(t) w := wt.W From ab6b1f1d772fa84adc6fd8cab954fbfd5f38a1ba Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 3 Jan 2025 12:32:03 +0100 Subject: [PATCH 22/27] Upgrade golang.org/x/net from v0.26.0 to v0.33.0 (#2073) Fixes https://github.com/databricks/cli/security/dependabot/20 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2dda0cd60..86bc1c368 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect diff --git a/go.sum b/go.sum index 1e806ea03..f6cf79607 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= From 4f3cf6ac2cad002ac3e6d24dbda657b4c22fb314 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 3 Jan 2025 15:12:55 +0100 Subject: [PATCH 23/27] Limit comment about integration tests to PRs from forks (#2075) ## Changes The comment block appears on all PRs, even if the integration tests are automatically triggered. This is quite noisy. This change limits those comments to PRs from forks. ## Tests Have to try by merging... --- .github/workflows/external-message.yml | 6 +++++- .github/workflows/integration-pr.yml | 25 +++---------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index 1970735f9..28e61503c 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -17,6 +17,10 @@ jobs: permissions: pull-requests: write + # Only run this job for PRs from forks. + # Integration tests are not run automatically for PRs from forks. + if: "${{ github.event.pull_request.head.repo.fork }}" + steps: - uses: actions/checkout@v4 @@ -43,7 +47,7 @@ jobs: run: | gh pr comment ${{ github.event.pull_request.number }} --body \ " - If integration tests don't run automatically, an authorized user can run them manually by following the instructions below: + An authorized user can trigger integration tests manually by following the instructions below: Trigger: [go/deco-tests-run/cli](https://go/deco-tests-run/cli) diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index bf2dcd8bc..c1e3a9a29 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -5,36 +5,17 @@ on: types: [opened, synchronize] jobs: - check-token: - runs-on: ubuntu-latest - environment: "test-trigger-is" - - outputs: - has_token: ${{ steps.set-token-status.outputs.has_token }} - - steps: - - name: Check if DECO_WORKFLOW_TRIGGER_APP_ID is set - id: set-token-status - run: | - if [ -z "${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}" ]; then - echo "DECO_WORKFLOW_TRIGGER_APP_ID is empty. User has no access to secrets." - echo "::set-output name=has_token::false" - else - echo "DECO_WORKFLOW_TRIGGER_APP_ID is set. User has access to secrets." - echo "::set-output name=has_token::true" - fi - # Trigger for pull requests. # # This workflow triggers the integration test workflow in a different repository. # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users. - # It depends on the "check-token" workflow to confirm access to this environment to avoid failures. trigger: runs-on: ubuntu-latest environment: "test-trigger-is" - if: needs.check-token.outputs.has_token == 'true' - needs: check-token + # Only run this job for PRs from branches on the main repository and not from forks. + # Workflows triggered by PRs from forks don't have access to the "test-trigger-is" environment. + if: "${{ !github.event.pull_request.head.repo.fork }}" steps: - name: Generate GitHub App Token From 8af98accee855eb336a8f658d6d63e6de278e132 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 6 Jan 2025 14:12:34 +0100 Subject: [PATCH 24/27] Set gotestsum --format to github-actions when running on github (#2082) This gives grouping and ability to open individual test logs. --- .github/workflows/push.yml | 3 +++ Makefile | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f7c4d5456..a4a35420a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -14,6 +14,9 @@ on: branches: - main +env: + GOTESTSUM_FORMAT: github-actions + jobs: tests: runs-on: ${{ matrix.os }} diff --git a/Makefile b/Makefile index b6a62765f..f8b725900 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ default: build PACKAGES=./libs/... ./internal/... ./cmd/... ./bundle/... . +GOTESTSUM_FORMAT ?= pkgname-and-test-fails + lint: ./lint.sh ./... @@ -9,10 +11,10 @@ lintcheck: golangci-lint run ./... test: - gotestsum --format pkgname-and-test-fails --no-summary=skipped -- ${PACKAGES} + gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- ${PACKAGES} cover: - gotestsum --format pkgname-and-test-fails --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} + gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped -- -coverprofile=coverage.txt ${PACKAGES} showcover: go tool cover -html=coverage.txt From 31552852ff6856a70f54454322d7407f6fdcfb06 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 6 Jan 2025 15:30:48 +0100 Subject: [PATCH 25/27] Disable integration workflows (#2085) See https://github.com/databricks/cli/issues/2084. --- .github/workflows/external-message.yml | 12 ++++++++---- .github/workflows/integration-approve.yml | 6 +++++- .github/workflows/integration-main.yml | 10 +++++++--- .github/workflows/integration-pr.yml | 8 ++++++-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index 28e61503c..9c91242a7 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -6,10 +6,14 @@ name: PR Comment # DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR. on: - pull_request_target: - types: [opened, reopened, synchronize] - branches: - - main + workflow_dispatch: + + # Disable because of https://github.com/databricks/cli/issues/2084. + # + # pull_request_target: + # types: [opened, reopened, synchronize] + # branches: + # - main jobs: comment-on-pr: diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml index 4bdeb62a3..265574bb5 100644 --- a/.github/workflows/integration-approve.yml +++ b/.github/workflows/integration-approve.yml @@ -1,7 +1,11 @@ name: integration-approve on: - merge_group: + workflow_dispatch: + + # Disable because of https://github.com/databricks/cli/issues/2084. + # + # merge_group: jobs: # Trigger for merge groups. diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index 064e439cf..55fc98ae4 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -1,9 +1,13 @@ name: integration-main on: - push: - branches: - - main + workflow_dispatch: + + # Disable because of https://github.com/databricks/cli/issues/2084. + # + # push: + # branches: + # - main jobs: # Trigger for pushes to the main branch. diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index c1e3a9a29..965d74fbe 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -1,8 +1,12 @@ name: integration-pr on: - pull_request: - types: [opened, synchronize] + workflow_dispatch: + + # Disable because of https://github.com/databricks/cli/issues/2084. + # + # pull_request: + # types: [opened, synchronize] jobs: # Trigger for pull requests. From c262b30ef4a28685ef5e52c9d4985d0fe0591382 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 6 Jan 2025 16:34:42 +0100 Subject: [PATCH 26/27] Migrate workflows that need write access to use hosted runners (#2077) ## Changes Migrate workflows to Databricks-hosted GitHub Actions runners. The GitHub-hosted runners can no longer be used because of security hardening. --- .github/workflows/close-stale-issues.yml | 8 ++++++-- .github/workflows/external-message.yml | 5 ++++- .github/workflows/integration-approve.yml | 4 +++- .github/workflows/integration-main.yml | 5 ++++- .github/workflows/integration-pr.yml | 5 ++++- .github/workflows/release-snapshot.yml | 5 ++++- .github/workflows/release.yml | 6 +++++- 7 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml index ffe550132..273b89a9c 100644 --- a/.github/workflows/close-stale-issues.yml +++ b/.github/workflows/close-stale-issues.yml @@ -7,12 +7,16 @@ on: jobs: cleanup: + name: Stale issue job + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + permissions: issues: write contents: read pull-requests: write - runs-on: ubuntu-latest - name: Stale issue job + steps: - uses: actions/stale@v9 with: diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index 9c91242a7..eb68a36e4 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -17,7 +17,10 @@ on: jobs: comment-on-pr: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + permissions: pull-requests: write diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml index 265574bb5..0f6b209cb 100644 --- a/.github/workflows/integration-approve.yml +++ b/.github/workflows/integration-approve.yml @@ -21,7 +21,9 @@ jobs: # * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing. # trigger: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco steps: - name: Auto-approve squashed commit diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index 55fc98ae4..5a78d4fd8 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -15,7 +15,10 @@ jobs: # This workflow triggers the integration test workflow in a different repository. # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users. trigger: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + environment: "test-trigger-is" steps: diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index 965d74fbe..fd170f77e 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -14,7 +14,10 @@ jobs: # This workflow triggers the integration test workflow in a different repository. # It requires secrets from the "test-trigger-is" environment, which are only available to authorized users. trigger: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + environment: "test-trigger-is" # Only run this job for PRs from branches on the main repository and not from forks. diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index 7ef8b43c9..5c56a294e 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -20,7 +20,10 @@ on: jobs: goreleaser: - runs-on: ubuntu-latest + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + steps: - name: Checkout repository and submodules uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4a253531..88e338a8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,13 @@ on: jobs: goreleaser: + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + outputs: artifacts: ${{ steps.releaser.outputs.artifacts }} - runs-on: ubuntu-latest + steps: - name: Checkout repository and submodules uses: actions/checkout@v4 From 3629c9e4062f0de58c7c8ce9524e050621be3cb7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 6 Jan 2025 17:07:15 +0100 Subject: [PATCH 27/27] Enable integration workflows (#2086) ## Changes This reverts commit 31552852ff6856a70f54454322d7407f6fdcfb06. These workflows were disabled in #2085. They should work again now that we're using self-hosted runners (see #2077). ## Tests (inline) --- .github/workflows/external-message.yml | 12 ++++-------- .github/workflows/integration-approve.yml | 6 +----- .github/workflows/integration-main.yml | 10 +++------- .github/workflows/integration-pr.yml | 8 ++------ 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml index eb68a36e4..f06d81a47 100644 --- a/.github/workflows/external-message.yml +++ b/.github/workflows/external-message.yml @@ -6,14 +6,10 @@ name: PR Comment # DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR. on: - workflow_dispatch: - - # Disable because of https://github.com/databricks/cli/issues/2084. - # - # pull_request_target: - # types: [opened, reopened, synchronize] - # branches: - # - main + pull_request_target: + types: [opened, reopened, synchronize] + branches: + - main jobs: comment-on-pr: diff --git a/.github/workflows/integration-approve.yml b/.github/workflows/integration-approve.yml index 0f6b209cb..293d31a2a 100644 --- a/.github/workflows/integration-approve.yml +++ b/.github/workflows/integration-approve.yml @@ -1,11 +1,7 @@ name: integration-approve on: - workflow_dispatch: - - # Disable because of https://github.com/databricks/cli/issues/2084. - # - # merge_group: + merge_group: jobs: # Trigger for merge groups. diff --git a/.github/workflows/integration-main.yml b/.github/workflows/integration-main.yml index 5a78d4fd8..0b6032d50 100644 --- a/.github/workflows/integration-main.yml +++ b/.github/workflows/integration-main.yml @@ -1,13 +1,9 @@ name: integration-main on: - workflow_dispatch: - - # Disable because of https://github.com/databricks/cli/issues/2084. - # - # push: - # branches: - # - main + push: + branches: + - main jobs: # Trigger for pushes to the main branch. diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml index fd170f77e..0f9c4797a 100644 --- a/.github/workflows/integration-pr.yml +++ b/.github/workflows/integration-pr.yml @@ -1,12 +1,8 @@ name: integration-pr on: - workflow_dispatch: - - # Disable because of https://github.com/databricks/cli/issues/2084. - # - # pull_request: - # types: [opened, synchronize] + pull_request: + types: [opened, synchronize] jobs: # Trigger for pull requests.