From 27caf413f2d601de8d81d6cc3fb2614ff1bf354c Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 5 Feb 2025 14:24:15 +0100 Subject: [PATCH 01/21] Add support for extras to the labs CLI (#2288) ## Changes Added support for extras / optional Python dependencies in the labs CLI. Added new `extras` field under install. Example: ```yaml install: script: install.py extras: cli ``` Resolves: #2257 ## Tests Manual test --- cmd/labs/project/installer.go | 5 +++++ cmd/labs/project/schema.json | 5 +++++ .../.databricks/labs/blueprint/lib/labs.yml | 1 + 3 files changed, 11 insertions(+) diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 05f7d68aa..2e42ce43d 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -32,6 +32,7 @@ type hook struct { RequireDatabricksConnect bool `yaml:"require_databricks_connect,omitempty"` MinRuntimeVersion string `yaml:"min_runtime_version,omitempty"` WarehouseTypes whTypes `yaml:"warehouse_types,omitempty"` + Extras string `yaml:"extras,omitempty"` } func (h *hook) RequireRunningCluster() bool { @@ -258,6 +259,10 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr } } feedback <- "Installing Python library dependencies" + if i.Installer.Extras != "" { + // install main and optional dependencies + return i.installPythonDependencies(ctx, fmt.Sprintf(".[%s]", i.Installer.Extras)) + } return i.installPythonDependencies(ctx, ".") } diff --git a/cmd/labs/project/schema.json b/cmd/labs/project/schema.json index a779b15e4..7aa65813c 100644 --- a/cmd/labs/project/schema.json +++ b/cmd/labs/project/schema.json @@ -42,6 +42,11 @@ }, "warehouse_types": { "enum": [ "PRO", "CLASSIC", "TYPE_UNSPECIFIED" ] + }, + "extras": { + "type": "string", + "pattern": "^([^,]+)(,([^,]+))*$", + "default": "" } } }, diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml index 0ac4bf826..b8a0e695e 100644 --- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml +++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml @@ -8,6 +8,7 @@ install: warehouse_types: - PRO script: install.py + extras: "" entrypoint: main.py min_python: 3.9 commands: From e0903fbd3712c252874f193076a88d6a83cd966e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 5 Feb 2025 15:58:29 +0100 Subject: [PATCH 02/21] Include 'go mod tidy' into 'make' and 'make tidy' (#2298) Apparently, it's not part of golangci-lint, so you can send PRs that fail this check on CI. --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e18727934..7da7e4789 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -default: vendor fmt lint +default: vendor fmt lint tidy PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... . @@ -9,6 +9,10 @@ GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped lint: golangci-lint run --fix +tidy: + # not part of golangci-lint, apparently + go mod tidy + lintcheck: golangci-lint run ./... @@ -59,4 +63,4 @@ integration: vendor integration-short: vendor $(INTEGRATION) -short -.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs +.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs From 27eb0c40725aa978191ce3fe43eb07993fd6cfe4 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 6 Feb 2025 20:27:55 +0100 Subject: [PATCH 03/21] Allow 'any' examples in JSON schema (#2289) ## Changes 1. Allow `any` examples in json-schema type since we have many of them in open api spec 2. Fix issue with missing overrides annotations when re-generating the schema ## Tests --- bundle/docsgen/main.go | 2 +- bundle/docsgen/nodes.go | 4 +- bundle/docsgen/refs.go | 12 +- .../schema/annotations_openapi_overrides.yml | 161 +++++++++--------- bundle/internal/schema/parser.go | 10 +- libs/jsonschema/schema.go | 2 +- 6 files changed, 101 insertions(+), 90 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index ad737feea..84bf4779f 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.MarkdownDescription = a.MarkdownDescription } if a.MarkdownExamples != "" { - s.Examples = []any{a.MarkdownExamples} + s.Examples = []string{a.MarkdownExamples} } } diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go index 68ed86450..61d2c21cc 100644 --- a/bundle/docsgen/nodes.go +++ b/bundle/docsgen/nodes.go @@ -220,9 +220,9 @@ func isCycleField(field string) bool { } func getExample(v *jsonschema.Schema) string { - examples := v.Examples + examples := getExamples(v.Examples) if len(examples) == 0 { return "" } - return examples[0].(string) + return examples[0] } diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go index ca45e6ab2..7a4451129 100644 --- a/bundle/docsgen/refs.go +++ b/bundle/docsgen/refs.go @@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j node := s description := s.Description markdownDescription := s.MarkdownDescription - examples := s.Examples + examples := getExamples(s.Examples) for node.Reference != nil { ref := getRefType(node) @@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j markdownDescription = newNode.MarkdownDescription } if len(examples) == 0 { - examples = newNode.Examples + examples = getExamples(newNode.Examples) } node = newNode @@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j return &newNode } +func getExamples(examples any) []string { + typedExamples, ok := examples.([]string) + if !ok { + return []string{} + } + return typedExamples +} + func getRefType(node *jsonschema.Schema) string { if node.Reference == nil { return "" diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 912a4fda0..585886313 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). - "markdown_examples": |- The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: @@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). - "embed_credentials": "description": |- PLACEHOLDER @@ -356,7 +354,6 @@ github.com/databricks/cli/bundle/config/resources.Volume: - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). - "markdown_examples": |- The following example creates a volume with the key `my_volume`: @@ -376,6 +373,42 @@ github.com/databricks/cli/bundle/config/resources.Volume: "volume_type": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "deployment_artifacts": + "description": |- + PLACEHOLDER + "deployment_id": + "description": |- + PLACEHOLDER + "mode": + "description": |- + PLACEHOLDER + "source_code_path": + "description": |- + PLACEHOLDER + "status": + "description": |- + PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: + "source_code_path": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/apps.AppResource: "job": "description": |- @@ -389,6 +422,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource: "sql_warehouse": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: + "key": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER + "scope": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: + "name": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: + "message": + "description": |- + PLACEHOLDER + "state": {} github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: "availability": "description": |- @@ -473,85 +549,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: - "create_time": - "description": |- - PLACEHOLDER - "creator": - "description": |- - PLACEHOLDER - "deployment_artifacts": - "description": |- - PLACEHOLDER - "deployment_id": - "description": |- - PLACEHOLDER - "mode": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "status": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: - "source_code_path": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: - "key": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER - "scope": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: - "name": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: - "message": - "description": |- - PLACEHOLDER - "state": github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 50e69e7c8..ca8c27d4c 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "os" @@ -9,8 +10,9 @@ import ( "strings" "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" - "gopkg.in/yaml.v3" ) type Components struct { @@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if err != nil { return err } - err = yaml.Unmarshal(b, &overrides) + overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b)) + if err != nil { + return err + } + err = convert.ToTyped(&overrides, overridesDyn) if err != nil { return err } diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 5028bb0d7..85f6a0328 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -79,7 +79,7 @@ type Schema struct { // Examples of the value for properties in the schema. // https://json-schema.org/understanding-json-schema/reference/annotations - Examples []any `json:"examples,omitempty"` + Examples any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string. From 75127fe42eb5fb2221b49c39977de32e3441ae0d Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 7 Feb 2025 11:26:20 +0100 Subject: [PATCH 04/21] Extend testserver for deployment (#2299) ## Changes Extend testserver for bundle deployment: - Allocate a new workspace per test case to isolate test cases from each other - Support jobs get/list/create - Support creation and listing of workspace files ## Tests Using existing acceptance tests --- acceptance/acceptance_test.go | 28 ++- acceptance/bundle/scripts/output.txt | 14 +- acceptance/cmd_server_test.go | 2 +- acceptance/server_test.go | 107 +++++++++-- .../workspace/jobs/create/out.requests.txt | 2 +- libs/testserver/fake_workspace.go | 169 ++++++++++++++++++ libs/testserver/server.go | 50 +++++- 7 files changed, 332 insertions(+), 40 deletions(-) create mode 100644 libs/testserver/fake_workspace.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f205217ff..4c4404d55 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -19,6 +19,8 @@ import ( "time" "unicode/utf8" + "github.com/google/uuid" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testdiff" @@ -123,7 +125,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { AddHandlers(defaultServer) // Redirect API access to local server: t.Setenv("DATABRICKS_HOST", defaultServer.URL) - t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() // Do not read user's ~/.databrickscfg @@ -146,7 +147,15 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") - workspaceClient, err := databricks.NewWorkspaceClient() + var config databricks.Config + if cloudEnv == "" { + // use fake token for local tests + config = databricks.Config{Token: "dbapi1234"} + } else { + // non-local tests rely on environment variables + config = databricks.Config{} + } + workspaceClient, err := databricks.NewWorkspaceClient(&config) require.NoError(t, err) user, err := workspaceClient.CurrentUser.Me(ctx) @@ -264,7 +273,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(req *http.Request) (any, int) { + server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { statusCode := http.StatusOK if stub.Response.StatusCode != 0 { statusCode = stub.Response.StatusCode @@ -285,6 +294,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } + // Each local test should use a new token that will result into a new fake workspace, + // so that test don't interfere with each other. + if cloudEnv == "" { + tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "") + token := "dbapi" + tokenSuffix + cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token) + repls.Set(token, "[DATABRICKS_TOKEN]") + } + // Write combined output to a file out, err := os.Create(filepath.Join(tmpDir, "output.txt")) require.NoError(t, err) @@ -303,8 +321,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont reqJson, err := json.Marshal(req) require.NoError(t, err) - line := fmt.Sprintf("%s\n", reqJson) - _, err = f.WriteString(line) + reqJsonWithRepls := repls.Replace(string(reqJson)) + _, err = f.WriteString(reqJsonWithRepls + "\n") require.NoError(t, err) } diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt index 2deedb0e7..68afb2fec 100644 --- a/acceptance/bundle/scripts/output.txt +++ b/acceptance/bundle/scripts/output.txt @@ -42,11 +42,9 @@ from myscript.py 0 postbuild: hello stderr! Executing 'predeploy' script from myscript.py 0 predeploy: hello stdout! from myscript.py 0 predeploy: hello stderr! -Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME]. -Please make sure the current user or one of their groups is listed under the permissions of this bundle. -For assistance, contact the owners of this project. -They may need to redeploy the bundle to apply the new permissions. -Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. - - -Exit code: 1 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/scripts/default/files... +Deploying resources... +Deployment complete! +Executing 'postdeploy' script +from myscript.py 0 postdeploy: hello stdout! +from myscript.py 0 postdeploy: hello stderr! diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 04d56c7d4..0166dfe32 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,7 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(r *http.Request) (any, int) { + server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/server_test.go b/acceptance/server_test.go index a7695b21e..d21ab66e8 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -1,17 +1,23 @@ package acceptance_test import ( + "bytes" + "encoding/json" + "fmt" "net/http" - "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + + "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/workspace" ) func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -26,7 +32,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -37,7 +43,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -52,31 +58,74 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return iam.User{ Id: "1000012345", UserName: "tester@databricks.com", }, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) { - return workspace.ObjectInfo{ - ObjectId: 1001, - ObjectType: "DIRECTORY", - Path: "", - ResourceId: "1001", - }, http.StatusOK + server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceGetStatus(path) }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) { + server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := workspace.Mkdirs{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceMkdirs(request) + }) + + server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceExport(path) + }) + + server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + recursiveStr := r.URL.Query().Get("recursive") + var recursive bool + + if recursiveStr == "true" { + recursive = true + } else { + recursive = false + } + + return fakeWorkspace.WorkspaceDelete(path, recursive) + }) + + server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.PathValue("path") + + body := new(bytes.Buffer) + _, err := body.ReadFrom(r.Body) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes()) + }) + + server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + objectId := r.PathValue("objectId") + return workspace.WorkspaceObjectPermissions{ - ObjectId: "1001", + ObjectId: objectId, ObjectType: "DIRECTORY", AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ { @@ -91,7 +140,29 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) { - return "{}", http.StatusOK + server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := jobs.CreateJob{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.JobsCreate(request) + }) + + server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + jobId := r.URL.Query().Get("job_id") + + return fakeWorkspace.JobsGet(jobId) + }) + + server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return fakeWorkspace.JobsList() }) } + +func internalError(err error) (any, int) { + return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError +} diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 4a85c4c43..60977e3e3 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go new file mode 100644 index 000000000..c3e4f9a71 --- /dev/null +++ b/libs/testserver/fake_workspace.go @@ -0,0 +1,169 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// FakeWorkspace holds a state of a workspace for acceptance tests. +type FakeWorkspace struct { + directories map[string]bool + files map[string][]byte + // normally, ids are not sequential, but we make them sequential for deterministic diff + nextJobId int64 + jobs map[int64]jobs.Job +} + +func NewFakeWorkspace() *FakeWorkspace { + return &FakeWorkspace{ + directories: map[string]bool{ + "/Workspace": true, + }, + files: map[string][]byte{}, + jobs: map[int64]jobs.Job{}, + nextJobId: 1, + } +} + +func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) { + if s.directories[path] { + return workspace.ObjectInfo{ + ObjectType: "DIRECTORY", + Path: path, + }, http.StatusOK + } else if _, ok := s.files[path]; ok { + return workspace.ObjectInfo{ + ObjectType: "FILE", + Path: path, + Language: "SCALA", + }, http.StatusOK + } else { + return workspace.ObjectInfo{}, http.StatusNotFound + } +} + +func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) { + s.directories[request.Path] = true + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) { + file := s.files[path] + + if file == nil { + return nil, http.StatusNotFound + } + + return file, http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) { + if !recursive { + s.files[path] = nil + } else { + for key := range s.files { + if strings.HasPrefix(key, path) { + s.files[key] = nil + } + } + } + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + s.files[path] = body + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) { + jobId := s.nextJobId + s.nextJobId++ + + jobSettings := jobs.JobSettings{} + err := jsonConvert(request, &jobSettings) + if err != nil { + return internalError(err) + } + + s.jobs[jobId] = jobs.Job{ + JobId: jobId, + Settings: &jobSettings, + } + + return jobs.CreateResponse{JobId: jobId}, http.StatusOK +} + +func (s *FakeWorkspace) JobsGet(jobId string) (any, int) { + id := jobId + + jobIdInt, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return internalError(fmt.Errorf("failed to parse job id: %s", err)) + } + + job, ok := s.jobs[jobIdInt] + if !ok { + return jobs.Job{}, http.StatusNotFound + } + + return job, http.StatusOK +} + +func (s *FakeWorkspace) JobsList() (any, int) { + list := make([]jobs.BaseJob, 0, len(s.jobs)) + for _, job := range s.jobs { + baseJob := jobs.BaseJob{} + err := jsonConvert(job, &baseJob) + if err != nil { + return internalError(fmt.Errorf("failed to convert job to base job: %w", err)) + } + + list = append(list, baseJob) + } + + // sort to have less non-determinism in tests + sort.Slice(list, func(i, j int) bool { + return list[i].JobId < list[j].JobId + }) + + return jobs.ListJobsResponse{ + Jobs: list, + }, http.StatusOK +} + +// jsonConvert saves input to a value pointed by output +func jsonConvert(input, output any) error { + writer := new(bytes.Buffer) + encoder := json.NewEncoder(writer) + err := encoder.Encode(input) + if err != nil { + return fmt.Errorf("failed to encode: %w", err) + } + + decoder := json.NewDecoder(writer) + err = decoder.Decode(output) + if err != nil { + return fmt.Errorf("failed to decode: %w", err) + } + + return nil +} + +func internalError(err error) (string, int) { + return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError +} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5e3efe1c5..ffb83a49c 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -6,6 +6,8 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" + "sync" "github.com/stretchr/testify/assert" @@ -18,6 +20,9 @@ type Server struct { t testutil.TestingT + fakeWorkspaces map[string]*FakeWorkspace + mu *sync.Mutex + RecordRequests bool IncludeRequestHeaders []string @@ -37,17 +42,36 @@ func New(t testutil.TestingT) *Server { t.Cleanup(server.Close) return &Server{ - Server: server, - Mux: mux, - t: t, + Server: server, + Mux: mux, + t: t, + mu: &sync.Mutex{}, + fakeWorkspaces: map[string]*FakeWorkspace{}, } } -type HandlerFunc func(req *http.Request) (resp any, statusCode int) +type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, statusCode := handler(r) + // For simplicity we process requests sequentially. It's fast enough because + // we don't do any IO except reading and writing request/response bodies. + s.mu.Lock() + defer s.mu.Unlock() + + // Each test uses unique DATABRICKS_TOKEN, we simulate each token having + // it's own fake fakeWorkspace to avoid interference between tests. + var fakeWorkspace *FakeWorkspace = nil + token := getToken(r) + if token != "" { + if _, ok := s.fakeWorkspaces[token]; !ok { + s.fakeWorkspaces[token] = NewFakeWorkspace() + } + + fakeWorkspace = s.fakeWorkspaces[token] + } + + resp, statusCode := handler(fakeWorkspace, r) if s.RecordRequests { body, err := io.ReadAll(r.Body) @@ -75,9 +99,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { var respBytes []byte var err error - respString, ok := resp.(string) - if ok { + if respString, ok := resp.(string); ok { respBytes = []byte(respString) + } else if respBytes0, ok := resp.([]byte); ok { + respBytes = respBytes0 } else { respBytes, err = json.MarshalIndent(resp, "", " ") if err != nil { @@ -92,3 +117,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } }) } + +func getToken(r *http.Request) string { + header := r.Header.Get("Authorization") + prefix := "Bearer " + + if !strings.HasPrefix(header, prefix) { + return "" + } + + return header[len(prefix):] +} From 54e16d5f62ccd169c5f64319f3b174f6dc58e326 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 12:29:40 +0100 Subject: [PATCH 05/21] Always print warnings and errors; clean up format (#2213) ## Changes - Print warnings and errors by default. - Fix ErrAlreadyPrinted not to be logged at Error level. - Format log messages as "Warn: message" instead of "WARN" to make it more readable and in-line with the rest of the output. - Only print attributes (pid, mutator, etc) and time when the overall level is debug (so --debug output has not changed much). ## Tests - Existing acceptance tests show how warning messages appear in various test case. - Added new test for `--debug` output. - Add sort_lines.py helper to avoid dependency on 'sort' which is locale-sensitive. --- acceptance/bin/sort_lines.py | 10 ++ acceptance/bundle/debug/databricks.yml | 2 + .../bundle/debug/out.stderr.parallel.txt | 15 +++ acceptance/bundle/debug/out.stderr.txt | 92 +++++++++++++++++++ acceptance/bundle/debug/output.txt | 7 ++ acceptance/bundle/debug/script | 4 + acceptance/bundle/debug/test.toml | 18 ++++ acceptance/bundle/git-permerror/output.txt | 9 ++ acceptance/bundle/git-permerror/test.toml | 2 +- bundle/artifacts/upload.go | 2 +- bundle/mutator.go | 2 +- bundle/mutator_read_only.go | 2 +- cmd/root/root.go | 11 ++- libs/flags/log_level_flag.go | 2 +- libs/flags/log_level_flag_test.go | 4 +- libs/log/handler/friendly.go | 60 ++++++------ 16 files changed, 206 insertions(+), 36 deletions(-) create mode 100755 acceptance/bin/sort_lines.py create mode 100644 acceptance/bundle/debug/databricks.yml create mode 100644 acceptance/bundle/debug/out.stderr.parallel.txt create mode 100644 acceptance/bundle/debug/out.stderr.txt create mode 100644 acceptance/bundle/debug/output.txt create mode 100644 acceptance/bundle/debug/script create mode 100644 acceptance/bundle/debug/test.toml diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py new file mode 100755 index 000000000..9ac87feee --- /dev/null +++ b/acceptance/bin/sort_lines.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +""" +Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH. +""" + +import sys + +lines = sys.stdin.readlines() +lines.sort() +sys.stdout.write("".join(lines)) diff --git a/acceptance/bundle/debug/databricks.yml b/acceptance/bundle/debug/databricks.yml new file mode 100644 index 000000000..2c9dd3c90 --- /dev/null +++ b/acceptance/bundle/debug/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: debug diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt new file mode 100644 index 000000000..7dd770068 --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.parallel.txt @@ -0,0 +1,15 @@ +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt new file mode 100644 index 000000000..9cac8bb2b --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.txt @@ -0,0 +1,92 @@ +10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug" +10:07:59 Debug: Found bundle root at [TMPDIR] (file [TMPDIR]/databricks.yml) pid=12345 +10:07:59 Debug: Apply pid=12345 mutator=load +10:07:59 Info: Phase: load pid=12345 mutator=load +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EntryPoint +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=VerifyCliVersion +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EnvironmentsToTargets +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ComputeIdToClusterId +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=InitializeVariables +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=DefineDefaultTarget(default) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=PythonMutator(load) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=validate:unique_resource_keys +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget mutator=SelectTarget(default) +10:07:59 Debug: Apply pid=12345 mutator= +10:07:59 Debug: Apply pid=12345 mutator=initialize +10:07:59 Info: Phase: initialize pid=12345 mutator=initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=validate:AllResourcesHaveValues +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteSyncPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncDefaultPath +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncInferRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser +10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me +< HTTP/1.1 200 OK +< { +< "id": "[USERID]", +< "userName": "[USERNAME]" +< } pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser sdk=true +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=LoadGitDetails +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplySourceLinkedDeploymentPreset +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefineDefaultWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultWorkspacePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PrependWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetVariables +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(init) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(load_resources) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(apply_mutators) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveResourceReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobParameters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobTasks +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergePipelineClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeApps +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CaptureSchemaDependency +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CheckPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetRunAs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=OverrideCompute +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureDashboardDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureVolumeDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ProcessTargetMode +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyPresets +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultQueueing +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandPipelineGlobPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureWSFS +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=TranslatePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonWrapperWarning +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=apps.Validate +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ValidateSharedRootPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyBundlePermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=FilterCurrentUserFromPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotateJobs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotatePipelines +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: Apply pid=12345 mutator=validate +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 404 Not Found +10:07:59 Debug: POST /api/2.0/workspace/mkdirs +> { +> "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +> } +< HTTP/1.1 200 OK +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 200 OK +< { +< "object_type": "DIRECTORY", +< "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +10:07:59 Info: completed execution pid=12345 exit_code=0 diff --git a/acceptance/bundle/debug/output.txt b/acceptance/bundle/debug/output.txt new file mode 100644 index 000000000..ed72b360e --- /dev/null +++ b/acceptance/bundle/debug/output.txt @@ -0,0 +1,7 @@ +Name: debug +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/debug/default + +Validation OK! diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script new file mode 100644 index 000000000..1b4cfc8f0 --- /dev/null +++ b/acceptance/bundle/debug/script @@ -0,0 +1,4 @@ +$CLI bundle validate --debug 2> full.stderr.txt +grep -vw parallel full.stderr.txt > out.stderr.txt +grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt +rm full.stderr.txt diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml new file mode 100644 index 000000000..bb0fcb395 --- /dev/null +++ b/acceptance/bundle/debug/test.toml @@ -0,0 +1,18 @@ +LocalOnly = true + +[[Repls]] +# The keys are unsorted and also vary per OS +Old = 'Environment variables for Terraform: ([A-Z_ ,]+) ' +New = 'Environment variables for Terraform: ...redacted... ' + +[[Repls]] +Old = 'pid=[0-9]+' +New = 'pid=12345' + +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = '10:07:59' + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt index 03ab93442..730e8255b 100644 --- a/acceptance/bundle/git-permerror/output.txt +++ b/acceptance/bundle/git-permerror/output.txt @@ -3,6 +3,7 @@ >>> chmod 000 .git >>> [CLI] bundle validate +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied Name: git-permerror @@ -16,6 +17,7 @@ Found 1 error Exit code: 1 >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -25,6 +27,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -39,11 +42,15 @@ Exit code: 1 >>> chmod 000 .git/HEAD >>> [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } @@ -54,6 +61,7 @@ Exit code: 1 >>> chmod 000 .git/config >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -63,6 +71,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml index 3f96e551c..15305cff1 100644 --- a/acceptance/bundle/git-permerror/test.toml +++ b/acceptance/bundle/git-permerror/test.toml @@ -1,4 +1,4 @@ -Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct" +Badness = "inferred flag is set to true incorrect; bundle_root_path is not correct; Warn and Error talk about the same; Warn goes to stderr, Error goes to stdout (for backward compat); Warning about permissions repeated twice" [GOOS] # This test relies on chmod which does not work on Windows diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index c69939e8c..d4625d85d 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -29,7 +29,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics // We intentionally ignore the error because it is not critical to the deployment err := client.Delete(ctx, ".", filer.DeleteRecursively) if err != nil { - log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err) + log.Debugf(ctx, "failed to delete %s: %v", uploadPath, err) } err = client.Mkdir(ctx, ".") diff --git a/bundle/mutator.go b/bundle/mutator.go index 6c9968aac..16ef79ee7 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -42,7 +42,7 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics { // such that they are not logged multiple times. // If this is done, we can omit this block. if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go index ee4e36e0f..700a90d8d 100644 --- a/bundle/mutator_read_only.go +++ b/bundle/mutator_read_only.go @@ -22,7 +22,7 @@ func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) di log.Debugf(ctx, "ApplyReadOnly") diags := m.Apply(ctx, rb) if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/cmd/root/root.go b/cmd/root/root.go index 3b37d0176..d7adf47f4 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -114,10 +114,15 @@ func Execute(ctx context.Context, cmd *cobra.Command) error { if err == nil { logger.Info("completed execution", slog.String("exit_code", "0")) - } else { - logger.Error("failed execution", + } else if errors.Is(err, ErrAlreadyPrinted) { + logger.Debug("failed execution", slog.String("exit_code", "1"), - slog.String("error", err.Error())) + ) + } else { + logger.Info("failed execution", + slog.String("exit_code", "1"), + slog.String("error", err.Error()), + ) } } diff --git a/libs/flags/log_level_flag.go b/libs/flags/log_level_flag.go index 836d84b70..82e2abc4c 100644 --- a/libs/flags/log_level_flag.go +++ b/libs/flags/log_level_flag.go @@ -25,7 +25,7 @@ type LogLevelFlag struct { func NewLogLevelFlag() LogLevelFlag { return LogLevelFlag{ - l: log.LevelDisabled, + l: log.LevelWarn, } } diff --git a/libs/flags/log_level_flag_test.go b/libs/flags/log_level_flag_test.go index 11a50bc45..c81f90d18 100644 --- a/libs/flags/log_level_flag_test.go +++ b/libs/flags/log_level_flag_test.go @@ -10,8 +10,8 @@ import ( func TestLogLevelFlagDefault(t *testing.T) { f := NewLogLevelFlag() - assert.Equal(t, log.LevelDisabled, f.Level()) - assert.Equal(t, "disabled", f.String()) + assert.Equal(t, log.LevelWarn, f.Level()) + assert.Equal(t, "warn", f.String()) } func TestLogLevelFlagSetValid(t *testing.T) { diff --git a/libs/log/handler/friendly.go b/libs/log/handler/friendly.go index 33b88a9e2..5c60eb13d 100644 --- a/libs/log/handler/friendly.go +++ b/libs/log/handler/friendly.go @@ -53,11 +53,11 @@ func NewFriendlyHandler(out io.Writer, opts *Options) slog.Handler { // Cache (colorized) level strings. // The colors to use for each level are configured in `colors.go`. - h.levelTrace = h.sprintf(ttyColorLevelTrace, "%5s", "TRACE") - h.levelDebug = h.sprintf(ttyColorLevelDebug, "%5s", "DEBUG") - h.levelInfo = h.sprintf(ttyColorLevelInfo, "%5s", "INFO") - h.levelWarn = h.sprintf(ttyColorLevelWarn, "%5s", "WARN") - h.levelError = h.sprintf(ttyColorLevelError, "%5s", "ERROR") + h.levelTrace = h.sprintf(ttyColorLevelTrace, "%s", "Trace:") + h.levelDebug = h.sprintf(ttyColorLevelDebug, "%s", "Debug:") + h.levelInfo = h.sprintf(ttyColorLevelInfo, "%s", "Info:") + h.levelWarn = h.sprintf(ttyColorLevelWarn, "%s", "Warn:") + h.levelError = h.sprintf(ttyColorLevelError, "%s", "Error:") return h } @@ -185,33 +185,41 @@ func (s *handleState) appendAttr(a slog.Attr) { // Handle implements slog.Handler. func (h *friendlyHandler) Handle(ctx context.Context, r slog.Record) error { state := h.handleState() - state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + + if h.opts.Level.Level() <= slog.LevelDebug { + state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + } + state.appendf("%s ", h.coloredLevel(r)) state.append(h.sprint(ttyColorMessage, r.Message)) - // Handle state from WithGroup and WithAttrs. - goas := h.goas - if r.NumAttrs() == 0 { - // If the record has no Attrs, remove groups at the end of the list; they are empty. - for len(goas) > 0 && goas[len(goas)-1].group != "" { - goas = goas[:len(goas)-1] - } - } - for _, goa := range goas { - if goa.group != "" { - state.openGroup(goa.group) - } else { - for _, a := range goa.attrs { - state.appendAttr(a) + if h.opts.Level.Level() <= slog.LevelDebug { + + // Handle state from WithGroup and WithAttrs. + goas := h.goas + if r.NumAttrs() == 0 { + // If the record has no Attrs, remove groups at the end of the list; they are empty. + for len(goas) > 0 && goas[len(goas)-1].group != "" { + goas = goas[:len(goas)-1] + } + } + for _, goa := range goas { + if goa.group != "" { + state.openGroup(goa.group) + } else { + for _, a := range goa.attrs { + state.appendAttr(a) + } } } - } - // Add attributes from the record. - r.Attrs(func(a slog.Attr) bool { - state.appendAttr(a) - return true - }) + // Add attributes from the record. + r.Attrs(func(a slog.Attr) bool { + state.appendAttr(a) + return true + }) + + } // Add newline. state.append("\n") From 65ac9a336a92c3d3f7e0dece648437024a361c90 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:21:37 +0530 Subject: [PATCH 06/21] Add doc string for the `auth token` command (#2302) ## Changes The intent of this PR is to clarify that the `databricks auth token` command is not supported for M2M auth. Fixes: https://github.com/databricks/cli/issues/1939 ## Tests Manually. --- cmd/auth/token.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/auth/token.go b/cmd/auth/token.go index fbf8b68f6..f3468df40 100644 --- a/cmd/auth/token.go +++ b/cmd/auth/token.go @@ -46,6 +46,10 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { cmd := &cobra.Command{ Use: "token [HOST]", Short: "Get authentication token", + Long: `Get authentication token from the local cache in ~/.databricks/token-cache.json. +Refresh the access token if it is expired. Note: This command only works with +U2M authentication (using the 'databricks auth login' command). M2M authentication +using a client ID and secret is not supported.`, } var tokenTimeout time.Duration From ecc05689cafffcbc80ec14e708696ae4beff6629 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 14:13:12 +0100 Subject: [PATCH 07/21] Add a couple of tests for bundle init with custom template (#2293) These test custom template and what happens if helper function returns an error. --- .../helpers-error/databricks_template_schema.json | 1 + acceptance/bundle/templates/helpers-error/output.txt | 3 +++ acceptance/bundle/templates/helpers-error/script | 1 + .../templates/helpers-error/template/helpers.txt.tmpl | 1 + acceptance/bundle/templates/helpers-error/test.toml | 7 +++++++ .../templates/helpers/databricks_template_schema.json | 1 + acceptance/bundle/templates/helpers/output.txt | 2 ++ acceptance/bundle/templates/helpers/script | 3 +++ .../bundle/templates/helpers/template/helpers.txt.tmpl | 1 + acceptance/bundle/templates/helpers/test.toml | 1 + 10 files changed, 21 insertions(+) create mode 100644 acceptance/bundle/templates/helpers-error/databricks_template_schema.json create mode 100644 acceptance/bundle/templates/helpers-error/output.txt create mode 100644 acceptance/bundle/templates/helpers-error/script create mode 100644 acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl create mode 100644 acceptance/bundle/templates/helpers-error/test.toml create mode 100644 acceptance/bundle/templates/helpers/databricks_template_schema.json create mode 100644 acceptance/bundle/templates/helpers/output.txt create mode 100644 acceptance/bundle/templates/helpers/script create mode 100644 acceptance/bundle/templates/helpers/template/helpers.txt.tmpl create mode 100644 acceptance/bundle/templates/helpers/test.toml diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates/helpers-error/output.txt new file mode 100644 index 000000000..6020e944f --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/output.txt @@ -0,0 +1,3 @@ +Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: + +Exit code: 1 diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates/helpers-error/script new file mode 100644 index 000000000..d9fcbf62c --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/script @@ -0,0 +1 @@ +$CLI bundle init . diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates/helpers-error/test.toml new file mode 100644 index 000000000..77f4ed94b --- /dev/null +++ b/acceptance/bundle/templates/helpers-error/test.toml @@ -0,0 +1,7 @@ +Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' +LocalOnly = true + +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/Me" +Response.Body = '{}' +Response.StatusCode = 500 diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates/helpers/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates/helpers/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates/helpers/output.txt new file mode 100644 index 000000000..3126ea5af --- /dev/null +++ b/acceptance/bundle/templates/helpers/output.txt @@ -0,0 +1,2 @@ +✨ Successfully initialized template +user_name: [USERNAME] diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates/helpers/script new file mode 100644 index 000000000..1773e7b03 --- /dev/null +++ b/acceptance/bundle/templates/helpers/script @@ -0,0 +1,3 @@ +$CLI bundle init . +cat helpers.txt +rm helpers.txt diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates/helpers/test.toml new file mode 100644 index 000000000..b76e712fb --- /dev/null +++ b/acceptance/bundle/templates/helpers/test.toml @@ -0,0 +1 @@ +LocalOnly = true From ecb816446e63382cb8871bb376eb26b9f80b29cc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 14:54:24 +0000 Subject: [PATCH 08/21] Update app deploy test to confirm app config changes on redeploy (#2301) ## Changes Adds additional step to integration test which changes the app config and confirms it's updated after redeploy ## Tests ``` helpers_test.go:156: stderr: Deleting files... helpers_test.go:156: stderr: Destroy complete! --- PASS: TestDeployBundleWithApp (470.25s) PASS coverage: [no statements] ok github.com/databricks/cli/integration/bundle 470.981s coverage: [no statements] ``` --- integration/bundle/apps_test.go | 18 ++++++++++++++++++ .../bundles/apps/template/databricks.yml.tmpl | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go index 01ab52e90..12bd2fcbf 100644 --- a/integration/bundle/apps_test.go +++ b/integration/bundle/apps_test.go @@ -98,6 +98,24 @@ func TestDeployBundleWithApp(t *testing.T) { - run env: - name: JOB_ID + value: "%d"`, job.JobId)) + + // Redeploy bundle with changed config env for app and confirm it's updated in app.yaml + deployBundleWithArgs(t, ctx, root, `--var="env_var_name=ANOTHER_JOB_ID"`, "--force-lock", "--auto-approve") + reader, err = wt.W.Workspace.Download(ctx, pathToAppYml) + require.NoError(t, err) + + data, err = io.ReadAll(reader) + require.NoError(t, err) + + content = string(data) + require.Contains(t, content, fmt.Sprintf(`command: + - flask + - --app + - app + - run +env: + - name: ANOTHER_JOB_ID value: "%d"`, job.JobId)) if testing.Short() { diff --git a/integration/bundle/bundles/apps/template/databricks.yml.tmpl b/integration/bundle/bundles/apps/template/databricks.yml.tmpl index 4d862a06f..e0937be71 100644 --- a/integration/bundle/bundles/apps/template/databricks.yml.tmpl +++ b/integration/bundle/bundles/apps/template/databricks.yml.tmpl @@ -4,6 +4,10 @@ bundle: workspace: root_path: "~/.bundle/{{.unique_id}}" +variables: + env_var_name: + default: "JOB_ID" + resources: apps: test_app: @@ -17,7 +21,7 @@ resources: - app - run env: - - name: JOB_ID + - name: ${var.env_var_name} value: ${resources.jobs.foo.id} resources: From 6b1a778fe10051d779beb22101825b5a72dcddae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 17:17:50 +0100 Subject: [PATCH 09/21] Fix flaky acceptance test (#2310) ## Changes Replace timestamps with fixed string before output is sorted (and before test runner replacements are applied). Otherwise the test sometimes fails with error below. Note, timestamps themselves do not show it, because they were replaced. ``` --- FAIL: TestAccept/bundle/debug (0.78s) acceptance_test.go:404: Diff: --- bundle/debug/out.stderr.parallel.txt +++ /var/folders/5y/9kkdnjw91p11vsqwk0cvmk200000gp/T/TestAcceptbundledebug1859985035/001/out.stderr.parallel.txt @@ -8,8 +8,8 @@ 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns -10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync 10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true < {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true ``` ## Tests Running `hyperfine --min-runs 10 'go test ../.. -run ^TestAccept$/^bundle$/^debug$ -count=1' --show-output` detects flakiness on main but not with this PR. --- acceptance/bundle/debug/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script index 1b4cfc8f0..005a1a341 100644 --- a/acceptance/bundle/debug/script +++ b/acceptance/bundle/debug/script @@ -1,4 +1,4 @@ $CLI bundle validate --debug 2> full.stderr.txt grep -vw parallel full.stderr.txt > out.stderr.txt -grep -w parallel full.stderr.txt | sort_lines.py > out.stderr.parallel.txt +grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt rm full.stderr.txt From f71583fbc09504dbbf0742e53bdcebca53d424a3 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 7 Feb 2025 21:56:48 +0530 Subject: [PATCH 10/21] Error when unknown API endpoint is used in testserver (#2292) ## Changes This PR fails the acceptance test when an unknown endpoint (i.e. not stubbed) is used. We want to ensure that all API endpoints used in an acceptance test are stubbed and do not otherwise silently fail with a 404. The logs on failure output include a configuration that developers can simply copy-paste to `test.toml` to stub the missing API endpoint. It'll look something like: ``` [[Server]] Pattern = " " Response.Body = ''' ''' Response.StatusCode = ``` ## Tests Manually: output.txt when an endpoint is not found: ``` >>> [CLI] jobs create --json {"name":"abc"} Error: No stub found for pattern: POST /api/2.1/jobs/create ``` How this renders in the test logs: ``` --- FAIL: TestAccept/workspace/jobs/create (0.03s) server.go:46: ---------------------------------------- No stub found for pattern: POST /api/2.1/jobs/create To stub a response for this request, you can add the following to test.toml: [[Server]] Pattern = "POST /api/2.1/jobs/create" Response.Body = ''' ''' Response.StatusCode = ---------------------------------------- ``` Manually checked that the debug mode still works. --- acceptance/cmd_server_test.go | 5 ++++- libs/testserver/server.go | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 0166dfe32..c8a52f4cd 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,10 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { + // {$} is a wildcard that only matches the end of the URL. We explicitly use + // /{$} to disambiguate it from the generic handler for '/' which is used to + // identify unhandled API endpoints in the test server. + server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/libs/testserver/server.go b/libs/testserver/server.go index ffb83a49c..9ccf34be0 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/apierr" ) type Server struct { @@ -41,13 +42,44 @@ func New(t testutil.TestingT) *Server { server := httptest.NewServer(mux) t.Cleanup(server.Close) - return &Server{ + s := &Server{ Server: server, Mux: mux, t: t, mu: &sync.Mutex{}, fakeWorkspaces: map[string]*FakeWorkspace{}, } + + // The server resolves conflicting handlers by using the one with higher + // specificity. This handler is the least specific, so it will be used as a + // fallback when no other handlers match. + s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) { + pattern := r.Method + " " + r.URL.Path + + t.Errorf(` + +---------------------------------------- +No stub found for pattern: %s + +To stub a response for this request, you can add +the following to test.toml: +[[Server]] +Pattern = %q +Response.Body = ''' + +''' +Response.StatusCode = +---------------------------------------- + + +`, pattern, pattern) + + return apierr.APIError{ + Message: "No stub found for pattern: " + pattern, + }, http.StatusNotFound + }) + + return s } type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) From ff4a5c22698dd31445524672c70f05d05031d4f3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 17:38:27 +0100 Subject: [PATCH 11/21] acc: Implement config merge (#2294) ## Changes Instead of using leaf-most config, all configs from root at acceptance/test.toml to all intermediate ones to leaf config are merged into one. Maps are merged, slices are appended, other values are overridden. I had to disable caching, because it is tricky when merging is involved - deep copy is needed. There is performance impact but currently it is tiny, about 1%. Also, remove empty root config. ## Tests Manually checked that inheritance of LocalOnly setting worked for these tests: Before - integration tests showed: ``` PASS acceptance.TestAccept/bundle/templates/wrong-url (0.70s) PASS acceptance.TestAccept/bundle/templates/wrong-path (0.44s) ``` After: ``` SKIP acceptance.TestAccept/bundle/templates/wrong-url (0.00s) SKIP acceptance.TestAccept/bundle/templates/wrong-path (0.00s) acceptance_test.go:216: Disabled via LocalOnly setting in bundle/templates/test.toml, bundle/templates/wrong-path/test.toml (CLOUD_ENV=***) ``` --- NOTICE | 5 ++++ acceptance/config_test.go | 56 +++++++++++++++++---------------------- acceptance/test.toml | 2 -- go.mod | 1 + go.sum | 4 +-- 5 files changed, 33 insertions(+), 35 deletions(-) delete mode 100644 acceptance/test.toml diff --git a/NOTICE b/NOTICE index ed22084cf..4331a2a32 100644 --- a/NOTICE +++ b/NOTICE @@ -109,3 +109,8 @@ License - https://github.com/hexops/gotextdiff/blob/main/LICENSE https://github.com/BurntSushi/toml Copyright (c) 2013 TOML authors https://github.com/BurntSushi/toml/blob/master/COPYING + +dario.cat/mergo +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. +https://github.com/darccio/mergo/blob/master/LICENSE diff --git a/acceptance/config_test.go b/acceptance/config_test.go index e24a683e7..920e713a1 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -3,9 +3,11 @@ package acceptance_test import ( "os" "path/filepath" - "sync" + "slices" + "strings" "testing" + "dario.cat/mergo" "github.com/BurntSushi/toml" "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" @@ -13,11 +15,6 @@ import ( const configFilename = "test.toml" -var ( - configCache map[string]TestConfig - configMutex sync.Mutex -) - type TestConfig struct { // Place to describe what's wrong with this test. Does not affect how the test is run. Badness string @@ -65,58 +62,55 @@ type ServerStub struct { } } -// FindConfig finds the closest config file. -func FindConfig(t *testing.T, dir string) (string, bool) { - shared := false +// FindConfigs finds all the config relevant for this test, +// ordered from the most outermost (at acceptance/) to current test directory (identified by dir). +// Argument dir must be a relative path from the root of acceptance tests (/acceptance/). +func FindConfigs(t *testing.T, dir string) []string { + configs := []string{} for { path := filepath.Join(dir, configFilename) _, err := os.Stat(path) if err == nil { - return path, shared + configs = append(configs, path) } - shared = true - if dir == "" || dir == "." { break } - if os.IsNotExist(err) { - dir = filepath.Dir(dir) + dir = filepath.Dir(dir) + + if err == nil || os.IsNotExist(err) { continue } t.Fatalf("Error while reading %s: %s", path, err) } - t.Fatal("Config not found: " + configFilename) - return "", shared + slices.Reverse(configs) + return configs } // LoadConfig loads the config file. Non-leaf configs are cached. func LoadConfig(t *testing.T, dir string) (TestConfig, string) { - path, leafConfig := FindConfig(t, dir) + configs := FindConfigs(t, dir) - if leafConfig { - return DoLoadConfig(t, path), path + if len(configs) == 0 { + return TestConfig{}, "(no config)" } - configMutex.Lock() - defer configMutex.Unlock() + result := DoLoadConfig(t, configs[0]) - if configCache == nil { - configCache = make(map[string]TestConfig) + for _, cfgName := range configs[1:] { + cfg := DoLoadConfig(t, cfgName) + err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + t.Fatalf("Error during config merge: %s: %s", cfgName, err) + } } - result, ok := configCache[path] - if ok { - return result, path - } - - result = DoLoadConfig(t, path) - configCache[path] = result - return result, path + return result, strings.Join(configs, ", ") } func DoLoadConfig(t *testing.T, path string) TestConfig { diff --git a/acceptance/test.toml b/acceptance/test.toml deleted file mode 100644 index eee94d0ea..000000000 --- a/acceptance/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration. -# The configurations are not merged across parents; the closest one is used fully. diff --git a/go.mod b/go.mod index b3f11e918..662fcd40b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 toolchain go1.23.4 require ( + dario.cat/mergo v1.0.1 // BSD 3-Clause github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 diff --git a/go.sum b/go.sum index 4e295a82d..bffc3b53d 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKF cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= From 5aa89230e96635efaec5aa32a6a06a8cab9dec2f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 17:22:51 +0000 Subject: [PATCH 12/21] Use CreatePipeline instead of PipelineSpec for resources.Pipeline struct (#2287) ## Changes `CreatePipeline` is a more complete structure (superset of PipelineSpec one) which enables support of additional fields such as `run_as` and `allow_duplicate_names` in DABs configuration. Note: these fields are subject to support in TF in order to correctly work. ## Tests Existing tests pass + no fields are removed from JSON schema --- bundle/config/mutator/apply_presets.go | 2 +- .../mutator/capture_schema_dependency.go | 4 +-- .../mutator/capture_schema_dependency_test.go | 30 +++++++++---------- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/initialize_urls_test.go | 4 +-- .../mutator/merge_pipeline_clusters_test.go | 4 +-- .../mutator/process_target_mode_test.go | 10 +++---- .../resolve_variable_references_test.go | 6 ++-- bundle/config/mutator/translate_paths_test.go | 16 +++++----- bundle/config/resources/pipeline.go | 4 +-- .../validate/single_node_cluster_test.go | 4 +-- bundle/deploy/metadata/annotate_pipelines.go | 4 +-- .../metadata/annotate_pipelines_test.go | 8 ++--- bundle/deploy/terraform/convert_test.go | 8 ++--- .../terraform/tfdyn/convert_pipeline.go | 5 ++++ .../terraform/tfdyn/convert_pipeline_test.go | 11 ++++++- bundle/internal/schema/annotations.yml | 10 +++++++ .../internal/schema/annotations_openapi.yml | 17 +++++++++++ .../schema/annotations_openapi_overrides.yml | 6 ++++ bundle/internal/schema/main.go | 15 ++++++++++ bundle/permissions/workspace_root_test.go | 8 ++--- bundle/render/render_text_output_test.go | 8 ++--- bundle/resources/completion_test.go | 4 +-- bundle/resources/lookup_test.go | 4 +-- bundle/run/pipeline.go | 4 +-- bundle/schema/jsonschema.json | 26 ++++++++++++++++ cmd/bundle/generate/pipeline.go | 2 +- libs/dyn/drop_keys.go | 27 +++++++++++++++++ libs/dyn/drop_keys_test.go | 24 +++++++++++++++ 29 files changed, 208 insertions(+), 69 deletions(-) create mode 100644 libs/dyn/drop_keys.go create mode 100644 libs/dyn/drop_keys_test.go diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index b402053e7..c8e7bf9e8 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -84,7 +84,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Pipelines presets: Prefix, PipelinesDevelopment for key, p := range r.Pipelines { - if p.PipelineSpec == nil { + if p.CreatePipeline == nil { diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) continue } diff --git a/bundle/config/mutator/capture_schema_dependency.go b/bundle/config/mutator/capture_schema_dependency.go index 5025c9a0d..2e17a8175 100644 --- a/bundle/config/mutator/capture_schema_dependency.go +++ b/bundle/config/mutator/capture_schema_dependency.go @@ -56,7 +56,7 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) { } func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Schema == "" { @@ -71,7 +71,7 @@ func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { } func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Target == "" { diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go index 0a94e7748..16fa636ee 100644 --- a/bundle/config/mutator/capture_schema_dependency_test.go +++ b/bundle/config/mutator/capture_schema_dependency_test.go @@ -118,43 +118,43 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Schema: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Schema: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "", Name: "whatever", @@ -179,7 +179,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema) assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"]) - assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec) + assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].CreatePipeline) for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} { assert.Empty(t, b.Config.Resources.Pipelines[k].Target) @@ -214,43 +214,43 @@ func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Target: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Target: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "", Name: "whatever", diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index 7cf3c9f3e..c5b1ad39d 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -47,7 +47,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index f07a7deb3..8c751079b 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -31,8 +31,8 @@ func TestInitializeURLs(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - ID: "3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}, + ID: "3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1"}, }, }, Experiments: map[string]*resources.MlflowExperiment{ diff --git a/bundle/config/mutator/merge_pipeline_clusters_test.go b/bundle/config/mutator/merge_pipeline_clusters_test.go index f117d9399..97ec44eea 100644 --- a/bundle/config/mutator/merge_pipeline_clusters_test.go +++ b/bundle/config/mutator/merge_pipeline_clusters_test.go @@ -19,7 +19,7 @@ func TestMergePipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { NodeTypeId: "i3.xlarge", @@ -68,7 +68,7 @@ func TestMergePipelineClustersCaseInsensitive(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { Label: "default", diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 723b01ee3..6a0fd8e03 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -88,7 +88,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}}, + "pipeline1": {CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}}, @@ -181,7 +181,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous) - assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.True(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) // Experiment 1 assert.Equal(t, "/Users/lennart.kats@databricks.com/[dev lennart] experiment1", b.Config.Resources.Experiments["experiment1"].Name) @@ -316,7 +316,7 @@ func TestProcessTargetModeDefault(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -362,7 +362,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -568,5 +568,5 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) { diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 44f6c8dbb..30969dc49 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -20,7 +20,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { true, func(t *testing.T, b *bundle.Bundle) { // Variables that use workspace file path should have SyncRootValue during resolution phase - require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) // The file path itself should remain the same require.Equal(t, "file/path", b.Config.Workspace.FilePath) @@ -29,7 +29,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { { false, func(t *testing.T, b *bundle.Bundle) { - require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) require.Equal(t, "file/path", b.Config.Workspace.FilePath) }, }, @@ -48,7 +48,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Configuration: map[string]string{ "source": "${workspace.file_path}", }, diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index aa6488ab0..6cfe5718a 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -179,7 +179,7 @@ func TestTranslatePaths(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -333,7 +333,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -488,7 +488,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -532,7 +532,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -572,7 +572,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -677,7 +677,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -712,7 +712,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -916,7 +916,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 5127d07ba..57d9c4f19 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,7 +16,7 @@ type Pipeline struct { ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` URL string `json:"url,omitempty" bundle:"internal"` - *pipelines.PipelineSpec + *pipelines.CreatePipeline } func (s *Pipeline) UnmarshalJSON(b []byte) error { @@ -59,5 +59,5 @@ func (s *Pipeline) GetURL() string { } func (s *Pipeline) IsNil() bool { - return s.PipelineSpec == nil + return s.CreatePipeline == nil } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index c3ead8ef6..be93420c6 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -238,7 +238,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, @@ -493,7 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go index 990f48907..407aaea6e 100644 --- a/bundle/deploy/metadata/annotate_pipelines.go +++ b/bundle/deploy/metadata/annotate_pipelines.go @@ -20,11 +20,11 @@ func (m *annotatePipelines) Name() string { func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for _, pipeline := range b.Config.Resources.Pipelines { - if pipeline.PipelineSpec == nil { + if pipeline.CreatePipeline == nil { continue } - pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{ + pipeline.CreatePipeline.Deployment = &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: metadataFilePath(b), } diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go index 448a022d0..606292724 100644 --- a/bundle/deploy/metadata/annotate_pipelines_test.go +++ b/bundle/deploy/metadata/annotate_pipelines_test.go @@ -21,12 +21,12 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "my-pipeline-1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline One", }, }, "my-pipeline-2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline Two", }, }, @@ -43,14 +43,14 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-1"].CreatePipeline.Deployment) assert.Equal(t, &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-2"].CreatePipeline.Deployment) } func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) { diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index afc1fb22a..53d861b32 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -203,7 +203,7 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) { func TestBundleToTerraformPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", Libraries: []pipelines.PipelineLibrary{ { @@ -759,7 +759,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, @@ -898,12 +898,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, "test_pipeline_new": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline_new", }, }, diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index ea0c94d66..53a986864 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,6 +21,11 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } + vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"}) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go index 0239bad18..d8de55bf0 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -15,8 +15,17 @@ import ( func TestConvertPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", + // This fields is not part of TF schema yet, but once we upgrade to TF version that supports it, this test will fail because run_as + // will be exposed which is expected and test will need to be updated. + RunAs: &pipelines.RunAs{ + UserName: "foo@bar.com", + }, + // We expect AllowDuplicateNames and DryRun to be ignored and not passed to the TF output. + // This is not supported by TF now, so we don't want to expose it. + AllowDuplicateNames: true, + DryRun: true, Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index c10f43b04..2d1a6a3d8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -414,6 +414,16 @@ github.com/databricks/cli/bundle/config/resources.Permission: "user_name": "description": |- The name of the user that has the permission set in level. +github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + PLACEHOLDER + "dry_run": + "description": |- + PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index d5a9bf69e..d9a0be50e 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -371,6 +371,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "description": |- Tags to be attached to the serving endpoint and automatically propagated to billing logs. github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + If false, deployment will fail if name conflicts with that of another pipeline. "budget_policy_id": "description": |- Budget policy of this pipeline. @@ -395,6 +398,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "development": "description": |- Whether the pipeline is in Development mode. Defaults to false. + "dry_run": {} "edition": "description": |- Pipeline product edition. @@ -425,6 +429,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "restart_window": "description": |- Restart window of this pipeline. + "run_as": {} "schema": "description": |- The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode. @@ -2624,6 +2629,18 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow: "description": |- Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. +github.com/databricks/databricks-sdk-go/service/pipelines.RunAs: + "_": + "description": |- + Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline. + + Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. + "service_principal_name": + "description": |- + Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": + "description": |- + The email of an active workspace user. Users can only set this field to their own email. github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec: "destination_catalog": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 585886313..be83af2d1 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -239,9 +239,15 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: - notebook: path: ./pipeline.py ``` + "dry_run": + "description": |- + PLACEHOLDER "permissions": "description": |- PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.QualityMonitor: "_": "markdown_description": |- diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 38e099ece..2e0120e62 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -109,6 +109,20 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { return s } +func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Pipeline{}): + // Even though DABs supports this field, TF provider does not. Thus, we + // should not expose it to the user. + delete(s.Properties, "dry_run") + delete(s.Properties, "allow_duplicate_names") + default: + // Do nothing + } + + return s +} + // While volume_type is required in the volume create API, DABs automatically sets // it's value to "MANAGED" if it's not provided. Thus, we make it optional // in the bundle schema. @@ -168,6 +182,7 @@ func generateSchema(workdir, outputFile string) { // Generate the JSON schema from the bundle Go struct. s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ removeJobsFields, + removePipelineFields, makeVolumeTypeOptional, a.addAnnotations, addInterpolationPatterns, diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index c48704a63..3e5f9c61b 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -38,8 +38,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, @@ -98,8 +98,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 506756f70..d092e77c8 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -530,12 +530,12 @@ func TestRenderSummary(t *testing.T) { "pipeline2": { ID: "4", // no URL - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"}, + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline2-name"}, }, "pipeline1": { - ID: "3", - URL: "https://url3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"}, + ID: "3", + URL: "https://url3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1-name"}, }, }, Schemas: map[string]*resources.Schema{ diff --git a/bundle/resources/completion_test.go b/bundle/resources/completion_test.go index 80412b6f1..56559f18c 100644 --- a/bundle/resources/completion_test.go +++ b/bundle/resources/completion_test.go @@ -25,7 +25,7 @@ func TestCompletions_SkipDuplicates(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -50,7 +50,7 @@ func TestCompletions_Filter(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/resources/lookup_test.go b/bundle/resources/lookup_test.go index 0ea5af7a2..d95da977a 100644 --- a/bundle/resources/lookup_test.go +++ b/bundle/resources/lookup_test.go @@ -56,7 +56,7 @@ func TestLookup_MultipleFound(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -107,7 +107,7 @@ func TestLookup_NominalWithFilters(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index bdcf0f142..1cd6e8743 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -79,10 +79,10 @@ type pipelineRunner struct { } func (r *pipelineRunner) Name() string { - if r.pipeline == nil || r.pipeline.PipelineSpec == nil { + if r.pipeline == nil || r.pipeline.CreatePipeline == nil { return "" } - return r.pipeline.PipelineSpec.Name + return r.pipeline.CreatePipeline.Name } func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 7c72c440e..9d4304cd8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -703,6 +703,9 @@ "description": "Restart window of this pipeline.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow" }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RunAs" + }, "schema": { "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", "$ref": "#/$defs/string" @@ -5385,6 +5388,29 @@ } ] }, + "pipelines.RunAs": { + "oneOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.SchemaSpec": { "oneOf": [ { diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 1d2c345d6..9bf9e9947 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -92,7 +92,7 @@ func NewGeneratePipelineCommand() *cobra.Command { } saver := yamlsaver.NewSaverWithStyle( - // Including all PipelineSpec and nested fields which are map[string]string type + // Including all CreatePipeline and nested fields which are map[string]string type map[string]yaml.Style{ "spark_conf": yaml.DoubleQuotedStyle, "custom_tags": yaml.DoubleQuotedStyle, diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go new file mode 100644 index 000000000..494f9b9cd --- /dev/null +++ b/libs/dyn/drop_keys.go @@ -0,0 +1,27 @@ +package dyn + +func DropKeys(v Value, drop []string) (Value, error) { + var err error + nv, err := Walk(v, func(p Path, v Value) (Value, error) { + if len(p) == 0 { + return v, nil + } + + // Check if this key should be dropped. + for _, key := range drop { + if p[0].Key() != key { + continue + } + + return InvalidValue, ErrDrop + } + + // Pass through all other values. + return v, ErrSkip + }) + if err != nil { + return InvalidValue, err + } + + return nv, nil +} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go new file mode 100644 index 000000000..83a9744ca --- /dev/null +++ b/libs/dyn/drop_keys_test.go @@ -0,0 +1,24 @@ +package dyn + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDropKeysTest(t *testing.T) { + v := V(map[string]Value{ + "key1": V("value1"), + "key2": V("value2"), + "key3": V("value3"), + }) + + vout, err := DropKeys(v, []string{"key1", "key3"}) + require.NoError(t, err) + + mv := vout.MustMap() + require.Equal(t, 1, mv.Len()) + v, ok := mv.GetByString("key2") + require.True(t, ok) + require.Equal(t, "value2", v.MustString()) +} From 989aabe5f1c4d91aa1f9b611f8b49c8241023744 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 18:42:35 +0100 Subject: [PATCH 13/21] acc: Make variable tests local-only (#2312) Makes use of #2294 --- acceptance/bundle/variables/env_overrides/test.toml | 2 -- acceptance/bundle/variables/resolve-builtin/test.toml | 2 -- .../bundle/variables/resolve-vars-in-root-path/test.toml | 2 -- acceptance/bundle/variables/test.toml | 3 +++ 4 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 acceptance/bundle/variables/env_overrides/test.toml delete mode 100644 acceptance/bundle/variables/resolve-builtin/test.toml delete mode 100644 acceptance/bundle/variables/resolve-vars-in-root-path/test.toml create mode 100644 acceptance/bundle/variables/test.toml diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml deleted file mode 100644 index 439c2fab1..000000000 --- a/acceptance/bundle/variables/env_overrides/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml deleted file mode 100644 index 085fab6c0..000000000 --- a/acceptance/bundle/variables/resolve-builtin/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml deleted file mode 100644 index d833bd848..000000000 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml new file mode 100644 index 000000000..32398e828 --- /dev/null +++ b/acceptance/bundle/variables/test.toml @@ -0,0 +1,3 @@ +# The tests here intend to test variable interpolation via "bundle validate". +# Even though "bundle validate" does a few API calls, that's not the focus there. +LocalOnly = true From 6d83ffd1090fc3b80f9b9957aa1402ea47cfb9ef Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 7 Feb 2025 18:42:47 +0100 Subject: [PATCH 14/21] acc: enable bundle/scripts on cloud (#2313) --- acceptance/bundle/scripts/test.toml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 acceptance/bundle/scripts/test.toml diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml deleted file mode 100644 index 1dbd78681..000000000 --- a/acceptance/bundle/scripts/test.toml +++ /dev/null @@ -1 +0,0 @@ -LocalOnly = true # Deployment currently fails when run locally; once that is fixed, remove this setting From 2a97dcaa45d14981610619f334428b1a6858c332 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 17:55:16 +0000 Subject: [PATCH 15/21] Raise an error when there are multiple local libraries with the same basename used (#2297) ## Changes Raise an error when there are multiple local libraries with the same basename used Fixes #1674 ## Tests Added an unit test --- .../same_name_libraries/databricks.yml | 50 ++++++++ .../artifacts/same_name_libraries/output.txt | 14 ++ .../artifacts/same_name_libraries/script | 2 + .../artifacts/same_name_libraries/test.toml | 0 .../same_name_libraries/whl1/setup.py | 36 ++++++ .../whl1/src/my_default_python/__init__.py | 1 + .../whl1/src/my_default_python/main.py | 1 + .../same_name_libraries/whl2/setup.py | 36 ++++++ .../whl2/src/my_default_python/__init__.py | 1 + .../whl2/src/my_default_python/main.py | 1 + bundle/libraries/expand_glob_references.go | 2 +- bundle/libraries/same_name_libraries.go | 97 ++++++++++++++ bundle/libraries/same_name_libraries_test.go | 121 ++++++++++++++++++ bundle/phases/deploy.go | 5 + 14 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml create mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt create mode 100644 acceptance/bundle/artifacts/same_name_libraries/script create mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py create mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py create mode 100644 bundle/libraries/same_name_libraries.go create mode 100644 bundle/libraries/same_name_libraries_test.go diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml new file mode 100644 index 000000000..a065bae76 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml @@ -0,0 +1,50 @@ +bundle: + name: same_name_libraries + +variables: + cluster: + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode + +artifacts: + whl1: + type: whl + path: ./whl1 + whl2: + type: whl + path: ./whl2 + +resources: + jobs: + test: + name: "test" + tasks: + - task_key: task1 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl + - task_key: task2 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl2/dist/*.whl + - task_key: task3 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt new file mode 100644 index 000000000..38cdd43c4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt @@ -0,0 +1,14 @@ + +>>> errcode [CLI] bundle deploy +Building whl1... +Building whl2... +Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl + at resources.jobs.test.tasks[0].libraries[0].whl + resources.jobs.test.tasks[1].libraries[0].whl + in databricks.yml:36:15 + databricks.yml:43:15 + +Local library names must be unique + + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script new file mode 100644 index 000000000..6c899df07 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/script @@ -0,0 +1,2 @@ +trace errcode $CLI bundle deploy +rm -rf whl1 whl2 diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index bb1905045..7a808f627 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic for _, match := range matches { output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.V(match), + libType: dyn.NewValue(match, lib.Locations()), }, lib.Locations())) } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go new file mode 100644 index 000000000..88b96ab54 --- /dev/null +++ b/bundle/libraries/same_name_libraries.go @@ -0,0 +1,97 @@ +package libraries + +import ( + "context" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type checkForSameNameLibraries struct{} + +var patterns = []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + envDepsPattern.Append(dyn.AnyIndex()), +} + +type libData struct { + fullPath string + locations []dyn.Location + paths []dyn.Path +} + +func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + libs := make(map[string]*libData) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + libPath := lv.MustString() + // If not local library, skip the check + if !IsLibraryLocal(libPath) { + return lv, nil + } + + libFullPath := lv.MustString() + lib := filepath.Base(libFullPath) + // If the same basename was seen already but full path is different + // then it's a duplicate. Add the location to the location list. + lp, ok := libs[lib] + if !ok { + libs[lib] = &libData{ + fullPath: libFullPath, + locations: []dyn.Location{lv.Location()}, + paths: []dyn.Path{p}, + } + } else if lp.fullPath != libFullPath { + lp.locations = append(lp.locations, lv.Location()) + lp.paths = append(lp.paths, p) + } + + return lv, nil + }) + if err != nil { + return dyn.InvalidValue, err + } + } + + if err != nil { + return dyn.InvalidValue, err + } + + return v, nil + }) + + // Iterate over all the libraries and check if there are any duplicates. + // Duplicates will have more than one location. + // If there are duplicates, add a diagnostic. + for lib, lv := range libs { + if len(lv.locations) > 1 { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Duplicate local library name " + lib, + Detail: "Local library names must be unique", + Locations: lv.locations, + Paths: lv.paths, + }) + } + } + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (c checkForSameNameLibraries) Name() string { + return "CheckForSameNameLibraries" +} + +func CheckForSameNameLibraries() bundle.Mutator { + return checkForSameNameLibraries{} +} diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go new file mode 100644 index 000000000..42c38773b --- /dev/null +++ b/bundle/libraries/same_name_libraries_test.go @@ -0,0 +1,121 @@ +package libraries + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestSameNameLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test.whl", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + }) + bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{ + {File: "databricks.yml", Line: 20, Column: 1}, + }) + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Len(t, diags, 1) + require.Equal(t, diag.Error, diags[0].Severity) + require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary) + require.Equal(t, []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + {File: "databricks.yml", Line: 20, Column: 1}, + }, diags[0].Locations) + + paths := make([]string, 0) + for _, p := range diags[0].Paths { + paths = append(paths, p.String()) + } + require.Equal(t, []string{ + "resources.jobs.test.tasks[0].libraries[0].whl", + "resources.jobs.test.tasks[1].libraries[0].whl", + }, paths) +} + +func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test-0.1.0.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Empty(t, diags) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c6ec04962..2e9211a7e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -155,6 +155,11 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { mutator.ValidateGitDetails(), artifacts.CleanUp(), libraries.ExpandGlobReferences(), + // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we + // know what are the actual library paths. + // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this + // mutator is part of the deploy step rather than validate. + libraries.CheckForSameNameLibraries(), libraries.Upload(), trampoline.TransformWheelTask(), files.Upload(outputHandler), From f8aaa7fce337ac95dba354ddb22cc5a8fa5de046 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 7 Feb 2025 18:37:03 +0000 Subject: [PATCH 16/21] Added support to generate Git based jobs (#2304) ## Changes This will generate bundle YAML configuration for Git based jobs but won't download any related files as they are in Git repo. Fixes #1423 ## Tests Added unit test --------- Co-authored-by: Pieter Noordhuis --- .../bundle/generate/git_job/databricks.yml | 2 + .../bundle/generate/git_job/out.job.yml | 17 +++++++ acceptance/bundle/generate/git_job/output.txt | 2 + acceptance/bundle/generate/git_job/script | 1 + acceptance/bundle/generate/git_job/test.toml | 33 +++++++++++++ bundle/config/generate/job.go | 1 - cmd/bundle/generate/job.go | 20 ++++++-- libs/dyn/yamlsaver/utils.go | 47 +++++++++++++++++- libs/dyn/yamlsaver/utils_test.go | 48 +++++++++++++++++++ 9 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/generate/git_job/databricks.yml create mode 100644 acceptance/bundle/generate/git_job/out.job.yml create mode 100644 acceptance/bundle/generate/git_job/output.txt create mode 100644 acceptance/bundle/generate/git_job/script create mode 100644 acceptance/bundle/generate/git_job/test.toml diff --git a/acceptance/bundle/generate/git_job/databricks.yml b/acceptance/bundle/generate/git_job/databricks.yml new file mode 100644 index 000000000..adaa7aab3 --- /dev/null +++ b/acceptance/bundle/generate/git_job/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: git_job diff --git a/acceptance/bundle/generate/git_job/out.job.yml b/acceptance/bundle/generate/git_job/out.job.yml new file mode 100644 index 000000000..0eb2a3fb1 --- /dev/null +++ b/acceptance/bundle/generate/git_job/out.job.yml @@ -0,0 +1,17 @@ +resources: + jobs: + out: + name: gitjob + tasks: + - task_key: test_task + notebook_task: + notebook_path: some/test/notebook.py + - task_key: test_task_2 + notebook_task: + notebook_path: /Workspace/Users/foo@bar.com/some/test/notebook.py + source: WORKSPACE + git_source: + git_branch: main + git_commit: abcdef + git_provider: github + git_url: https://git.databricks.com diff --git a/acceptance/bundle/generate/git_job/output.txt b/acceptance/bundle/generate/git_job/output.txt new file mode 100644 index 000000000..680c92ff9 --- /dev/null +++ b/acceptance/bundle/generate/git_job/output.txt @@ -0,0 +1,2 @@ +Job is using Git source, skipping downloading files +Job configuration successfully saved to out.job.yml diff --git a/acceptance/bundle/generate/git_job/script b/acceptance/bundle/generate/git_job/script new file mode 100644 index 000000000..7598966b0 --- /dev/null +++ b/acceptance/bundle/generate/git_job/script @@ -0,0 +1 @@ +$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml new file mode 100644 index 000000000..28b473245 --- /dev/null +++ b/acceptance/bundle/generate/git_job/test.toml @@ -0,0 +1,33 @@ +LocalOnly = true # This test needs to run against stubbed Databricks API + +[[Server]] +Pattern = "GET /api/2.1/jobs/get" +Response.Body = ''' +{ + "job_id": 11223344, + "settings": { + "name": "gitjob", + "git_source": { + "git_url": "https://git.databricks.com", + "git_provider": "github", + "git_branch": "main", + "git_commit": "abcdef" + }, + "tasks": [ + { + "task_key": "test_task", + "notebook_task": { + "notebook_path": "some/test/notebook.py" + } + }, + { + "task_key": "test_task_2", + "notebook_task": { + "source": "WORKSPACE", + "notebook_path": "/Workspace/Users/foo@bar.com/some/test/notebook.py" + } + } + ] + } +} +''' diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 0cdcbf3ad..934eda2cf 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -13,7 +13,6 @@ var ( func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { value := make(map[string]dyn.Value) - if job.Settings.Tasks != nil { tasks := make([]dyn.Value, 0) for _, task := range job.Settings.Tasks { diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index d97891cd5..438b235c9 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -50,10 +50,22 @@ func NewGenerateJobCommand() *cobra.Command { } downloader := newDownloader(w, sourceDir, configDir) - for _, task := range job.Settings.Tasks { - err := downloader.MarkTaskForDownload(ctx, &task) - if err != nil { - return err + + // Don't download files if the job is using Git source + // When Git source is used, the job will be using the files from the Git repository + // but specific tasks might override this behaviour by using `source: WORKSPACE` setting. + // In this case, we don't want to download the files as well for these specific tasks + // because it leads to confusion with relative paths between workspace and GIT files. + // Instead we keep these tasks as is and let the user handle the files manually. + // The configuration will be deployable as tasks paths for source: WORKSPACE tasks will be absolute workspace paths. + if job.Settings.GitSource != nil { + cmdio.LogString(ctx, "Job is using Git source, skipping downloading files") + } else { + for _, task := range job.Settings.Tasks { + err := downloader.MarkTaskForDownload(ctx, &task) + if err != nil { + return err + } } } diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index a162bf31f..c1b60b1b5 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -22,9 +22,50 @@ func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[str return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind()) } + mv, err = sortMapAlphabetically(mv) + if err != nil { + return dyn.InvalidValue, err + } + return skipAndOrder(mv, order, skipFields, dst) } +// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files. +func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { + sortedMap := dyn.NewMapping() + mapV := mv.MustMap() + keys := mapV.Keys() + slices.SortStableFunc(keys, func(i, j dyn.Value) int { + iKey := i.MustString() + jKey := j.MustString() + if iKey < jKey { + return -1 + } + + if iKey > jKey { + return 1 + } + return 0 + }) + + for _, key := range keys { + value, _ := mapV.Get(key) + var err error + if value.Kind() == dyn.KindMap { + value, err = sortMapAlphabetically(value) + if err != nil { + return dyn.InvalidValue, err + } + } + err = sortedMap.Set(key, value) + if err != nil { + return dyn.InvalidValue, err + } + } + + return dyn.V(sortedMap), nil +} + func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { for _, pair := range mv.MustMap().Pairs() { k := pair.Key.MustString() @@ -44,7 +85,11 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin continue } - dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + if order == nil { + dst[k] = v + } else { + dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + } } return dyn.V(dst), nil diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 1afab601a..f7ea3c96c 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -7,6 +7,54 @@ import ( assert "github.com/databricks/cli/libs/dyn/dynassert" ) +func TestConvertToMap(t *testing.T) { + type test struct { + Name string `json:"name"` + Map map[string]string `json:"map"` + List []string `json:"list"` + LongNameField string `json:"long_name_field"` + ForceSendFields []string `json:"-"` + Format string `json:"format"` + } + + v := &test{ + Name: "test", + Map: map[string]string{ + "key2": "value2", + "key1": "value1", + }, + List: []string{"a", "b", "c"}, + ForceSendFields: []string{ + "Name", + }, + LongNameField: "long name goes here", + } + result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) + assert.NoError(t, err) + assert.Equal(t, dyn.V(map[string]dyn.Value{ + "list": dyn.NewValue( + []dyn.Value{ + dyn.V("a"), + dyn.V("b"), + dyn.V("c"), + }, + []dyn.Location{}, + ), + "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{}), + "map": dyn.NewValue( + map[string]dyn.Value{ + "key1": dyn.V("value1"), + "key2": dyn.V("value2"), + }, + []dyn.Location{}, + ), + "name": dyn.NewValue( + "test", + []dyn.Location{}, + ), + }), result) +} + func TestConvertToMapValueWithOrder(t *testing.T) { type test struct { Name string `json:"name"` From 06e342afc58139264fabd34eec603d58f2b7f95a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 10:16:31 +0100 Subject: [PATCH 17/21] Silence a comment in Makefile (#2315) It was not indended to be printed. Follow up to #2298 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7da7e4789..fb3936184 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ lint: golangci-lint run --fix tidy: - # not part of golangci-lint, apparently + @# not part of golangci-lint, apparently go mod tidy lintcheck: From 2175dd24a4935963eea8ad1c95b72d309d4ea780 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:42:39 +0100 Subject: [PATCH 18/21] Do not gitignore .databricks and terraform (#2318) For acceptance/bundle/templates I'd like to run "bundle deploy". This would create .databricks directory inside materialized output. It might makes sense to commit some of this as part of golden files output. Even if we did not commit anything, the test runner will see those files and show the difference. Thus git should also see them. Also rename .gitignore to out.gitignore in those tests, since that includes .databricks as well. --- .gitignore | 4 ---- .../dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/dbt-sql/script | 3 +++ .../output/my_default_python/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/default-python/script | 3 +++ .../output/my_default_sql/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/default-sql/script | 3 +++ .../output/my_jobs_as_code/{.gitignore => out.gitignore} | 0 acceptance/bundle/templates/experimental-jobs-as-code/script | 3 +++ 9 files changed, 12 insertions(+), 4 deletions(-) rename acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/default-python/output/my_default_python/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/default-sql/output/my_default_sql/{.gitignore => out.gitignore} (100%) rename acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/{.gitignore => out.gitignore} (100%) diff --git a/.gitignore b/.gitignore index 2060b6bac..35aef1764 100644 --- a/.gitignore +++ b/.gitignore @@ -25,11 +25,7 @@ coverage-acceptance.txt __pycache__ *.pyc -.terraform -.terraform.lock.hcl - .vscode/launch.json .vscode/tasks.json -.databricks .ruff_cache diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script index c4ca817fe..3a2660de5 100644 --- a/acceptance/bundle/templates/dbt-sql/script +++ b/acceptance/bundle/templates/dbt-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output cd output/my_dbt_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.gitignore rename to acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/script index b11a7ea21..e5fcb7741 100644 --- a/acceptance/bundle/templates/default-python/script +++ b/acceptance/bundle/templates/default-python/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir ou cd output/my_default_python trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore rename to acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index 66e7a14a2..7ea0d863c 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-sql --config-file ./input.json --output-dir outpu cd output/my_default_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore similarity index 100% rename from acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore rename to acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index 10188aabd..08e48fc5f 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -8,3 +8,6 @@ uv sync -q trace $CLI bundle validate -t dev --output json | jq ".resources" rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore From cc073801855d0e0a27f32b25f2a89c1fa391dc8d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:53:00 +0100 Subject: [PATCH 19/21] acc: Summarize unexpected files (#2320) ## Changes When there are many unexpected files, it's good to see them as a list rather than scattered throughout the output. ## Tests Manually, example output: ``` acceptance_test.go:363: Test produced unexpected files: output/my_default_sql/.databricks/bundle/dev/sync-snapshots/71c79ded90615dc7.json output/my_default_sql/.databricks/bundle/dev/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64 output/my_default_sql/.databricks/bundle/dev/terraform/plan output/my_default_sql/.databricks/bundle/prod/sync-snapshots/83e677e75259c93b.json output/my_default_sql/.databricks/bundle/prod/terraform/.terraform/providers/registry.terraform.io/databricks/databricks/1.64.1/darwin_arm64 ``` --- acceptance/acceptance_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 4c4404d55..241ab42be 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -343,6 +343,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // Make sure there are not unaccounted for new files files := ListDir(t, tmpDir) + unexpected := []string{} for _, relPath := range files { if _, ok := inputs[relPath]; ok { continue @@ -350,13 +351,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := outputs[relPath]; ok { continue } - t.Errorf("Unexpected output: %s", relPath) + unexpected = append(unexpected, relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } } + + if len(unexpected) > 0 { + t.Error("Test produced unexpected files:\n" + strings.Join(unexpected, "\n")) + } } func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { From 4ebc86282fbbaec9293af022a8d9f5b9b0b80a4a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Feb 2025 11:55:34 +0100 Subject: [PATCH 20/21] acc: Split bundle/templates and bundle/templates-machinery (#2317) The tests in acceptance/bundle/template focus on standard templates. The plan is to extend them with "bundle deploy" and enable them on the cloud. The tests in acceptance/bundle/template-machinery focus on specific aspects of template implementation. Most of them are expected to remain local-only. --- .../helpers-error/databricks_template_schema.json | 0 .../{templates => templates-machinery}/helpers-error/output.txt | 0 .../{templates => templates-machinery}/helpers-error/script | 0 .../helpers-error/template/helpers.txt.tmpl | 0 .../{templates => templates-machinery}/helpers-error/test.toml | 0 .../helpers/databricks_template_schema.json | 0 .../{templates => templates-machinery}/helpers/output.txt | 0 .../bundle/{templates => templates-machinery}/helpers/script | 0 .../helpers/template/helpers.txt.tmpl | 0 .../bundle/{templates => templates-machinery}/helpers/test.toml | 0 acceptance/bundle/templates-machinery/test.toml | 2 ++ .../{templates => templates-machinery}/wrong-path/output.txt | 0 .../bundle/{templates => templates-machinery}/wrong-path/script | 0 .../{templates => templates-machinery}/wrong-path/test.toml | 0 .../{templates => templates-machinery}/wrong-url/output.txt | 0 .../bundle/{templates => templates-machinery}/wrong-url/script | 0 .../{templates => templates-machinery}/wrong-url/test.toml | 0 17 files changed, 2 insertions(+) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/databricks_template_schema.json (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/script (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/template/helpers.txt.tmpl (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers-error/test.toml (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/databricks_template_schema.json (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/script (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/template/helpers.txt.tmpl (100%) rename acceptance/bundle/{templates => templates-machinery}/helpers/test.toml (100%) create mode 100644 acceptance/bundle/templates-machinery/test.toml rename acceptance/bundle/{templates => templates-machinery}/wrong-path/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-path/script (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-path/test.toml (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/output.txt (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/script (100%) rename acceptance/bundle/{templates => templates-machinery}/wrong-url/test.toml (100%) diff --git a/acceptance/bundle/templates/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json similarity index 100% rename from acceptance/bundle/templates/helpers-error/databricks_template_schema.json rename to acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json diff --git a/acceptance/bundle/templates/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt similarity index 100% rename from acceptance/bundle/templates/helpers-error/output.txt rename to acceptance/bundle/templates-machinery/helpers-error/output.txt diff --git a/acceptance/bundle/templates/helpers-error/script b/acceptance/bundle/templates-machinery/helpers-error/script similarity index 100% rename from acceptance/bundle/templates/helpers-error/script rename to acceptance/bundle/templates-machinery/helpers-error/script diff --git a/acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl similarity index 100% rename from acceptance/bundle/templates/helpers-error/template/helpers.txt.tmpl rename to acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl diff --git a/acceptance/bundle/templates/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml similarity index 100% rename from acceptance/bundle/templates/helpers-error/test.toml rename to acceptance/bundle/templates-machinery/helpers-error/test.toml diff --git a/acceptance/bundle/templates/helpers/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json similarity index 100% rename from acceptance/bundle/templates/helpers/databricks_template_schema.json rename to acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json diff --git a/acceptance/bundle/templates/helpers/output.txt b/acceptance/bundle/templates-machinery/helpers/output.txt similarity index 100% rename from acceptance/bundle/templates/helpers/output.txt rename to acceptance/bundle/templates-machinery/helpers/output.txt diff --git a/acceptance/bundle/templates/helpers/script b/acceptance/bundle/templates-machinery/helpers/script similarity index 100% rename from acceptance/bundle/templates/helpers/script rename to acceptance/bundle/templates-machinery/helpers/script diff --git a/acceptance/bundle/templates/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl similarity index 100% rename from acceptance/bundle/templates/helpers/template/helpers.txt.tmpl rename to acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl diff --git a/acceptance/bundle/templates/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml similarity index 100% rename from acceptance/bundle/templates/helpers/test.toml rename to acceptance/bundle/templates-machinery/helpers/test.toml diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml new file mode 100644 index 000000000..9083ecd1b --- /dev/null +++ b/acceptance/bundle/templates-machinery/test.toml @@ -0,0 +1,2 @@ +# Testing template machinery, by default there is no need to check against cloud. +LocalOnly = true diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates-machinery/wrong-path/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-path/output.txt rename to acceptance/bundle/templates-machinery/wrong-path/output.txt diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates-machinery/wrong-path/script similarity index 100% rename from acceptance/bundle/templates/wrong-path/script rename to acceptance/bundle/templates-machinery/wrong-path/script diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates-machinery/wrong-path/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-path/test.toml rename to acceptance/bundle/templates-machinery/wrong-path/test.toml diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates-machinery/wrong-url/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-url/output.txt rename to acceptance/bundle/templates-machinery/wrong-url/output.txt diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates-machinery/wrong-url/script similarity index 100% rename from acceptance/bundle/templates/wrong-url/script rename to acceptance/bundle/templates-machinery/wrong-url/script diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates-machinery/wrong-url/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-url/test.toml rename to acceptance/bundle/templates-machinery/wrong-url/test.toml From ee440e65fec623fbf3e4cba05ac21d67ee6306db Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:48:05 +0530 Subject: [PATCH 21/21] Serialize all header values in acceptance tests (#2311) ## Changes Based on feedback in https://github.com/databricks/cli/pull/2296#discussion_r1946660650. Previously we only serialized the first value for a header in the requests log. Now we serialise all values for a header key. ## Tests Existing test --- .../workspace/jobs/create/out.requests.txt | 2 +- libs/testserver/server.go | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 60977e3e3..2510762db 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"headers":{"Authorization":"Bearer [DATABRICKS_TOKEN]","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 9ccf34be0..d0c340c12 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -31,10 +31,10 @@ type Server struct { } type Request struct { - Headers map[string]string `json:"headers,omitempty"` - Method string `json:"method"` - Path string `json:"path"` - Body any `json:"body"` + Headers http.Header `json:"headers,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -109,12 +109,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { body, err := io.ReadAll(r.Body) assert.NoError(s.t, err) - headers := make(map[string]string) + headers := make(http.Header) for k, v := range r.Header { - if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) { + if !slices.Contains(s.IncludeRequestHeaders, k) { continue } - headers[k] = v[0] + for _, vv := range v { + headers.Add(k, vv) + } } s.Requests = append(s.Requests, Request{