diff --git a/.gitignore b/.gitignore index 2060b6bac..35aef1764 100644 --- a/.gitignore +++ b/.gitignore @@ -25,11 +25,7 @@ coverage-acceptance.txt __pycache__ *.pyc -.terraform -.terraform.lock.hcl - .vscode/launch.json .vscode/tasks.json -.databricks .ruff_cache diff --git a/Makefile b/Makefile index e18727934..fb3936184 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -default: vendor fmt lint +default: vendor fmt lint tidy PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... . @@ -9,6 +9,10 @@ GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped lint: golangci-lint run --fix +tidy: + @# not part of golangci-lint, apparently + go mod tidy + lintcheck: golangci-lint run ./... @@ -59,4 +63,4 @@ integration: vendor integration-short: vendor $(INTEGRATION) -short -.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs +.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs diff --git a/NOTICE b/NOTICE index ed22084cf..4331a2a32 100644 --- a/NOTICE +++ b/NOTICE @@ -109,3 +109,8 @@ License - https://github.com/hexops/gotextdiff/blob/main/LICENSE https://github.com/BurntSushi/toml Copyright (c) 2013 TOML authors https://github.com/BurntSushi/toml/blob/master/COPYING + +dario.cat/mergo +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. +https://github.com/darccio/mergo/blob/master/LICENSE diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f205217ff..241ab42be 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -19,6 +19,8 @@ import ( "time" "unicode/utf8" + "github.com/google/uuid" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testdiff" @@ -123,7 +125,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { AddHandlers(defaultServer) // Redirect API access to local server: t.Setenv("DATABRICKS_HOST", defaultServer.URL) - t.Setenv("DATABRICKS_TOKEN", "dapi1234") homeDir := t.TempDir() // Do not read user's ~/.databrickscfg @@ -146,7 +147,15 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") - workspaceClient, err := databricks.NewWorkspaceClient() + var config databricks.Config + if cloudEnv == "" { + // use fake token for local tests + config = databricks.Config{Token: "dbapi1234"} + } else { + // non-local tests rely on environment variables + config = databricks.Config{} + } + workspaceClient, err := databricks.NewWorkspaceClient(&config) require.NoError(t, err) user, err := workspaceClient.CurrentUser.Me(ctx) @@ -264,7 +273,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont for _, stub := range config.Server { require.NotEmpty(t, stub.Pattern) - server.Handle(stub.Pattern, func(req *http.Request) (any, int) { + server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) { statusCode := http.StatusOK if stub.Response.StatusCode != 0 { statusCode = stub.Response.StatusCode @@ -285,6 +294,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) } + // Each local test should use a new token that will result into a new fake workspace, + // so that test don't interfere with each other. + if cloudEnv == "" { + tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "") + token := "dbapi" + tokenSuffix + cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token) + repls.Set(token, "[DATABRICKS_TOKEN]") + } + // Write combined output to a file out, err := os.Create(filepath.Join(tmpDir, "output.txt")) require.NoError(t, err) @@ -303,8 +321,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont reqJson, err := json.Marshal(req) require.NoError(t, err) - line := fmt.Sprintf("%s\n", reqJson) - _, err = f.WriteString(line) + reqJsonWithRepls := repls.Replace(string(reqJson)) + _, err = f.WriteString(reqJsonWithRepls + "\n") require.NoError(t, err) } @@ -325,6 +343,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont // Make sure there are not unaccounted for new files files := ListDir(t, tmpDir) + unexpected := []string{} for _, relPath := range files { if _, ok := inputs[relPath]; ok { continue @@ -332,13 +351,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := outputs[relPath]; ok { continue } - t.Errorf("Unexpected output: %s", relPath) + unexpected = append(unexpected, relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } } + + if len(unexpected) > 0 { + t.Error("Test produced unexpected files:\n" + strings.Join(unexpected, "\n")) + } } func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py new file mode 100755 index 000000000..9ac87feee --- /dev/null +++ b/acceptance/bin/sort_lines.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +""" +Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH. +""" + +import sys + +lines = sys.stdin.readlines() +lines.sort() +sys.stdout.write("".join(lines)) diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml new file mode 100644 index 000000000..a065bae76 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml @@ -0,0 +1,50 @@ +bundle: + name: same_name_libraries + +variables: + cluster: + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode + +artifacts: + whl1: + type: whl + path: ./whl1 + whl2: + type: whl + path: ./whl2 + +resources: + jobs: + test: + name: "test" + tasks: + - task_key: task1 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl + - task_key: task2 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl2/dist/*.whl + - task_key: task3 + new_cluster: ${var.cluster} + python_wheel_task: + entry_point: main + package_name: my_default_python + libraries: + - whl: ./whl1/dist/*.whl diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt new file mode 100644 index 000000000..38cdd43c4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt @@ -0,0 +1,14 @@ + +>>> errcode [CLI] bundle deploy +Building whl1... +Building whl2... +Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl + at resources.jobs.test.tasks[0].libraries[0].whl + resources.jobs.test.tasks[1].libraries[0].whl + in databricks.yml:36:15 + databricks.yml:43:15 + +Local library names must be unique + + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script new file mode 100644 index 000000000..6c899df07 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/script @@ -0,0 +1,2 @@ +trace errcode $CLI bundle deploy +rm -rf whl1 whl2 diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py new file mode 100644 index 000000000..1afaf3a4f --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py @@ -0,0 +1,36 @@ +""" +setup.py configuration script describing how to build and package this project. + +This file is primarily used by the setuptools library and typically should not +be executed directly. See README.md for how to deploy, test, and run +the my_default_python project. +""" + +from setuptools import setup, find_packages + +import sys + +sys.path.append("./src") + +import my_default_python + +setup( + name="my_default_python", + version=my_default_python.__version__, + url="https://databricks.com", + author="[USERNAME]", + description="wheel file based on my_default_python/src", + packages=find_packages(where="./src"), + package_dir={"": "src"}, + entry_points={ + "packages": [ + "main=my_default_python.main:main", + ], + }, + install_requires=[ + # Dependencies in case the output wheel file is used as a library dependency. + # For defining dependencies, when this package is used in Databricks, see: + # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + "setuptools" + ], +) diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py new file mode 100644 index 000000000..f102a9cad --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py new file mode 100644 index 000000000..11b15b1a4 --- /dev/null +++ b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/debug/databricks.yml b/acceptance/bundle/debug/databricks.yml new file mode 100644 index 000000000..2c9dd3c90 --- /dev/null +++ b/acceptance/bundle/debug/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: debug diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt new file mode 100644 index 000000000..7dd770068 --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.parallel.txt @@ -0,0 +1,15 @@ +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions +10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync +10:07:59 Debug: non-retriable error: pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true +< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt new file mode 100644 index 000000000..9cac8bb2b --- /dev/null +++ b/acceptance/bundle/debug/out.stderr.txt @@ -0,0 +1,92 @@ +10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug" +10:07:59 Debug: Found bundle root at [TMPDIR] (file [TMPDIR]/databricks.yml) pid=12345 +10:07:59 Debug: Apply pid=12345 mutator=load +10:07:59 Info: Phase: load pid=12345 mutator=load +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EntryPoint +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=load mutator=seq mutator=scripts.preinit +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=VerifyCliVersion +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EnvironmentsToTargets +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ComputeIdToClusterId +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=InitializeVariables +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=DefineDefaultTarget(default) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=PythonMutator(load) +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=validate:unique_resource_keys +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget +10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget mutator=SelectTarget(default) +10:07:59 Debug: Apply pid=12345 mutator= +10:07:59 Debug: Apply pid=12345 mutator=initialize +10:07:59 Info: Phase: initialize pid=12345 mutator=initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=validate:AllResourcesHaveValues +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteSyncPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncDefaultPath +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncInferRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser +10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me +< HTTP/1.1 200 OK +< { +< "id": "[USERID]", +< "userName": "[USERNAME]" +< } pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser sdk=true +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=LoadGitDetails +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplySourceLinkedDeploymentPreset +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefineDefaultWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandWorkspaceRoot +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultWorkspacePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PrependWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteWorkspacePrefix +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetVariables +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(init) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(load_resources) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(apply_mutators) +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveResourceReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobParameters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobTasks +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergePipelineClusters +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeApps +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CaptureSchemaDependency +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CheckPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetRunAs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=OverrideCompute +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureDashboardDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureVolumeDefaults +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ProcessTargetMode +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyPresets +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultQueueing +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandPipelineGlobPaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureWSFS +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=TranslatePaths +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonWrapperWarning +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=apps.Validate +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ValidateSharedRootPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyBundlePermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=FilterCurrentUserFromPermissions +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotateJobs +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotatePipelines +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize +10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit +10:07:59 Debug: Apply pid=12345 mutator=validate +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 404 Not Found +10:07:59 Debug: POST /api/2.0/workspace/mkdirs +> { +> "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +> } +< HTTP/1.1 200 OK +10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files +< HTTP/1.1 200 OK +< { +< "object_type": "DIRECTORY", +< "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +10:07:59 Info: completed execution pid=12345 exit_code=0 diff --git a/acceptance/bundle/debug/output.txt b/acceptance/bundle/debug/output.txt new file mode 100644 index 000000000..ed72b360e --- /dev/null +++ b/acceptance/bundle/debug/output.txt @@ -0,0 +1,7 @@ +Name: debug +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/debug/default + +Validation OK! diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script new file mode 100644 index 000000000..005a1a341 --- /dev/null +++ b/acceptance/bundle/debug/script @@ -0,0 +1,4 @@ +$CLI bundle validate --debug 2> full.stderr.txt +grep -vw parallel full.stderr.txt > out.stderr.txt +grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt +rm full.stderr.txt diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml new file mode 100644 index 000000000..bb0fcb395 --- /dev/null +++ b/acceptance/bundle/debug/test.toml @@ -0,0 +1,18 @@ +LocalOnly = true + +[[Repls]] +# The keys are unsorted and also vary per OS +Old = 'Environment variables for Terraform: ([A-Z_ ,]+) ' +New = 'Environment variables for Terraform: ...redacted... ' + +[[Repls]] +Old = 'pid=[0-9]+' +New = 'pid=12345' + +[[Repls]] +Old = '\d\d:\d\d:\d\d' +New = '10:07:59' + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/generate/git_job/databricks.yml b/acceptance/bundle/generate/git_job/databricks.yml new file mode 100644 index 000000000..adaa7aab3 --- /dev/null +++ b/acceptance/bundle/generate/git_job/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: git_job diff --git a/acceptance/bundle/generate/git_job/out.job.yml b/acceptance/bundle/generate/git_job/out.job.yml new file mode 100644 index 000000000..0eb2a3fb1 --- /dev/null +++ b/acceptance/bundle/generate/git_job/out.job.yml @@ -0,0 +1,17 @@ +resources: + jobs: + out: + name: gitjob + tasks: + - task_key: test_task + notebook_task: + notebook_path: some/test/notebook.py + - task_key: test_task_2 + notebook_task: + notebook_path: /Workspace/Users/foo@bar.com/some/test/notebook.py + source: WORKSPACE + git_source: + git_branch: main + git_commit: abcdef + git_provider: github + git_url: https://git.databricks.com diff --git a/acceptance/bundle/generate/git_job/output.txt b/acceptance/bundle/generate/git_job/output.txt new file mode 100644 index 000000000..680c92ff9 --- /dev/null +++ b/acceptance/bundle/generate/git_job/output.txt @@ -0,0 +1,2 @@ +Job is using Git source, skipping downloading files +Job configuration successfully saved to out.job.yml diff --git a/acceptance/bundle/generate/git_job/script b/acceptance/bundle/generate/git_job/script new file mode 100644 index 000000000..7598966b0 --- /dev/null +++ b/acceptance/bundle/generate/git_job/script @@ -0,0 +1 @@ +$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml new file mode 100644 index 000000000..28b473245 --- /dev/null +++ b/acceptance/bundle/generate/git_job/test.toml @@ -0,0 +1,33 @@ +LocalOnly = true # This test needs to run against stubbed Databricks API + +[[Server]] +Pattern = "GET /api/2.1/jobs/get" +Response.Body = ''' +{ + "job_id": 11223344, + "settings": { + "name": "gitjob", + "git_source": { + "git_url": "https://git.databricks.com", + "git_provider": "github", + "git_branch": "main", + "git_commit": "abcdef" + }, + "tasks": [ + { + "task_key": "test_task", + "notebook_task": { + "notebook_path": "some/test/notebook.py" + } + }, + { + "task_key": "test_task_2", + "notebook_task": { + "source": "WORKSPACE", + "notebook_path": "/Workspace/Users/foo@bar.com/some/test/notebook.py" + } + } + ] + } +} +''' diff --git a/acceptance/bundle/git-permerror/output.txt b/acceptance/bundle/git-permerror/output.txt index 03ab93442..730e8255b 100644 --- a/acceptance/bundle/git-permerror/output.txt +++ b/acceptance/bundle/git-permerror/output.txt @@ -3,6 +3,7 @@ >>> chmod 000 .git >>> [CLI] bundle validate +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied Name: git-permerror @@ -16,6 +17,7 @@ Found 1 error Exit code: 1 >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -25,6 +27,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -39,11 +42,15 @@ Exit code: 1 >>> chmod 000 .git/HEAD >>> [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to load current branch: open HEAD: permission denied +Warn: failed to load latest commit: open HEAD: permission denied { "bundle_root_path": "." } @@ -54,6 +61,7 @@ Exit code: 1 >>> chmod 000 .git/config >>> [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied @@ -63,6 +71,7 @@ Exit code: 1 } >>> withdir subdir/a/b [CLI] bundle validate -o json +Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied Error: unable to load repository specific gitconfig: open config: permission denied diff --git a/acceptance/bundle/git-permerror/test.toml b/acceptance/bundle/git-permerror/test.toml index 3f96e551c..15305cff1 100644 --- a/acceptance/bundle/git-permerror/test.toml +++ b/acceptance/bundle/git-permerror/test.toml @@ -1,4 +1,4 @@ -Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct" +Badness = "inferred flag is set to true incorrect; bundle_root_path is not correct; Warn and Error talk about the same; Warn goes to stderr, Error goes to stdout (for backward compat); Warning about permissions repeated twice" [GOOS] # This test relies on chmod which does not work on Windows diff --git a/acceptance/bundle/scripts/output.txt b/acceptance/bundle/scripts/output.txt index 2deedb0e7..68afb2fec 100644 --- a/acceptance/bundle/scripts/output.txt +++ b/acceptance/bundle/scripts/output.txt @@ -42,11 +42,9 @@ from myscript.py 0 postbuild: hello stderr! Executing 'predeploy' script from myscript.py 0 predeploy: hello stdout! from myscript.py 0 predeploy: hello stderr! -Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME]. -Please make sure the current user or one of their groups is listed under the permissions of this bundle. -For assistance, contact the owners of this project. -They may need to redeploy the bundle to apply the new permissions. -Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions. - - -Exit code: 1 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/scripts/default/files... +Deploying resources... +Deployment complete! +Executing 'postdeploy' script +from myscript.py 0 postdeploy: hello stdout! +from myscript.py 0 postdeploy: hello stderr! diff --git a/acceptance/bundle/scripts/test.toml b/acceptance/bundle/scripts/test.toml deleted file mode 100644 index 1dbd78681..000000000 --- a/acceptance/bundle/scripts/test.toml +++ /dev/null @@ -1 +0,0 @@ -LocalOnly = true # Deployment currently fails when run locally; once that is fixed, remove this setting diff --git a/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates-machinery/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt new file mode 100644 index 000000000..6020e944f --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/output.txt @@ -0,0 +1,3 @@ +Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: + +Exit code: 1 diff --git a/acceptance/bundle/templates-machinery/helpers-error/script b/acceptance/bundle/templates-machinery/helpers-error/script new file mode 100644 index 000000000..d9fcbf62c --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/script @@ -0,0 +1 @@ +$CLI bundle init . diff --git a/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml new file mode 100644 index 000000000..77f4ed94b --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/test.toml @@ -0,0 +1,7 @@ +Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' +LocalOnly = true + +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/Me" +Response.Body = '{}' +Response.StatusCode = 500 diff --git a/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json b/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/templates-machinery/helpers/output.txt b/acceptance/bundle/templates-machinery/helpers/output.txt new file mode 100644 index 000000000..3126ea5af --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers/output.txt @@ -0,0 +1,2 @@ +✨ Successfully initialized template +user_name: [USERNAME] diff --git a/acceptance/bundle/templates-machinery/helpers/script b/acceptance/bundle/templates-machinery/helpers/script new file mode 100644 index 000000000..1773e7b03 --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers/script @@ -0,0 +1,3 @@ +$CLI bundle init . +cat helpers.txt +rm helpers.txt diff --git a/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl b/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl new file mode 100644 index 000000000..70e60edac --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl @@ -0,0 +1 @@ +user_name: {{ user_name }} diff --git a/acceptance/bundle/templates-machinery/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml new file mode 100644 index 000000000..b76e712fb --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers/test.toml @@ -0,0 +1 @@ +LocalOnly = true diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml new file mode 100644 index 000000000..9083ecd1b --- /dev/null +++ b/acceptance/bundle/templates-machinery/test.toml @@ -0,0 +1,2 @@ +# Testing template machinery, by default there is no need to check against cloud. +LocalOnly = true diff --git a/acceptance/bundle/templates/wrong-path/output.txt b/acceptance/bundle/templates-machinery/wrong-path/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-path/output.txt rename to acceptance/bundle/templates-machinery/wrong-path/output.txt diff --git a/acceptance/bundle/templates/wrong-path/script b/acceptance/bundle/templates-machinery/wrong-path/script similarity index 100% rename from acceptance/bundle/templates/wrong-path/script rename to acceptance/bundle/templates-machinery/wrong-path/script diff --git a/acceptance/bundle/templates/wrong-path/test.toml b/acceptance/bundle/templates-machinery/wrong-path/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-path/test.toml rename to acceptance/bundle/templates-machinery/wrong-path/test.toml diff --git a/acceptance/bundle/templates/wrong-url/output.txt b/acceptance/bundle/templates-machinery/wrong-url/output.txt similarity index 100% rename from acceptance/bundle/templates/wrong-url/output.txt rename to acceptance/bundle/templates-machinery/wrong-url/output.txt diff --git a/acceptance/bundle/templates/wrong-url/script b/acceptance/bundle/templates-machinery/wrong-url/script similarity index 100% rename from acceptance/bundle/templates/wrong-url/script rename to acceptance/bundle/templates-machinery/wrong-url/script diff --git a/acceptance/bundle/templates/wrong-url/test.toml b/acceptance/bundle/templates-machinery/wrong-url/test.toml similarity index 100% rename from acceptance/bundle/templates/wrong-url/test.toml rename to acceptance/bundle/templates-machinery/wrong-url/test.toml diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/.gitignore rename to acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script index c4ca817fe..3a2660de5 100644 --- a/acceptance/bundle/templates/dbt-sql/script +++ b/acceptance/bundle/templates/dbt-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output cd output/my_dbt_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-python/output/my_default_python/.gitignore rename to acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/script index b11a7ea21..e5fcb7741 100644 --- a/acceptance/bundle/templates/default-python/script +++ b/acceptance/bundle/templates/default-python/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir ou cd output/my_default_python trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore similarity index 100% rename from acceptance/bundle/templates/default-sql/output/my_default_sql/.gitignore rename to acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index 66e7a14a2..7ea0d863c 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -3,3 +3,6 @@ trace $CLI bundle init default-sql --config-file ./input.json --output-dir outpu cd output/my_default_sql trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore similarity index 100% rename from acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/.gitignore rename to acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index 10188aabd..08e48fc5f 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -8,3 +8,6 @@ uv sync -q trace $CLI bundle validate -t dev --output json | jq ".resources" rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore diff --git a/acceptance/bundle/variables/env_overrides/test.toml b/acceptance/bundle/variables/env_overrides/test.toml deleted file mode 100644 index 439c2fab1..000000000 --- a/acceptance/bundle/variables/env_overrides/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-builtin/test.toml b/acceptance/bundle/variables/resolve-builtin/test.toml deleted file mode 100644 index 085fab6c0..000000000 --- a/acceptance/bundle/variables/resolve-builtin/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml deleted file mode 100644 index d833bd848..000000000 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/' -LocalOnly = true diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml new file mode 100644 index 000000000..32398e828 --- /dev/null +++ b/acceptance/bundle/variables/test.toml @@ -0,0 +1,3 @@ +# The tests here intend to test variable interpolation via "bundle validate". +# Even though "bundle validate" does a few API calls, that's not the focus there. +LocalOnly = true diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go index 04d56c7d4..c8a52f4cd 100644 --- a/acceptance/cmd_server_test.go +++ b/acceptance/cmd_server_test.go @@ -15,7 +15,10 @@ import ( func StartCmdServer(t *testing.T) *testserver.Server { server := testserver.New(t) - server.Handle("/", func(r *http.Request) (any, int) { + // {$} is a wildcard that only matches the end of the URL. We explicitly use + // /{$} to disambiguate it from the generic handler for '/' which is used to + // identify unhandled API endpoints in the test server. + server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) { q := r.URL.Query() args := strings.Split(q.Get("args"), " ") diff --git a/acceptance/config_test.go b/acceptance/config_test.go index e24a683e7..920e713a1 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -3,9 +3,11 @@ package acceptance_test import ( "os" "path/filepath" - "sync" + "slices" + "strings" "testing" + "dario.cat/mergo" "github.com/BurntSushi/toml" "github.com/databricks/cli/libs/testdiff" "github.com/stretchr/testify/require" @@ -13,11 +15,6 @@ import ( const configFilename = "test.toml" -var ( - configCache map[string]TestConfig - configMutex sync.Mutex -) - type TestConfig struct { // Place to describe what's wrong with this test. Does not affect how the test is run. Badness string @@ -65,58 +62,55 @@ type ServerStub struct { } } -// FindConfig finds the closest config file. -func FindConfig(t *testing.T, dir string) (string, bool) { - shared := false +// FindConfigs finds all the config relevant for this test, +// ordered from the most outermost (at acceptance/) to current test directory (identified by dir). +// Argument dir must be a relative path from the root of acceptance tests (/acceptance/). +func FindConfigs(t *testing.T, dir string) []string { + configs := []string{} for { path := filepath.Join(dir, configFilename) _, err := os.Stat(path) if err == nil { - return path, shared + configs = append(configs, path) } - shared = true - if dir == "" || dir == "." { break } - if os.IsNotExist(err) { - dir = filepath.Dir(dir) + dir = filepath.Dir(dir) + + if err == nil || os.IsNotExist(err) { continue } t.Fatalf("Error while reading %s: %s", path, err) } - t.Fatal("Config not found: " + configFilename) - return "", shared + slices.Reverse(configs) + return configs } // LoadConfig loads the config file. Non-leaf configs are cached. func LoadConfig(t *testing.T, dir string) (TestConfig, string) { - path, leafConfig := FindConfig(t, dir) + configs := FindConfigs(t, dir) - if leafConfig { - return DoLoadConfig(t, path), path + if len(configs) == 0 { + return TestConfig{}, "(no config)" } - configMutex.Lock() - defer configMutex.Unlock() + result := DoLoadConfig(t, configs[0]) - if configCache == nil { - configCache = make(map[string]TestConfig) + for _, cfgName := range configs[1:] { + cfg := DoLoadConfig(t, cfgName) + err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice) + if err != nil { + t.Fatalf("Error during config merge: %s: %s", cfgName, err) + } } - result, ok := configCache[path] - if ok { - return result, path - } - - result = DoLoadConfig(t, path) - configCache[path] = result - return result, path + return result, strings.Join(configs, ", ") } func DoLoadConfig(t *testing.T, path string) TestConfig { diff --git a/acceptance/server_test.go b/acceptance/server_test.go index d714bdcd2..556a4a1d8 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -1,17 +1,23 @@ package acceptance_test import ( + "bytes" + "encoding/json" + "fmt" "net/http" - "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + + "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/service/workspace" ) func AddHandlers(server *testserver.Server) { - server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListPoliciesResponse{ Policies: []compute.Policy{ { @@ -26,7 +32,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListInstancePools{ InstancePools: []compute.InstancePoolAndStats{ { @@ -37,7 +43,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -52,31 +58,74 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return iam.User{ Id: "1000012345", UserName: "tester@databricks.com", }, http.StatusOK }) - server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) { - return workspace.ObjectInfo{ - ObjectId: 1001, - ObjectType: "DIRECTORY", - Path: "", - ResourceId: "1001", - }, http.StatusOK + server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceGetStatus(path) }) - server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) { + server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := workspace.Mkdirs{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceMkdirs(request) + }) + + server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + + return fakeWorkspace.WorkspaceExport(path) + }) + + server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.URL.Query().Get("path") + recursiveStr := r.URL.Query().Get("recursive") + var recursive bool + + if recursiveStr == "true" { + recursive = true + } else { + recursive = false + } + + return fakeWorkspace.WorkspaceDelete(path, recursive) + }) + + server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + path := r.PathValue("path") + + body := new(bytes.Buffer) + _, err := body.ReadFrom(r.Body) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes()) + }) + + server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { return catalog.MetastoreAssignment{ DefaultCatalogName: "main", }, http.StatusOK }) - server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) { + server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + objectId := r.PathValue("objectId") + return workspace.WorkspaceObjectPermissions{ - ObjectId: "1001", + ObjectId: objectId, ObjectType: "DIRECTORY", AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ { @@ -91,8 +140,26 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) - server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) { - return "{}", http.StatusOK + server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + request := jobs.CreateJob{} + decoder := json.NewDecoder(r.Body) + + err := decoder.Decode(&request) + if err != nil { + return internalError(err) + } + + return fakeWorkspace.JobsCreate(request) + }) + + server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + jobId := r.URL.Query().Get("job_id") + + return fakeWorkspace.JobsGet(jobId) + }) + + server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) { + return fakeWorkspace.JobsList() }) server.Handle("GET /oidc/.well-known/oauth-authorization-server", func(r *http.Request) (any, int) { @@ -111,3 +178,7 @@ func AddHandlers(server *testserver.Server) { }, http.StatusOK }) } + +func internalError(err error) (any, int) { + return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError +} diff --git a/acceptance/test.toml b/acceptance/test.toml deleted file mode 100644 index eee94d0ea..000000000 --- a/acceptance/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration. -# The configurations are not merged across parents; the closest one is used fully. diff --git a/acceptance/workspace/jobs/create/out.requests.txt b/acceptance/workspace/jobs/create/out.requests.txt index 4a85c4c43..2510762db 100644 --- a/acceptance/workspace/jobs/create/out.requests.txt +++ b/acceptance/workspace/jobs/create/out.requests.txt @@ -1 +1 @@ -{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} +{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}} diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index c69939e8c..d4625d85d 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -29,7 +29,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics // We intentionally ignore the error because it is not critical to the deployment err := client.Delete(ctx, ".", filer.DeleteRecursively) if err != nil { - log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err) + log.Debugf(ctx, "failed to delete %s: %v", uploadPath, err) } err = client.Mkdir(ctx, ".") diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 0cdcbf3ad..934eda2cf 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -13,7 +13,6 @@ var ( func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { value := make(map[string]dyn.Value) - if job.Settings.Tasks != nil { tasks := make([]dyn.Value, 0) for _, task := range job.Settings.Tasks { diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index b402053e7..c8e7bf9e8 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -84,7 +84,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Pipelines presets: Prefix, PipelinesDevelopment for key, p := range r.Pipelines { - if p.PipelineSpec == nil { + if p.CreatePipeline == nil { diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) continue } diff --git a/bundle/config/mutator/capture_schema_dependency.go b/bundle/config/mutator/capture_schema_dependency.go index 5025c9a0d..2e17a8175 100644 --- a/bundle/config/mutator/capture_schema_dependency.go +++ b/bundle/config/mutator/capture_schema_dependency.go @@ -56,7 +56,7 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) { } func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Schema == "" { @@ -71,7 +71,7 @@ func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) { } func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) { - if p == nil || p.PipelineSpec == nil { + if p == nil || p.CreatePipeline == nil { return } if p.Target == "" { diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go index 0a94e7748..16fa636ee 100644 --- a/bundle/config/mutator/capture_schema_dependency_test.go +++ b/bundle/config/mutator/capture_schema_dependency_test.go @@ -118,43 +118,43 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Schema: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Schema: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Schema: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Schema: "", Name: "whatever", @@ -179,7 +179,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema) assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"]) - assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec) + assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].CreatePipeline) for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} { assert.Empty(t, b.Config.Resources.Pipelines[k].Target) @@ -214,43 +214,43 @@ func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "foobar", }, }, "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog2", Target: "foobar", }, }, "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "barfoo", }, }, "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalogX", Target: "foobar", }, }, "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "catalog1", Target: "schemaX", }, }, "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "foobar", }, }, "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Catalog: "", Target: "", Name: "whatever", diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index 7cf3c9f3e..c5b1ad39d 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -47,7 +47,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index f07a7deb3..8c751079b 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -31,8 +31,8 @@ func TestInitializeURLs(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - ID: "3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}, + ID: "3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1"}, }, }, Experiments: map[string]*resources.MlflowExperiment{ diff --git a/bundle/config/mutator/merge_pipeline_clusters_test.go b/bundle/config/mutator/merge_pipeline_clusters_test.go index f117d9399..97ec44eea 100644 --- a/bundle/config/mutator/merge_pipeline_clusters_test.go +++ b/bundle/config/mutator/merge_pipeline_clusters_test.go @@ -19,7 +19,7 @@ func TestMergePipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { NodeTypeId: "i3.xlarge", @@ -68,7 +68,7 @@ func TestMergePipelineClustersCaseInsensitive(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { Label: "default", diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 723b01ee3..6a0fd8e03 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -88,7 +88,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}}, + "pipeline1": {CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}}, @@ -181,7 +181,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous) - assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.True(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) // Experiment 1 assert.Equal(t, "/Users/lennart.kats@databricks.com/[dev lennart] experiment1", b.Config.Resources.Experiments["experiment1"].Name) @@ -316,7 +316,7 @@ func TestProcessTargetModeDefault(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -362,7 +362,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) @@ -568,5 +568,5 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) { diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) - assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development) } diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go index 44f6c8dbb..30969dc49 100644 --- a/bundle/config/mutator/resolve_variable_references_test.go +++ b/bundle/config/mutator/resolve_variable_references_test.go @@ -20,7 +20,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { true, func(t *testing.T, b *bundle.Bundle) { // Variables that use workspace file path should have SyncRootValue during resolution phase - require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) // The file path itself should remain the same require.Equal(t, "file/path", b.Config.Workspace.FilePath) @@ -29,7 +29,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { { false, func(t *testing.T, b *bundle.Bundle) { - require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"]) + require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"]) require.Equal(t, "file/path", b.Config.Workspace.FilePath) }, }, @@ -48,7 +48,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Configuration: map[string]string{ "source": "${workspace.file_path}", }, diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index aa6488ab0..6cfe5718a 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -179,7 +179,7 @@ func TestTranslatePaths(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -333,7 +333,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -488,7 +488,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -532,7 +532,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -572,7 +572,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -677,7 +677,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ @@ -712,7 +712,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { File: &pipelines.FileLibrary{ @@ -916,7 +916,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 5127d07ba..57d9c4f19 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,7 +16,7 @@ type Pipeline struct { ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` URL string `json:"url,omitempty" bundle:"internal"` - *pipelines.PipelineSpec + *pipelines.CreatePipeline } func (s *Pipeline) UnmarshalJSON(b []byte) error { @@ -59,5 +59,5 @@ func (s *Pipeline) GetURL() string { } func (s *Pipeline) IsNil() bool { - return s.PipelineSpec == nil + return s.CreatePipeline == nil } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index c3ead8ef6..be93420c6 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -238,7 +238,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, @@ -493,7 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Clusters: []pipelines.PipelineCluster{ { SparkConf: tc.sparkConf, diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go index 990f48907..407aaea6e 100644 --- a/bundle/deploy/metadata/annotate_pipelines.go +++ b/bundle/deploy/metadata/annotate_pipelines.go @@ -20,11 +20,11 @@ func (m *annotatePipelines) Name() string { func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for _, pipeline := range b.Config.Resources.Pipelines { - if pipeline.PipelineSpec == nil { + if pipeline.CreatePipeline == nil { continue } - pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{ + pipeline.CreatePipeline.Deployment = &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: metadataFilePath(b), } diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go index 448a022d0..606292724 100644 --- a/bundle/deploy/metadata/annotate_pipelines_test.go +++ b/bundle/deploy/metadata/annotate_pipelines_test.go @@ -21,12 +21,12 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "my-pipeline-1": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline One", }, }, "my-pipeline-2": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "My Pipeline Two", }, }, @@ -43,14 +43,14 @@ func TestAnnotatePipelinesMutator(t *testing.T) { Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-1"].CreatePipeline.Deployment) assert.Equal(t, &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, MetadataFilePath: "/a/b/c/metadata.json", }, - b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment) + b.Config.Resources.Pipelines["my-pipeline-2"].CreatePipeline.Deployment) } func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) { diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index afc1fb22a..53d861b32 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -203,7 +203,7 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) { func TestBundleToTerraformPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", Libraries: []pipelines.PipelineLibrary{ { @@ -759,7 +759,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, @@ -898,12 +898,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "test_pipeline": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline", }, }, "test_pipeline_new": { - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "test_pipeline_new", }, }, diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index ea0c94d66..53a986864 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,6 +21,11 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } + vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"}) + if err != nil { + return dyn.InvalidValue, err + } + // Normalize the output value to the target schema. vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout) for _, diag := range diags { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go index 0239bad18..d8de55bf0 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -15,8 +15,17 @@ import ( func TestConvertPipeline(t *testing.T) { src := resources.Pipeline{ - PipelineSpec: &pipelines.PipelineSpec{ + CreatePipeline: &pipelines.CreatePipeline{ Name: "my pipeline", + // This fields is not part of TF schema yet, but once we upgrade to TF version that supports it, this test will fail because run_as + // will be exposed which is expected and test will need to be updated. + RunAs: &pipelines.RunAs{ + UserName: "foo@bar.com", + }, + // We expect AllowDuplicateNames and DryRun to be ignored and not passed to the TF output. + // This is not supported by TF now, so we don't want to expose it. + AllowDuplicateNames: true, + DryRun: true, Libraries: []pipelines.PipelineLibrary{ { Notebook: &pipelines.NotebookLibrary{ diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index ad737feea..84bf4779f 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.MarkdownDescription = a.MarkdownDescription } if a.MarkdownExamples != "" { - s.Examples = []any{a.MarkdownExamples} + s.Examples = []string{a.MarkdownExamples} } } diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go index 68ed86450..61d2c21cc 100644 --- a/bundle/docsgen/nodes.go +++ b/bundle/docsgen/nodes.go @@ -220,9 +220,9 @@ func isCycleField(field string) bool { } func getExample(v *jsonschema.Schema) string { - examples := v.Examples + examples := getExamples(v.Examples) if len(examples) == 0 { return "" } - return examples[0].(string) + return examples[0] } diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go index ca45e6ab2..7a4451129 100644 --- a/bundle/docsgen/refs.go +++ b/bundle/docsgen/refs.go @@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j node := s description := s.Description markdownDescription := s.MarkdownDescription - examples := s.Examples + examples := getExamples(s.Examples) for node.Reference != nil { ref := getRefType(node) @@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j markdownDescription = newNode.MarkdownDescription } if len(examples) == 0 { - examples = newNode.Examples + examples = getExamples(newNode.Examples) } node = newNode @@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j return &newNode } +func getExamples(examples any) []string { + typedExamples, ok := examples.([]string) + if !ok { + return []string{} + } + return typedExamples +} + func getRefType(node *jsonschema.Schema) string { if node.Reference == nil { return "" diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index c10f43b04..2d1a6a3d8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -414,6 +414,16 @@ github.com/databricks/cli/bundle/config/resources.Permission: "user_name": "description": |- The name of the user that has the permission set in level. +github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + PLACEHOLDER + "dry_run": + "description": |- + PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index d5a9bf69e..d9a0be50e 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -371,6 +371,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint: "description": |- Tags to be attached to the serving endpoint and automatically propagated to billing logs. github.com/databricks/cli/bundle/config/resources.Pipeline: + "allow_duplicate_names": + "description": |- + If false, deployment will fail if name conflicts with that of another pipeline. "budget_policy_id": "description": |- Budget policy of this pipeline. @@ -395,6 +398,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "development": "description": |- Whether the pipeline is in Development mode. Defaults to false. + "dry_run": {} "edition": "description": |- Pipeline product edition. @@ -425,6 +429,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "restart_window": "description": |- Restart window of this pipeline. + "run_as": {} "schema": "description": |- The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode. @@ -2624,6 +2629,18 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow: "description": |- Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details. If not specified, UTC will be used. +github.com/databricks/databricks-sdk-go/service/pipelines.RunAs: + "_": + "description": |- + Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline. + + Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown. + "service_principal_name": + "description": |- + Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. + "user_name": + "description": |- + The email of an active workspace user. Users can only set this field to their own email. github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec: "destination_catalog": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 912a4fda0..be83af2d1 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). - "markdown_examples": |- The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: @@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). - "embed_credentials": "description": |- PLACEHOLDER @@ -241,9 +239,15 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: - notebook: path: ./pipeline.py ``` + "dry_run": + "description": |- + PLACEHOLDER "permissions": "description": |- PLACEHOLDER + "run_as": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.QualityMonitor: "_": "markdown_description": |- @@ -356,7 +360,6 @@ github.com/databricks/cli/bundle/config/resources.Volume: - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). - "markdown_examples": |- The following example creates a volume with the key `my_volume`: @@ -376,6 +379,42 @@ github.com/databricks/cli/bundle/config/resources.Volume: "volume_type": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "deployment_artifacts": + "description": |- + PLACEHOLDER + "deployment_id": + "description": |- + PLACEHOLDER + "mode": + "description": |- + PLACEHOLDER + "source_code_path": + "description": |- + PLACEHOLDER + "status": + "description": |- + PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: + "source_code_path": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/apps.AppResource: "job": "description": |- @@ -389,6 +428,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource: "sql_warehouse": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: + "key": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER + "scope": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: + "name": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: + "message": + "description": |- + PLACEHOLDER + "state": {} github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: "availability": "description": |- @@ -473,85 +555,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: - "create_time": - "description": |- - PLACEHOLDER - "creator": - "description": |- - PLACEHOLDER - "deployment_artifacts": - "description": |- - PLACEHOLDER - "deployment_id": - "description": |- - PLACEHOLDER - "mode": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "status": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: - "source_code_path": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: - "key": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER - "scope": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: - "name": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: - "message": - "description": |- - PLACEHOLDER - "state": github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 38e099ece..2e0120e62 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -109,6 +109,20 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { return s } +func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Pipeline{}): + // Even though DABs supports this field, TF provider does not. Thus, we + // should not expose it to the user. + delete(s.Properties, "dry_run") + delete(s.Properties, "allow_duplicate_names") + default: + // Do nothing + } + + return s +} + // While volume_type is required in the volume create API, DABs automatically sets // it's value to "MANAGED" if it's not provided. Thus, we make it optional // in the bundle schema. @@ -168,6 +182,7 @@ func generateSchema(workdir, outputFile string) { // Generate the JSON schema from the bundle Go struct. s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ removeJobsFields, + removePipelineFields, makeVolumeTypeOptional, a.addAnnotations, addInterpolationPatterns, diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 50e69e7c8..ca8c27d4c 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "os" @@ -9,8 +10,9 @@ import ( "strings" "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" - "gopkg.in/yaml.v3" ) type Components struct { @@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if err != nil { return err } - err = yaml.Unmarshal(b, &overrides) + overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b)) + if err != nil { + return err + } + err = convert.ToTyped(&overrides, overridesDyn) if err != nil { return err } diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index bb1905045..7a808f627 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic for _, match := range matches { output = append(output, dyn.NewValue(map[string]dyn.Value{ - libType: dyn.V(match), + libType: dyn.NewValue(match, lib.Locations()), }, lib.Locations())) } } diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go new file mode 100644 index 000000000..88b96ab54 --- /dev/null +++ b/bundle/libraries/same_name_libraries.go @@ -0,0 +1,97 @@ +package libraries + +import ( + "context" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type checkForSameNameLibraries struct{} + +var patterns = []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + envDepsPattern.Append(dyn.AnyIndex()), +} + +type libData struct { + fullPath string + locations []dyn.Location + paths []dyn.Path +} + +func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + libs := make(map[string]*libData) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + libPath := lv.MustString() + // If not local library, skip the check + if !IsLibraryLocal(libPath) { + return lv, nil + } + + libFullPath := lv.MustString() + lib := filepath.Base(libFullPath) + // If the same basename was seen already but full path is different + // then it's a duplicate. Add the location to the location list. + lp, ok := libs[lib] + if !ok { + libs[lib] = &libData{ + fullPath: libFullPath, + locations: []dyn.Location{lv.Location()}, + paths: []dyn.Path{p}, + } + } else if lp.fullPath != libFullPath { + lp.locations = append(lp.locations, lv.Location()) + lp.paths = append(lp.paths, p) + } + + return lv, nil + }) + if err != nil { + return dyn.InvalidValue, err + } + } + + if err != nil { + return dyn.InvalidValue, err + } + + return v, nil + }) + + // Iterate over all the libraries and check if there are any duplicates. + // Duplicates will have more than one location. + // If there are duplicates, add a diagnostic. + for lib, lv := range libs { + if len(lv.locations) > 1 { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Duplicate local library name " + lib, + Detail: "Local library names must be unique", + Locations: lv.locations, + Paths: lv.paths, + }) + } + } + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (c checkForSameNameLibraries) Name() string { + return "CheckForSameNameLibraries" +} + +func CheckForSameNameLibraries() bundle.Mutator { + return checkForSameNameLibraries{} +} diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go new file mode 100644 index 000000000..42c38773b --- /dev/null +++ b/bundle/libraries/same_name_libraries_test.go @@ -0,0 +1,121 @@ +package libraries + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestSameNameLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test.whl", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + }) + bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{ + {File: "databricks.yml", Line: 20, Column: 1}, + }) + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Len(t, diags, 1) + require.Equal(t, diag.Error, diags[0].Severity) + require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary) + require.Equal(t, []dyn.Location{ + {File: "databricks.yml", Line: 10, Column: 1}, + {File: "databricks.yml", Line: 20, Column: 1}, + }, diags[0].Locations) + + paths := make([]string, 0) + for _, p := range diags[0].Paths { + paths = append(paths, p.String()) + } + require.Equal(t, []string{ + "resources.jobs.test.tasks[0].libraries[0].whl", + "resources.jobs.test.tasks[1].libraries[0].whl", + }, paths) +} + +func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "other/path/test-0.1.0.whl", + }, + + { + Whl: "cowsay", + }, + }, + }, + { + Libraries: []compute.Library{ + { + Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries()) + require.Empty(t, diags) +} diff --git a/bundle/mutator.go b/bundle/mutator.go index 6c9968aac..16ef79ee7 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -42,7 +42,7 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics { // such that they are not logged multiple times. // If this is done, we can omit this block. if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go index ee4e36e0f..700a90d8d 100644 --- a/bundle/mutator_read_only.go +++ b/bundle/mutator_read_only.go @@ -22,7 +22,7 @@ func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) di log.Debugf(ctx, "ApplyReadOnly") diags := m.Apply(ctx, rb) if err := diags.Error(); err != nil { - log.Errorf(ctx, "Error: %s", err) + log.Debugf(ctx, "Error: %s", err) } return diags diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index c48704a63..3e5f9c61b 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -38,8 +38,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, @@ -98,8 +98,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) { "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, - "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}}, + "pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ "model_1": {Model: &ml.Model{}}, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c6ec04962..2e9211a7e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -155,6 +155,11 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { mutator.ValidateGitDetails(), artifacts.CleanUp(), libraries.ExpandGlobReferences(), + // libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we + // know what are the actual library paths. + // libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this + // mutator is part of the deploy step rather than validate. + libraries.CheckForSameNameLibraries(), libraries.Upload(), trampoline.TransformWheelTask(), files.Upload(outputHandler), diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 506756f70..d092e77c8 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -530,12 +530,12 @@ func TestRenderSummary(t *testing.T) { "pipeline2": { ID: "4", // no URL - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"}, + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline2-name"}, }, "pipeline1": { - ID: "3", - URL: "https://url3", - PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"}, + ID: "3", + URL: "https://url3", + CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1-name"}, }, }, Schemas: map[string]*resources.Schema{ diff --git a/bundle/resources/completion_test.go b/bundle/resources/completion_test.go index 80412b6f1..56559f18c 100644 --- a/bundle/resources/completion_test.go +++ b/bundle/resources/completion_test.go @@ -25,7 +25,7 @@ func TestCompletions_SkipDuplicates(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -50,7 +50,7 @@ func TestCompletions_Filter(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/resources/lookup_test.go b/bundle/resources/lookup_test.go index 0ea5af7a2..d95da977a 100644 --- a/bundle/resources/lookup_test.go +++ b/bundle/resources/lookup_test.go @@ -56,7 +56,7 @@ func TestLookup_MultipleFound(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "foo": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, @@ -107,7 +107,7 @@ func TestLookup_NominalWithFilters(t *testing.T) { }, Pipelines: map[string]*resources.Pipeline{ "bar": { - PipelineSpec: &pipelines.PipelineSpec{}, + CreatePipeline: &pipelines.CreatePipeline{}, }, }, }, diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index bdcf0f142..1cd6e8743 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -79,10 +79,10 @@ type pipelineRunner struct { } func (r *pipelineRunner) Name() string { - if r.pipeline == nil || r.pipeline.PipelineSpec == nil { + if r.pipeline == nil || r.pipeline.CreatePipeline == nil { return "" } - return r.pipeline.PipelineSpec.Name + return r.pipeline.CreatePipeline.Name } func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 7c72c440e..9d4304cd8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -703,6 +703,9 @@ "description": "Restart window of this pipeline.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow" }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RunAs" + }, "schema": { "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", "$ref": "#/$defs/string" @@ -5385,6 +5388,29 @@ } ] }, + "pipelines.RunAs": { + "oneOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.SchemaSpec": { "oneOf": [ { diff --git a/cmd/auth/token.go b/cmd/auth/token.go index fbf8b68f6..f3468df40 100644 --- a/cmd/auth/token.go +++ b/cmd/auth/token.go @@ -46,6 +46,10 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { cmd := &cobra.Command{ Use: "token [HOST]", Short: "Get authentication token", + Long: `Get authentication token from the local cache in ~/.databricks/token-cache.json. +Refresh the access token if it is expired. Note: This command only works with +U2M authentication (using the 'databricks auth login' command). M2M authentication +using a client ID and secret is not supported.`, } var tokenTimeout time.Duration diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index d97891cd5..438b235c9 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -50,10 +50,22 @@ func NewGenerateJobCommand() *cobra.Command { } downloader := newDownloader(w, sourceDir, configDir) - for _, task := range job.Settings.Tasks { - err := downloader.MarkTaskForDownload(ctx, &task) - if err != nil { - return err + + // Don't download files if the job is using Git source + // When Git source is used, the job will be using the files from the Git repository + // but specific tasks might override this behaviour by using `source: WORKSPACE` setting. + // In this case, we don't want to download the files as well for these specific tasks + // because it leads to confusion with relative paths between workspace and GIT files. + // Instead we keep these tasks as is and let the user handle the files manually. + // The configuration will be deployable as tasks paths for source: WORKSPACE tasks will be absolute workspace paths. + if job.Settings.GitSource != nil { + cmdio.LogString(ctx, "Job is using Git source, skipping downloading files") + } else { + for _, task := range job.Settings.Tasks { + err := downloader.MarkTaskForDownload(ctx, &task) + if err != nil { + return err + } } } diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 1d2c345d6..9bf9e9947 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -92,7 +92,7 @@ func NewGeneratePipelineCommand() *cobra.Command { } saver := yamlsaver.NewSaverWithStyle( - // Including all PipelineSpec and nested fields which are map[string]string type + // Including all CreatePipeline and nested fields which are map[string]string type map[string]yaml.Style{ "spark_conf": yaml.DoubleQuotedStyle, "custom_tags": yaml.DoubleQuotedStyle, diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 05f7d68aa..2e42ce43d 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -32,6 +32,7 @@ type hook struct { RequireDatabricksConnect bool `yaml:"require_databricks_connect,omitempty"` MinRuntimeVersion string `yaml:"min_runtime_version,omitempty"` WarehouseTypes whTypes `yaml:"warehouse_types,omitempty"` + Extras string `yaml:"extras,omitempty"` } func (h *hook) RequireRunningCluster() bool { @@ -258,6 +259,10 @@ func (i *installer) setupPythonVirtualEnvironment(ctx context.Context, w *databr } } feedback <- "Installing Python library dependencies" + if i.Installer.Extras != "" { + // install main and optional dependencies + return i.installPythonDependencies(ctx, fmt.Sprintf(".[%s]", i.Installer.Extras)) + } return i.installPythonDependencies(ctx, ".") } diff --git a/cmd/labs/project/schema.json b/cmd/labs/project/schema.json index a779b15e4..7aa65813c 100644 --- a/cmd/labs/project/schema.json +++ b/cmd/labs/project/schema.json @@ -42,6 +42,11 @@ }, "warehouse_types": { "enum": [ "PRO", "CLASSIC", "TYPE_UNSPECIFIED" ] + }, + "extras": { + "type": "string", + "pattern": "^([^,]+)(,([^,]+))*$", + "default": "" } } }, diff --git a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml index 0ac4bf826..b8a0e695e 100644 --- a/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml +++ b/cmd/labs/project/testdata/installed-in-home/.databricks/labs/blueprint/lib/labs.yml @@ -8,6 +8,7 @@ install: warehouse_types: - PRO script: install.py + extras: "" entrypoint: main.py min_python: 3.9 commands: diff --git a/cmd/root/root.go b/cmd/root/root.go index 3b37d0176..d7adf47f4 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -114,10 +114,15 @@ func Execute(ctx context.Context, cmd *cobra.Command) error { if err == nil { logger.Info("completed execution", slog.String("exit_code", "0")) - } else { - logger.Error("failed execution", + } else if errors.Is(err, ErrAlreadyPrinted) { + logger.Debug("failed execution", slog.String("exit_code", "1"), - slog.String("error", err.Error())) + ) + } else { + logger.Info("failed execution", + slog.String("exit_code", "1"), + slog.String("error", err.Error()), + ) } } diff --git a/go.mod b/go.mod index b3f11e918..662fcd40b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 toolchain go1.23.4 require ( + dario.cat/mergo v1.0.1 // BSD 3-Clause github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 diff --git a/go.sum b/go.sum index 4e295a82d..bffc3b53d 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKF cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= diff --git a/integration/bundle/apps_test.go b/integration/bundle/apps_test.go index 01ab52e90..12bd2fcbf 100644 --- a/integration/bundle/apps_test.go +++ b/integration/bundle/apps_test.go @@ -98,6 +98,24 @@ func TestDeployBundleWithApp(t *testing.T) { - run env: - name: JOB_ID + value: "%d"`, job.JobId)) + + // Redeploy bundle with changed config env for app and confirm it's updated in app.yaml + deployBundleWithArgs(t, ctx, root, `--var="env_var_name=ANOTHER_JOB_ID"`, "--force-lock", "--auto-approve") + reader, err = wt.W.Workspace.Download(ctx, pathToAppYml) + require.NoError(t, err) + + data, err = io.ReadAll(reader) + require.NoError(t, err) + + content = string(data) + require.Contains(t, content, fmt.Sprintf(`command: + - flask + - --app + - app + - run +env: + - name: ANOTHER_JOB_ID value: "%d"`, job.JobId)) if testing.Short() { diff --git a/integration/bundle/bundles/apps/template/databricks.yml.tmpl b/integration/bundle/bundles/apps/template/databricks.yml.tmpl index 4d862a06f..e0937be71 100644 --- a/integration/bundle/bundles/apps/template/databricks.yml.tmpl +++ b/integration/bundle/bundles/apps/template/databricks.yml.tmpl @@ -4,6 +4,10 @@ bundle: workspace: root_path: "~/.bundle/{{.unique_id}}" +variables: + env_var_name: + default: "JOB_ID" + resources: apps: test_app: @@ -17,7 +21,7 @@ resources: - app - run env: - - name: JOB_ID + - name: ${var.env_var_name} value: ${resources.jobs.foo.id} resources: diff --git a/libs/dyn/drop_keys.go b/libs/dyn/drop_keys.go new file mode 100644 index 000000000..494f9b9cd --- /dev/null +++ b/libs/dyn/drop_keys.go @@ -0,0 +1,27 @@ +package dyn + +func DropKeys(v Value, drop []string) (Value, error) { + var err error + nv, err := Walk(v, func(p Path, v Value) (Value, error) { + if len(p) == 0 { + return v, nil + } + + // Check if this key should be dropped. + for _, key := range drop { + if p[0].Key() != key { + continue + } + + return InvalidValue, ErrDrop + } + + // Pass through all other values. + return v, ErrSkip + }) + if err != nil { + return InvalidValue, err + } + + return nv, nil +} diff --git a/libs/dyn/drop_keys_test.go b/libs/dyn/drop_keys_test.go new file mode 100644 index 000000000..83a9744ca --- /dev/null +++ b/libs/dyn/drop_keys_test.go @@ -0,0 +1,24 @@ +package dyn + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDropKeysTest(t *testing.T) { + v := V(map[string]Value{ + "key1": V("value1"), + "key2": V("value2"), + "key3": V("value3"), + }) + + vout, err := DropKeys(v, []string{"key1", "key3"}) + require.NoError(t, err) + + mv := vout.MustMap() + require.Equal(t, 1, mv.Len()) + v, ok := mv.GetByString("key2") + require.True(t, ok) + require.Equal(t, "value2", v.MustString()) +} diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index a162bf31f..c1b60b1b5 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -22,9 +22,50 @@ func ConvertToMapValue(strct any, order *Order, skipFields []string, dst map[str return dyn.InvalidValue, fmt.Errorf("expected map, got %s", mv.Kind()) } + mv, err = sortMapAlphabetically(mv) + if err != nil { + return dyn.InvalidValue, err + } + return skipAndOrder(mv, order, skipFields, dst) } +// Sort the map alphabetically by keys. This is used to produce stable output for generated YAML files. +func sortMapAlphabetically(mv dyn.Value) (dyn.Value, error) { + sortedMap := dyn.NewMapping() + mapV := mv.MustMap() + keys := mapV.Keys() + slices.SortStableFunc(keys, func(i, j dyn.Value) int { + iKey := i.MustString() + jKey := j.MustString() + if iKey < jKey { + return -1 + } + + if iKey > jKey { + return 1 + } + return 0 + }) + + for _, key := range keys { + value, _ := mapV.Get(key) + var err error + if value.Kind() == dyn.KindMap { + value, err = sortMapAlphabetically(value) + if err != nil { + return dyn.InvalidValue, err + } + } + err = sortedMap.Set(key, value) + if err != nil { + return dyn.InvalidValue, err + } + } + + return dyn.V(sortedMap), nil +} + func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[string]dyn.Value) (dyn.Value, error) { for _, pair := range mv.MustMap().Pairs() { k := pair.Key.MustString() @@ -44,7 +85,11 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin continue } - dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + if order == nil { + dst[k] = v + } else { + dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) + } } return dyn.V(dst), nil diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 1afab601a..f7ea3c96c 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -7,6 +7,54 @@ import ( assert "github.com/databricks/cli/libs/dyn/dynassert" ) +func TestConvertToMap(t *testing.T) { + type test struct { + Name string `json:"name"` + Map map[string]string `json:"map"` + List []string `json:"list"` + LongNameField string `json:"long_name_field"` + ForceSendFields []string `json:"-"` + Format string `json:"format"` + } + + v := &test{ + Name: "test", + Map: map[string]string{ + "key2": "value2", + "key1": "value1", + }, + List: []string{"a", "b", "c"}, + ForceSendFields: []string{ + "Name", + }, + LongNameField: "long name goes here", + } + result, err := ConvertToMapValue(v, nil, []string{"format"}, map[string]dyn.Value{}) + assert.NoError(t, err) + assert.Equal(t, dyn.V(map[string]dyn.Value{ + "list": dyn.NewValue( + []dyn.Value{ + dyn.V("a"), + dyn.V("b"), + dyn.V("c"), + }, + []dyn.Location{}, + ), + "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{}), + "map": dyn.NewValue( + map[string]dyn.Value{ + "key1": dyn.V("value1"), + "key2": dyn.V("value2"), + }, + []dyn.Location{}, + ), + "name": dyn.NewValue( + "test", + []dyn.Location{}, + ), + }), result) +} + func TestConvertToMapValueWithOrder(t *testing.T) { type test struct { Name string `json:"name"` diff --git a/libs/flags/log_level_flag.go b/libs/flags/log_level_flag.go index 836d84b70..82e2abc4c 100644 --- a/libs/flags/log_level_flag.go +++ b/libs/flags/log_level_flag.go @@ -25,7 +25,7 @@ type LogLevelFlag struct { func NewLogLevelFlag() LogLevelFlag { return LogLevelFlag{ - l: log.LevelDisabled, + l: log.LevelWarn, } } diff --git a/libs/flags/log_level_flag_test.go b/libs/flags/log_level_flag_test.go index 11a50bc45..c81f90d18 100644 --- a/libs/flags/log_level_flag_test.go +++ b/libs/flags/log_level_flag_test.go @@ -10,8 +10,8 @@ import ( func TestLogLevelFlagDefault(t *testing.T) { f := NewLogLevelFlag() - assert.Equal(t, log.LevelDisabled, f.Level()) - assert.Equal(t, "disabled", f.String()) + assert.Equal(t, log.LevelWarn, f.Level()) + assert.Equal(t, "warn", f.String()) } func TestLogLevelFlagSetValid(t *testing.T) { diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 5028bb0d7..85f6a0328 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -79,7 +79,7 @@ type Schema struct { // Examples of the value for properties in the schema. // https://json-schema.org/understanding-json-schema/reference/annotations - Examples []any `json:"examples,omitempty"` + Examples any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string. diff --git a/libs/log/handler/friendly.go b/libs/log/handler/friendly.go index 33b88a9e2..5c60eb13d 100644 --- a/libs/log/handler/friendly.go +++ b/libs/log/handler/friendly.go @@ -53,11 +53,11 @@ func NewFriendlyHandler(out io.Writer, opts *Options) slog.Handler { // Cache (colorized) level strings. // The colors to use for each level are configured in `colors.go`. - h.levelTrace = h.sprintf(ttyColorLevelTrace, "%5s", "TRACE") - h.levelDebug = h.sprintf(ttyColorLevelDebug, "%5s", "DEBUG") - h.levelInfo = h.sprintf(ttyColorLevelInfo, "%5s", "INFO") - h.levelWarn = h.sprintf(ttyColorLevelWarn, "%5s", "WARN") - h.levelError = h.sprintf(ttyColorLevelError, "%5s", "ERROR") + h.levelTrace = h.sprintf(ttyColorLevelTrace, "%s", "Trace:") + h.levelDebug = h.sprintf(ttyColorLevelDebug, "%s", "Debug:") + h.levelInfo = h.sprintf(ttyColorLevelInfo, "%s", "Info:") + h.levelWarn = h.sprintf(ttyColorLevelWarn, "%s", "Warn:") + h.levelError = h.sprintf(ttyColorLevelError, "%s", "Error:") return h } @@ -185,33 +185,41 @@ func (s *handleState) appendAttr(a slog.Attr) { // Handle implements slog.Handler. func (h *friendlyHandler) Handle(ctx context.Context, r slog.Record) error { state := h.handleState() - state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + + if h.opts.Level.Level() <= slog.LevelDebug { + state.append(h.sprintf(ttyColorTime, "%02d:%02d:%02d ", r.Time.Hour(), r.Time.Minute(), r.Time.Second())) + } + state.appendf("%s ", h.coloredLevel(r)) state.append(h.sprint(ttyColorMessage, r.Message)) - // Handle state from WithGroup and WithAttrs. - goas := h.goas - if r.NumAttrs() == 0 { - // If the record has no Attrs, remove groups at the end of the list; they are empty. - for len(goas) > 0 && goas[len(goas)-1].group != "" { - goas = goas[:len(goas)-1] - } - } - for _, goa := range goas { - if goa.group != "" { - state.openGroup(goa.group) - } else { - for _, a := range goa.attrs { - state.appendAttr(a) + if h.opts.Level.Level() <= slog.LevelDebug { + + // Handle state from WithGroup and WithAttrs. + goas := h.goas + if r.NumAttrs() == 0 { + // If the record has no Attrs, remove groups at the end of the list; they are empty. + for len(goas) > 0 && goas[len(goas)-1].group != "" { + goas = goas[:len(goas)-1] + } + } + for _, goa := range goas { + if goa.group != "" { + state.openGroup(goa.group) + } else { + for _, a := range goa.attrs { + state.appendAttr(a) + } } } - } - // Add attributes from the record. - r.Attrs(func(a slog.Attr) bool { - state.appendAttr(a) - return true - }) + // Add attributes from the record. + r.Attrs(func(a slog.Attr) bool { + state.appendAttr(a) + return true + }) + + } // Add newline. state.append("\n") diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go new file mode 100644 index 000000000..c3e4f9a71 --- /dev/null +++ b/libs/testserver/fake_workspace.go @@ -0,0 +1,169 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// FakeWorkspace holds a state of a workspace for acceptance tests. +type FakeWorkspace struct { + directories map[string]bool + files map[string][]byte + // normally, ids are not sequential, but we make them sequential for deterministic diff + nextJobId int64 + jobs map[int64]jobs.Job +} + +func NewFakeWorkspace() *FakeWorkspace { + return &FakeWorkspace{ + directories: map[string]bool{ + "/Workspace": true, + }, + files: map[string][]byte{}, + jobs: map[int64]jobs.Job{}, + nextJobId: 1, + } +} + +func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) { + if s.directories[path] { + return workspace.ObjectInfo{ + ObjectType: "DIRECTORY", + Path: path, + }, http.StatusOK + } else if _, ok := s.files[path]; ok { + return workspace.ObjectInfo{ + ObjectType: "FILE", + Path: path, + Language: "SCALA", + }, http.StatusOK + } else { + return workspace.ObjectInfo{}, http.StatusNotFound + } +} + +func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) { + s.directories[request.Path] = true + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) { + file := s.files[path] + + if file == nil { + return nil, http.StatusNotFound + } + + return file, http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) { + if !recursive { + s.files[path] = nil + } else { + for key := range s.files { + if strings.HasPrefix(key, path) { + s.files[key] = nil + } + } + } + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + s.files[path] = body + + return "{}", http.StatusOK +} + +func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) { + jobId := s.nextJobId + s.nextJobId++ + + jobSettings := jobs.JobSettings{} + err := jsonConvert(request, &jobSettings) + if err != nil { + return internalError(err) + } + + s.jobs[jobId] = jobs.Job{ + JobId: jobId, + Settings: &jobSettings, + } + + return jobs.CreateResponse{JobId: jobId}, http.StatusOK +} + +func (s *FakeWorkspace) JobsGet(jobId string) (any, int) { + id := jobId + + jobIdInt, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return internalError(fmt.Errorf("failed to parse job id: %s", err)) + } + + job, ok := s.jobs[jobIdInt] + if !ok { + return jobs.Job{}, http.StatusNotFound + } + + return job, http.StatusOK +} + +func (s *FakeWorkspace) JobsList() (any, int) { + list := make([]jobs.BaseJob, 0, len(s.jobs)) + for _, job := range s.jobs { + baseJob := jobs.BaseJob{} + err := jsonConvert(job, &baseJob) + if err != nil { + return internalError(fmt.Errorf("failed to convert job to base job: %w", err)) + } + + list = append(list, baseJob) + } + + // sort to have less non-determinism in tests + sort.Slice(list, func(i, j int) bool { + return list[i].JobId < list[j].JobId + }) + + return jobs.ListJobsResponse{ + Jobs: list, + }, http.StatusOK +} + +// jsonConvert saves input to a value pointed by output +func jsonConvert(input, output any) error { + writer := new(bytes.Buffer) + encoder := json.NewEncoder(writer) + err := encoder.Encode(input) + if err != nil { + return fmt.Errorf("failed to encode: %w", err) + } + + decoder := json.NewDecoder(writer) + err = decoder.Decode(output) + if err != nil { + return fmt.Errorf("failed to decode: %w", err) + } + + return nil +} + +func internalError(err error) (string, int) { + return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError +} diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 42fd5fde3..a62a5e584 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -6,10 +6,13 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" + "sync" "github.com/stretchr/testify/assert" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/apierr" ) type Server struct { @@ -18,6 +21,9 @@ type Server struct { t testutil.TestingT + fakeWorkspaces map[string]*FakeWorkspace + mu *sync.Mutex + RecordRequests bool IncludeRequestHeaders []string @@ -25,10 +31,10 @@ type Server struct { } type Request struct { - Headers map[string]string `json:"headers,omitempty"` - Method string `json:"method"` - Path string `json:"path"` - Body any `json:"body"` + Headers http.Header `json:"headers,omitempty"` + Method string `json:"method"` + Path string `json:"path"` + Body any `json:"body"` } func New(t testutil.TestingT) *Server { @@ -36,29 +42,81 @@ func New(t testutil.TestingT) *Server { server := httptest.NewServer(mux) t.Cleanup(server.Close) - return &Server{ - Server: server, - Mux: mux, - t: t, + s := &Server{ + Server: server, + Mux: mux, + t: t, + mu: &sync.Mutex{}, + fakeWorkspaces: map[string]*FakeWorkspace{}, } + + // The server resolves conflicting handlers by using the one with higher + // specificity. This handler is the least specific, so it will be used as a + // fallback when no other handlers match. + s.Handle("/", func(fakeWorkspace *FakeWorkspace, r *http.Request) (any, int) { + pattern := r.Method + " " + r.URL.Path + + t.Errorf(` + +---------------------------------------- +No stub found for pattern: %s + +To stub a response for this request, you can add +the following to test.toml: +[[Server]] +Pattern = %q +Response.Body = ''' + +''' +Response.StatusCode = +---------------------------------------- + + +`, pattern, pattern) + + return apierr.APIError{ + Message: "No stub found for pattern: " + pattern, + }, http.StatusNotFound + }) + + return s } -type HandlerFunc func(req *http.Request) (resp any, statusCode int) +type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int) func (s *Server) Handle(pattern string, handler HandlerFunc) { s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { - resp, statusCode := handler(r) + // For simplicity we process requests sequentially. It's fast enough because + // we don't do any IO except reading and writing request/response bodies. + s.mu.Lock() + defer s.mu.Unlock() + + // Each test uses unique DATABRICKS_TOKEN, we simulate each token having + // it's own fake fakeWorkspace to avoid interference between tests. + var fakeWorkspace *FakeWorkspace = nil + token := getToken(r) + if token != "" { + if _, ok := s.fakeWorkspaces[token]; !ok { + s.fakeWorkspaces[token] = NewFakeWorkspace() + } + + fakeWorkspace = s.fakeWorkspaces[token] + } + + resp, statusCode := handler(fakeWorkspace, r) if s.RecordRequests { body, err := io.ReadAll(r.Body) assert.NoError(s.t, err) - headers := make(map[string]string) + headers := make(http.Header) for k, v := range r.Header { - if len(v) == 0 || !slices.Contains(s.IncludeRequestHeaders, k) { + if !slices.Contains(s.IncludeRequestHeaders, k) { continue } - headers[k] = v[0] + for _, vv := range v { + headers.Add(k, vv) + } } var reqBody any @@ -87,9 +145,10 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { var respBytes []byte var err error - respString, ok := resp.(string) - if ok { + if respString, ok := resp.(string); ok { respBytes = []byte(respString) + } else if respBytes0, ok := resp.([]byte); ok { + respBytes = respBytes0 } else { respBytes, err = json.MarshalIndent(resp, "", " ") if err != nil { @@ -104,3 +163,14 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) { } }) } + +func getToken(r *http.Request) string { + header := r.Header.Get("Authorization") + prefix := "Bearer " + + if !strings.HasPrefix(header, prefix) { + return "" + } + + return header[len(prefix):] +}