diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 562b72fcc..02c4790ad 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -99f644e72261ef5ecf8d74db20f4b7a1e09723cc \ No newline at end of file +e5c870006a536121442cfd2441bdc8a5fb76ae1e \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5be087016..2f8597999 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,10 @@ ## Changes - + + +## Why + ## Tests - + diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c41afc18c..670a3d270 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -53,7 +53,7 @@ jobs: go-version-file: go.mod - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: '3.9' @@ -95,7 +95,7 @@ jobs: # Exit with status code 1 if there are differences (i.e. unformatted files) git diff --exit-code - name: golangci-lint - uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 + uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 with: version: v1.63.4 args: --timeout=15m @@ -145,7 +145,10 @@ jobs: go run main.go bundle schema > schema.json # Add markdownDescription keyword to ajv - echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js + echo "module.exports = function(a) { + a.addKeyword('markdownDescription'); + a.addKeyword('deprecationMessage'); + }" >> keywords.js for file in ./bundle/internal/schema/testdata/pass/*.yml; do ajv test -s schema.json -d $file --valid -c=./keywords.js diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index 8b4684eab..e37de8920 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -54,21 +54,21 @@ jobs: args: release --snapshot --skip docker - name: Upload macOS binaries - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: cli_darwin_snapshot path: | dist/*_darwin_*/ - name: Upload Linux binaries - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: cli_linux_snapshot path: | dist/*_linux_*/ - name: Upload Windows binaries - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: cli_windows_snapshot path: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe5b4170b..81810c606 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: # QEMU is required to build cross platform docker images using buildx. # It allows virtualization of the CPU architecture at the application level. - name: Set up QEMU dependency - uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - name: Run GoReleaser id: releaser diff --git a/.vscode/settings.json b/.vscode/settings.json index f8b04f126..f103538b7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,8 @@ "python.envFile": "${workspaceRoot}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "files.associations": { + "script": "shellscript" + } } diff --git a/Makefile b/Makefile index 0c3860e29..a66fa646d 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ lintcheck: # formatting/goimports will not be applied by 'make lint'. However, it will be applied by 'make fmt'. # If you need to ensure that formatting & imports are always fixed, do "make fmt lint" fmt: - ruff format -q + ruff format -qn golangci-lint run --enable-only="gofmt,gofumpt,goimports" --fix ./... test: diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 066a84299..afdc42abc 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -390,6 +390,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont if _, ok := Ignored[relPath]; ok { continue } + if config.CompiledIgnoreObject.MatchesPath(relPath) { + continue + } unexpected = append(unexpected, relPath) if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py new file mode 100755 index 000000000..d122404b2 --- /dev/null +++ b/acceptance/bin/find.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +Usage: find.py +Finds all files within current directory matching regex. The output is sorted and slashes are always forward. + +If --expect N is provided, the number of matches must be N or error is printed. +""" + +import sys +import os +import re +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument("regex") +parser.add_argument("--expect", type=int) +args = parser.parse_args() + +regex = re.compile(args.regex) +result = [] + +for root, dirs, files in os.walk("."): + for filename in files: + path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/") + if regex.search(path): + result.append(path) + +result.sort() +for item in result: + print(item) +sys.stdout.flush() + +if args.expect is not None: + if args.expect != len(result): + sys.exit(f"Expected {args.expect}, got {len(result)}") diff --git a/acceptance/bundle/apps/app_yaml/app/app.py b/acceptance/bundle/apps/app_yaml/app/app.py new file mode 100644 index 000000000..e69de29bb diff --git a/acceptance/bundle/apps/app_yaml/app/app.yml b/acceptance/bundle/apps/app_yaml/app/app.yml new file mode 100644 index 000000000..61471358d --- /dev/null +++ b/acceptance/bundle/apps/app_yaml/app/app.yml @@ -0,0 +1,3 @@ +command: + - python + - app.py diff --git a/acceptance/bundle/apps/app_yaml/databricks.yml b/acceptance/bundle/apps/app_yaml/databricks.yml new file mode 100644 index 000000000..0064e6c6b --- /dev/null +++ b/acceptance/bundle/apps/app_yaml/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: apps_yaml + +resources: + apps: + myapp: + name: myapp + source_code_path: ./app diff --git a/acceptance/bundle/apps/app_yaml/out.app.yml.txt b/acceptance/bundle/apps/app_yaml/out.app.yml.txt new file mode 100644 index 000000000..eccd4eb13 --- /dev/null +++ b/acceptance/bundle/apps/app_yaml/out.app.yml.txt @@ -0,0 +1,5 @@ +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files/app/app.yml", + "raw_body": "command:\n - python\n - app.py\n" +} diff --git a/acceptance/bundle/apps/app_yaml/output.txt b/acceptance/bundle/apps/app_yaml/output.txt new file mode 100644 index 000000000..2a946e5ee --- /dev/null +++ b/acceptance/bundle/apps/app_yaml/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate +Name: apps_yaml +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/apps_yaml/default + +Validation OK! + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/apps/app_yaml/script b/acceptance/bundle/apps/app_yaml/script new file mode 100644 index 000000000..8cb34c62d --- /dev/null +++ b/acceptance/bundle/apps/app_yaml/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate +trace $CLI bundle deploy +jq 'select(.path == "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_yaml/default/files/app/app.yml")' out.requests.txt | sed 's/\\r//g' > out.app.yml.txt +rm out.requests.txt diff --git a/acceptance/bundle/apps/config_section/app/app.py b/acceptance/bundle/apps/config_section/app/app.py new file mode 100644 index 000000000..f1a18139c --- /dev/null +++ b/acceptance/bundle/apps/config_section/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/apps/config_section/databricks.yml b/acceptance/bundle/apps/config_section/databricks.yml new file mode 100644 index 000000000..25ab2f261 --- /dev/null +++ b/acceptance/bundle/apps/config_section/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: apps_config_section + +resources: + apps: + myapp: + name: myapp + source_code_path: ./app + config: + command: + - python + - app.py diff --git a/acceptance/bundle/apps/config_section/out.app.yml.txt b/acceptance/bundle/apps/config_section/out.app.yml.txt new file mode 100644 index 000000000..a3e69351b --- /dev/null +++ b/acceptance/bundle/apps/config_section/out.app.yml.txt @@ -0,0 +1,5 @@ +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files/app/app.yml", + "raw_body": "command:\n - python\n - app.py\n" +} diff --git a/acceptance/bundle/apps/config_section/output.txt b/acceptance/bundle/apps/config_section/output.txt new file mode 100644 index 000000000..800945278 --- /dev/null +++ b/acceptance/bundle/apps/config_section/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle validate +Warning: App config section detected + +remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead + +Name: apps_config_section +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/apps_config_section/default + +Found 1 warning + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Warning: App config section detected + +remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead + diff --git a/acceptance/bundle/apps/config_section/script b/acceptance/bundle/apps/config_section/script new file mode 100644 index 000000000..5769918be --- /dev/null +++ b/acceptance/bundle/apps/config_section/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate +trace $CLI bundle deploy +jq 'select(.path == "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/apps_config_section/default/files/app/app.yml")' out.requests.txt > out.app.yml.txt +rm out.requests.txt diff --git a/acceptance/bundle/apps/test.toml b/acceptance/bundle/apps/test.toml new file mode 100644 index 000000000..6999f2089 --- /dev/null +++ b/acceptance/bundle/apps/test.toml @@ -0,0 +1,26 @@ +Cloud = false +RecordRequests = true + +Ignore = [ + '.databricks', +] + +[[Server]] +Pattern = "POST /api/2.0/apps" + +[[Server]] +Pattern = "GET /api/2.0/apps/myapp" +Response.Body = ''' +{ + "name": "myapp", + "description": "", + "compute_status": { + "state": "ACTIVE", + "message": "App compute is active." + }, + "app_status": { + "state": "RUNNING", + "message": "Application is running." + } +} +''' diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml index a298217f2..a17f2659f 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/test.toml @@ -1,3 +1,5 @@ +RecordRequests = false + [[Repls]] Old = '\\' New = '/' diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare new file mode 100644 index 000000000..673b20af9 --- /dev/null +++ b/acceptance/bundle/artifacts/script.prepare @@ -0,0 +1,9 @@ +export PYTHONDONTWRITEBYTECODE=1 + +uv venv -q --python 3.12 .venv +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + source .venv/Scripts/activate +else + source .venv/bin/activate +fi +uv pip install -q setuptools diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml new file mode 100644 index 000000000..42432706e --- /dev/null +++ b/acceptance/bundle/artifacts/test.toml @@ -0,0 +1,18 @@ +Cloud = false +RecordRequests = true +Ignore = [ + '.venv', + 'dist', + 'build', + '*egg-info', + '.databricks', +] + +[[Server]] +Pattern = "GET /api/2.1/clusters/get" +Response.Body = ''' +{ + "cluster_id": "0717-132531-5opeqon1", + "spark_version": "13.3.x-scala2.12" +} +''' diff --git a/acceptance/bundle/artifacts/unique_name_libraries/output.txt b/acceptance/bundle/artifacts/unique_name_libraries/output.txt index ecc7bf57b..634574bf5 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/output.txt +++ b/acceptance/bundle/artifacts/unique_name_libraries/output.txt @@ -2,8 +2,8 @@ >>> errcode [CLI] bundle deploy Building whl1... Building whl2... -Uploading [package name] -Uploading [package name] +Uploading [package name]... +Uploading [package name]... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/unique_name_libraries/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/unique_name_libraries/test.toml b/acceptance/bundle/artifacts/unique_name_libraries/test.toml index 280338bd6..7956c1909 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/test.toml @@ -1,6 +1,6 @@ -Cloud = false +RecordRequests = false # The order in which files are uploaded can be different, so we just replace the name [[Repls]] -Old="Uploading .*-0.0.1-py3-none-any.whl..." +Old="Uploading (my_package|my_other_package)-0.0.1-py3-none-any.whl" New="Uploading [package name]" diff --git a/bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml b/acceptance/bundle/artifacts/whl_dbfs/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml rename to acceptance/bundle/artifacts/whl_dbfs/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt new file mode 100644 index 000000000..f0615c558 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -0,0 +1,32 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Expecting to find no wheels +>>> errcode find.py --expect 0 whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "dbfs:/path/to/dist/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting no wheels to be uploaded +>>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script new file mode 100644 index 000000000..d3b12cb8e --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -0,0 +1,12 @@ +trace $CLI bundle deploy + +title "Expecting to find no wheels" +trace errcode find.py --expect 0 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting no wheels to be uploaded" +trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/environment_key/.gitignore b/acceptance/bundle/artifacts/whl_explicit/.gitignore similarity index 100% rename from bundle/tests/python_wheel/environment_key/.gitignore rename to acceptance/bundle/artifacts/whl_explicit/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel/bundle.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml similarity index 77% rename from bundle/tests/python_wheel/python_wheel/bundle.yml rename to acceptance/bundle/artifacts/whl_explicit/databricks.yml index 017fe1c43..45106c0b0 100644 --- a/bundle/tests/python_wheel/python_wheel/bundle.yml +++ b/acceptance/bundle/artifacts/whl_explicit/databricks.yml @@ -5,7 +5,8 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + # using 'python' there because 'python3' does not exist in virtualenv on windows + build: python setup.py bdist_wheel resources: jobs: diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt new file mode 100644 index 000000000..1018501db --- /dev/null +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script new file mode 100644 index 000000000..bb7e26ae1 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel/.gitignore b/acceptance/bundle/artifacts/whl_implicit/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel/.gitignore rename to acceptance/bundle/artifacts/whl_implicit/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml b/acceptance/bundle/artifacts/whl_implicit/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit/databricks.yml diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt new file mode 100644 index 000000000..69ff56c42 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script new file mode 100644 index 000000000..78c4d75e0 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/setup.py b/acceptance/bundle/artifacts/whl_implicit/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/setup.py rename to acceptance/bundle/artifacts/whl_implicit/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt new file mode 100644 index 000000000..0658dce3a --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -0,0 +1,46 @@ + +>>> [CLI] bundle deploy +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +package/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "[default] My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files/package/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script new file mode 100644 index 000000000..d4c2438db --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore b/acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml rename to acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt new file mode 100644 index 000000000..9c7296b9a --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/notebook.py" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script new file mode 100644 index 000000000..bb7e26ae1 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/acceptance/bundle/artifacts/whl_implicit_notebook/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore b/acceptance/bundle/artifacts/whl_multiple/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore rename to acceptance/bundle/artifacts/whl_multiple/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/acceptance/bundle/artifacts/whl_multiple/databricks.yml similarity index 85% rename from bundle/tests/python_wheel/python_wheel_multiple/bundle.yml rename to acceptance/bundle/artifacts/whl_multiple/databricks.yml index 770110416..2bfd85abd 100644 --- a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml +++ b/acceptance/bundle/artifacts/whl_multiple/databricks.yml @@ -5,11 +5,11 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: "python setup.py bdist_wheel" my_test_code_2: type: whl path: "./my_test_code" - build: "python3 setup2.py bdist_wheel" + build: "python setup2.py bdist_wheel" resources: jobs: diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt new file mode 100644 index 000000000..9335b9cc5 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -0,0 +1,42 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Building my_test_code_2... +Deploying resources... +Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code_2-0.0.1-py3-none-any.whl... + +>>> find.py --expect 2 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script new file mode 100644 index 000000000..a475e9f73 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order + +trace find.py --expect 2 whl + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm -fr out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml similarity index 84% rename from bundle/tests/python_wheel/python_wheel_no_build/bundle.yml rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml index e10e3993d..4ad0c6afa 100644 --- a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml @@ -13,4 +13,4 @@ resources: entry_point: "run" libraries: - whl: ./dist/*.whl - - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl + - whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt new file mode 100644 index 000000000..c9cd82fb8 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -0,0 +1,45 @@ + +>>> find.py --expect 2 whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl + +>>> [CLI] bundle deploy +Deploying resources... +Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading other_test_code-0.0.1-py3-none-any.whl... + +=== Expecting to find 2 wheels, same as initially provided +>>> find.py --expect 2 whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script new file mode 100644 index 000000000..5265151e2 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -0,0 +1,14 @@ +trace find.py --expect 2 whl + +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order + +title "Expecting to find 2 wheels, same as initially provided" +trace find.py --expect 2 whl + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/acceptance/bundle/artifacts/whl_via_environment_key/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/.gitignore rename to acceptance/bundle/artifacts/whl_via_environment_key/.gitignore diff --git a/bundle/tests/python_wheel/environment_key/databricks.yml b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml similarity index 92% rename from bundle/tests/python_wheel/environment_key/databricks.yml rename to acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml index 198f8c0d2..4ca3f9113 100644 --- a/bundle/tests/python_wheel/environment_key/databricks.yml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: python setup.py bdist_wheel resources: jobs: diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt new file mode 100644 index 000000000..8b6b781aa --- /dev/null +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -0,0 +1,54 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in environments section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "test_env", + "spec": { + "client": "1", + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + ] + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "environment_key": "test_env", + "existing_cluster_id": "0717-132531-5opeqon1", + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script new file mode 100644 index 000000000..1914aeb8c --- /dev/null +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -0,0 +1,11 @@ +trace $CLI bundle deploy + +trace find.py --expect 1 whl + +title "Expecting 1 wheel in environments section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt deleted file mode 100644 index 13c81c511..000000000 --- a/acceptance/bundle/debug/out.stderr.parallel.txt +++ /dev/null @@ -1,15 +0,0 @@ -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions -10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns -10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync -10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true -< HTTP/0.0 000 OK pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true -< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true -< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt index 147b33a45..b9ac5c4d9 100644 --- a/acceptance/bundle/debug/out.stderr.txt +++ b/acceptance/bundle/debug/out.stderr.txt @@ -72,18 +72,30 @@ 10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=terraform.Initialize 10:07:59 Debug: Apply pid=12345 mutator=scripts.postinit 10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=scripts.postinit -10:07:59 Debug: Apply pid=12345 mutator=validate +10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) +10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:files_to_sync +10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:folder_permissions +10:07:59 Debug: ApplyParallel pid=12345 mutator=validate:validate_sync_patterns +10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_cluster_key_defined +10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:job_task_cluster_spec +10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:SingleNodeCluster +10:07:59 Debug: ApplyParallel pid=12345 mutator=fast_validate(readonly) mutator=validate:artifact_paths 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files < HTTP/1.1 404 Not Found < { < "message": "Workspace path not found" +< } pid=12345 mutator=validate:files_to_sync sdk=true +10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate:files_to_sync sdk=true 10:07:59 Debug: POST /api/2.0/workspace/mkdirs > { > "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" > } +< HTTP/1.1 200 OK pid=12345 mutator=validate:files_to_sync sdk=true 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files < HTTP/1.1 200 OK < { < "object_type": "DIRECTORY", < "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files" +< } pid=12345 mutator=validate:files_to_sync sdk=true +10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate:files_to_sync 10:07:59 Info: completed execution pid=12345 exit_code=0 diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script index 005a1a341..913f07e41 100644 --- a/acceptance/bundle/debug/script +++ b/acceptance/bundle/debug/script @@ -1,4 +1 @@ -$CLI bundle validate --debug 2> full.stderr.txt -grep -vw parallel full.stderr.txt > out.stderr.txt -grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt -rm full.stderr.txt +$CLI bundle validate --debug 2> out.stderr.txt diff --git a/acceptance/bundle/includes/include_outside_root/output.txt b/acceptance/bundle/includes/include_outside_root/output.txt index 720c3a826..f4f49f3ac 100644 --- a/acceptance/bundle/includes/include_outside_root/output.txt +++ b/acceptance/bundle/includes/include_outside_root/output.txt @@ -2,10 +2,8 @@ Warning: Include section is defined outside root file at include in a.yml:2:3 -The include section is defined in a file that is not the root file. -These values will be ignored because only the includes defined in -the bundle root file (that is databricks.yml or databricks.yaml) -are loaded. +An include section is defined in a file that is not databricks.yml. +Only includes defined in databricks.yml are applied. Name: include_outside_root Target: default diff --git a/acceptance/bundle/state/databricks.yml b/acceptance/bundle/state/databricks.yml new file mode 100644 index 000000000..4775ba33d --- /dev/null +++ b/acceptance/bundle/state/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: state + +resources: + jobs: + test: + name: "test" + tasks: + - task_key: "test-task" + spark_python_task: + python_file: ./test.py + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode diff --git a/acceptance/bundle/state/out.state.txt b/acceptance/bundle/state/out.state.txt new file mode 100644 index 000000000..96966812b --- /dev/null +++ b/acceptance/bundle/state/out.state.txt @@ -0,0 +1,4 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/state/default/state/terraform.tfstate" +} diff --git a/acceptance/bundle/state/output.txt b/acceptance/bundle/state/output.txt new file mode 100644 index 000000000..ac13a7ba4 --- /dev/null +++ b/acceptance/bundle/state/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/state/script b/acceptance/bundle/state/script new file mode 100644 index 000000000..9977f7d52 --- /dev/null +++ b/acceptance/bundle/state/script @@ -0,0 +1,4 @@ +trace $CLI bundle deploy +trace $CLI bundle deploy # We do 2 deploys because only 2nd deploy will pull state from remote after 1st created it +jq 'select(.path == "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/state/default/state/terraform.tfstate")' out.requests.txt > out.state.txt +rm out.requests.txt diff --git a/acceptance/bundle/state/test.py b/acceptance/bundle/state/test.py new file mode 100644 index 000000000..f1a18139c --- /dev/null +++ b/acceptance/bundle/state/test.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml new file mode 100644 index 000000000..7061886de --- /dev/null +++ b/acceptance/bundle/state/test.toml @@ -0,0 +1,2 @@ +Cloud = false +RecordRequests = true diff --git a/acceptance/bundle/templates/dbt-sql/script b/acceptance/bundle/templates/dbt-sql/script index 3a2660de5..427f655a6 100644 --- a/acceptance/bundle/templates/dbt-sql/script +++ b/acceptance/bundle/templates/dbt-sql/script @@ -6,3 +6,4 @@ trace $CLI bundle validate -t prod # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore +rm .databricks/.gitignore diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 7e5524065..955e05925 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -6,6 +6,7 @@ trace $CLI bundle validate -t prod # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore +rm .databricks/.gitignore cd ../../ diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index e5fcb7741..d7b047fec 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -6,3 +6,4 @@ trace $CLI bundle validate -t prod # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore +rm .databricks/.gitignore diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/out.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/out.gitignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/out.gitignore @@ -0,0 +1 @@ +* diff --git a/acceptance/bundle/templates/default-sql/script b/acceptance/bundle/templates/default-sql/script index 7ea0d863c..01b393a8c 100644 --- a/acceptance/bundle/templates/default-sql/script +++ b/acceptance/bundle/templates/default-sql/script @@ -6,3 +6,6 @@ trace $CLI bundle validate -t prod # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore + +# Only for this test (default-sql), record .databricks/.gitignore in the output +mv .databricks/.gitignore .databricks/out.gitignore diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/script b/acceptance/bundle/templates/experimental-jobs-as-code/script index 08e48fc5f..ed9a07dd5 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/script +++ b/acceptance/bundle/templates/experimental-jobs-as-code/script @@ -11,3 +11,4 @@ rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore +rm .databricks/.gitignore diff --git a/acceptance/cmd/workspace/apps/output.txt b/acceptance/cmd/workspace/apps/output.txt index 4d9f80f44..cbe5f8b57 100644 --- a/acceptance/cmd/workspace/apps/output.txt +++ b/acceptance/cmd/workspace/apps/output.txt @@ -35,9 +35,10 @@ Usage: databricks apps update NAME [flags] Flags: - --description string The description of the app. - -h, --help help for update - --json JSON either inline JSON string or @path/to/file.json with request body (default JSON (0 bytes)) + --budget-policy-id string + --description string The description of the app. + -h, --help help for update + --json JSON either inline JSON string or @path/to/file.json with request body (default JSON (0 bytes)) Global Flags: --debug enable debug logging diff --git a/acceptance/config_test.go b/acceptance/config_test.go index 4edfee69d..cc5257c65 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -11,6 +11,7 @@ import ( "github.com/BurntSushi/toml" "github.com/databricks/cli/libs/testdiff" "github.com/databricks/cli/libs/testserver" + ignore "github.com/sabhiram/go-gitignore" "github.com/stretchr/testify/require" ) @@ -51,6 +52,11 @@ type TestConfig struct { // List of request headers to include when recording requests. IncludeRequestHeaders []string + + // List of gitignore patterns to ignore when checking output files + Ignore []string + + CompiledIgnoreObject *ignore.GitIgnore } type ServerStub struct { @@ -111,6 +117,8 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) { } } + result.CompiledIgnoreObject = ignore.CompileIgnoreLines(result.Ignore...) + return result, strings.Join(configs, ", ") } diff --git a/acceptance/selftest/basic/script b/acceptance/selftest/basic/script index bccf30e71..a3ec98402 100644 --- a/acceptance/selftest/basic/script +++ b/acceptance/selftest/basic/script @@ -27,3 +27,7 @@ echo 123456 printf "\n=== Testing --version" trace $CLI --version + +touch ignored_file.txt +mkdir ignored_dir +touch ignored_dir/hello.txt diff --git a/acceptance/selftest/basic/test.toml b/acceptance/selftest/basic/test.toml index 762e28ceb..3ca3d9255 100644 --- a/acceptance/selftest/basic/test.toml +++ b/acceptance/selftest/basic/test.toml @@ -1,5 +1,8 @@ # Badness = "Brief description of what's wrong with the test output, if anything" +Ignore = ['ignore*'] + + #[GOOS] # Disable on Windows #windows = false diff --git a/acceptance/server_test.go b/acceptance/server_test.go index 402e3ca5f..7ef705931 100644 --- a/acceptance/server_test.go +++ b/acceptance/server_test.go @@ -111,6 +111,11 @@ func AddHandlers(server *testserver.Server) { return "" }) + server.Handle("GET", "/api/2.0/workspace-files/{path:.*}", func(req testserver.Request) any { + path := req.Vars["path"] + return req.Workspace.WorkspaceFilesExportFile(path) + }) + server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(req testserver.Request) any { return testMetastore }) diff --git a/acceptance/terraform/main.tf b/acceptance/terraform/main.tf index 674b41a3c..822bc1a5a 100644 --- a/acceptance/terraform/main.tf +++ b/acceptance/terraform/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { databricks = { source = "databricks/databricks" - version = "1.65.1" + version = "1.68.0" } } diff --git a/acceptance/terraform/output.txt b/acceptance/terraform/output.txt index 851785827..f6280c5dc 100644 --- a/acceptance/terraform/output.txt +++ b/acceptance/terraform/output.txt @@ -4,9 +4,9 @@ Initializing the backend... Initializing provider plugins... -- Finding databricks/databricks versions matching "1.65.1"... -- Installing databricks/databricks v1.65.1... -- Installed databricks/databricks v1.65.1 (unauthenticated) +- Finding databricks/databricks versions matching "1.68.0"... +- Installing databricks/databricks v1.68.0... +- Installed databricks/databricks v1.68.0 (unauthenticated) Terraform has created a lock file .terraform.lock.hcl to record the provider selections it made above. Include this file in your version control repository diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index fc50aeafc..7a4fe7126 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -3,8 +3,6 @@ package apps import ( "context" "fmt" - "path" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -14,7 +12,6 @@ type validate struct{} func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - possibleConfigFiles := []string{"app.yml", "app.yaml"} usedSourceCodePaths := make(map[string]string) for key, app := range b.Config.Resources.Apps { @@ -28,16 +25,12 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } usedSourceCodePaths[app.SourceCodePath] = key - for _, configFile := range possibleConfigFiles { - appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath) - cf := path.Join(appPath, configFile) - if _, err := b.SyncRoot.Stat(cf); err == nil { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: configFile + " detected", - Detail: fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name), - }) - } + if app.Config != nil { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "App config section detected", + Detail: fmt.Sprintf("remove 'config' from app resource '%s' section and use app.yml file in the root of this app instead", key), + }) } } diff --git a/bundle/apps/validate_test.go b/bundle/apps/validate_test.go index 11270198e..d218f96ca 100644 --- a/bundle/apps/validate_test.go +++ b/bundle/apps/validate_test.go @@ -17,46 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestAppsValidate(t *testing.T) { - tmpDir := t.TempDir() - testutil.Touch(t, tmpDir, "app1", "app.yml") - testutil.Touch(t, tmpDir, "app2", "app.py") - - b := &bundle.Bundle{ - BundleRootPath: tmpDir, - SyncRootPath: tmpDir, - SyncRoot: vfs.MustNew(tmpDir), - Config: config.Root{ - Workspace: config.Workspace{ - FilePath: "/foo/bar/", - }, - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "app1": { - App: &apps.App{ - Name: "app1", - }, - SourceCodePath: "./app1", - }, - "app2": { - App: &apps.App{ - Name: "app2", - }, - SourceCodePath: "./app2", - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) - - diags := bundle.ApplySeq(context.Background(), b, mutator.TranslatePaths(), Validate()) - require.Len(t, diags, 1) - require.Equal(t, "app.yml detected", diags[0].Summary) - require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource") -} - func TestAppsValidateSameSourcePath(t *testing.T) { tmpDir := t.TempDir() testutil.Touch(t, tmpDir, "app1", "app.py") diff --git a/bundle/bundle.go b/bundle/bundle.go index 9cb8916f5..dad99d6e1 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -21,6 +21,7 @@ import ( "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/locker" "github.com/databricks/cli/libs/log" + libsync "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/tags" "github.com/databricks/cli/libs/terraform" "github.com/databricks/cli/libs/vfs" @@ -198,6 +199,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) return "", err } + libsync.WriteGitIgnore(ctx, b.BundleRootPath) return dir, nil } diff --git a/bundle/bundle_read_only.go b/bundle/bundle_read_only.go deleted file mode 100644 index 4bdd94e59..000000000 --- a/bundle/bundle_read_only.go +++ /dev/null @@ -1,49 +0,0 @@ -package bundle - -import ( - "context" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/vfs" - "github.com/databricks/databricks-sdk-go" -) - -type ReadOnlyBundle struct { - b *Bundle -} - -func ReadOnly(b *Bundle) ReadOnlyBundle { - return ReadOnlyBundle{b: b} -} - -func (r ReadOnlyBundle) Config() config.Root { - return r.b.Config -} - -func (r ReadOnlyBundle) RootPath() string { - return r.b.BundleRootPath -} - -func (r ReadOnlyBundle) BundleRoot() vfs.Path { - return r.b.BundleRoot -} - -func (r ReadOnlyBundle) SyncRoot() vfs.Path { - return r.b.SyncRoot -} - -func (r ReadOnlyBundle) WorktreeRoot() vfs.Path { - return r.b.WorktreeRoot -} - -func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient { - return r.b.WorkspaceClient() -} - -func (r ReadOnlyBundle) CacheDir(ctx context.Context, paths ...string) (string, error) { - return r.b.CacheDir(ctx, paths...) -} - -func (r ReadOnlyBundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { - return r.b.GetSyncIncludePatterns(ctx) -} diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 4f7dd47ca..9458ff09e 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -165,10 +165,8 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "Include section is defined outside root file", - Detail: `The include section is defined in a file that is not the root file. -These values will be ignored because only the includes defined in -the bundle root file (that is databricks.yml or databricks.yaml) -are loaded.`, + Detail: `An include section is defined in a file that is not databricks.yml. +Only includes defined in databricks.yml are applied.`, Locations: this.GetLocations("include"), Paths: []dyn.Path{dyn.MustPathFromString("include")}, }) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 1f523fed3..866bd7641 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -112,6 +112,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) } } + for k := range r.Schemas { + if k == key { + found = append(found, r.Schemas[k]) + } + } + if len(found) == 0 { return nil, fmt.Errorf("no such resource: %s", key) } diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go index b638907ac..d9849fd2d 100644 --- a/bundle/config/resources/schema.go +++ b/bundle/config/resources/schema.go @@ -6,6 +6,10 @@ import ( "net/url" "strings" + "github.com/databricks/databricks-sdk-go/apierr" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -25,8 +29,23 @@ type Schema struct { URL string `json:"url,omitempty" bundle:"internal"` } -func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { - return false, errors.New("schema.Exists() is not supported") +func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { + log.Tracef(ctx, "Checking if schema with fullName=%s exists", fullName) + + _, err := w.Schemas.GetByFullName(ctx, fullName) + if err != nil { + log.Debugf(ctx, "schema with full name %s does not exist: %v", fullName, err) + + var aerr *apierr.APIError + if errors.As(err, &aerr) { + if aerr.StatusCode == 404 { + return false, nil + } + } + + return false, err + } + return true, nil } func (s *Schema) TerraformResourceName() string { diff --git a/bundle/config/resources/schema_test.go b/bundle/config/resources/schema_test.go new file mode 100644 index 000000000..b609b6565 --- /dev/null +++ b/bundle/config/resources/schema_test.go @@ -0,0 +1,26 @@ +package resources + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestSchemaNotFound(t *testing.T) { + ctx := context.Background() + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockSchemasAPI().On("GetByFullName", mock.Anything, "non-existent-schema").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + + s := &Schema{} + exists, err := s.Exists(ctx, m.WorkspaceClient, "non-existent-schema") + + require.Falsef(t, exists, "Exists should return false when getting a 404 response from Workspace") + require.NoErrorf(t, err, "Exists should not return an error when getting a 404 response from Workspace") +} diff --git a/bundle/config/root.go b/bundle/config/root.go index b974bcec5..d44e25fa2 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -52,7 +52,7 @@ type Root struct { Targets map[string]*Target `json:"targets,omitempty"` // DEPRECATED. Left for backward compatibility with Targets - Environments map[string]*Target `json:"environments,omitempty" bundle:"deprecated"` + Environments map[string]*Target `json:"environments,omitempty"` // Sync section specifies options for files synchronization Sync Sync `json:"sync,omitempty"` diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index 47d83036d..e116034ad 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -14,18 +14,18 @@ import ( // 2. The validation is blocking for bundle deployments. // // The full suite of validation mutators is available in the [Validate] mutator. -type fastValidateReadonly struct{} +type fastValidate struct{ bundle.RO } -func FastValidateReadonly() bundle.ReadOnlyMutator { - return &fastValidateReadonly{} +func FastValidate() bundle.ReadOnlyMutator { + return &fastValidate{} } -func (f *fastValidateReadonly) Name() string { +func (f *fastValidate) Name() string { return "fast_validate(readonly)" } -func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { - return bundle.ApplyReadOnly(ctx, rb, bundle.Parallel( +func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagnostics { + return bundle.ApplyParallel(ctx, rb, // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), @@ -33,19 +33,5 @@ func (f *fastValidateReadonly) Apply(ctx context.Context, rb bundle.ReadOnlyBund // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), - )) -} - -type fastValidate struct{} - -func FastValidate() bundle.Mutator { - return &fastValidate{} -} - -func (f *fastValidate) Name() string { - return "fast_validate" -} - -func (f *fastValidate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), FastValidateReadonly()) + ) } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index b4de06773..aea78f710 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -13,20 +13,20 @@ func FilesToSync() bundle.ReadOnlyMutator { return &filesToSync{} } -type filesToSync struct{} +type filesToSync struct{ bundle.RO } func (v *filesToSync) Name() string { return "validate:files_to_sync" } -func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { +func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // The user may be intentional about not synchronizing any files. // In this case, we should not show any warnings. - if len(rb.Config().Sync.Paths) == 0 { + if len(b.Config.Sync.Paths) == 0 { return nil } - sync, err := files.GetSync(ctx, rb) + sync, err := files.GetSync(ctx, b) if err != nil { return diag.FromErr(err) } @@ -42,20 +42,20 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag. } diags := diag.Diagnostics{} - if len(rb.Config().Sync.Exclude) == 0 { + if len(b.Config.Sync.Exclude) == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore", }) } else { - loc := location{path: "sync.exclude", rb: rb} + path := "sync.exclude" diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. - Locations: loc.Locations(), - Paths: []dyn.Path{loc.Path()}, + Locations: b.Config.GetLocations(path), + Paths: []dyn.Path{dyn.MustPathFromString(path)}, }) } diff --git a/bundle/config/validate/files_to_sync_test.go b/bundle/config/validate/files_to_sync_test.go index dd40295c3..0a5f69727 100644 --- a/bundle/config/validate/files_to_sync_test.go +++ b/bundle/config/validate/files_to_sync_test.go @@ -29,8 +29,7 @@ func TestFilesToSync_NoPaths(t *testing.T) { } ctx := context.Background() - rb := bundle.ReadOnly(b) - diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) + diags := FilesToSync().Apply(ctx, b) assert.Empty(t, diags) } @@ -85,8 +84,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) { testutil.WriteFile(t, filepath.Join(b.BundleRootPath, ".gitignore"), "*\n.*\n") ctx := context.Background() - rb := bundle.ReadOnly(b) - diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) + diags := FilesToSync().Apply(ctx, b) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary) @@ -99,8 +97,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) { b.Config.Sync.Exclude = []string{"*"} ctx := context.Background() - rb := bundle.ReadOnly(b) - diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync()) + diags := FilesToSync().Apply(ctx, b) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary) diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go index 7b12b4b16..575702c34 100644 --- a/bundle/config/validate/folder_permissions.go +++ b/bundle/config/validate/folder_permissions.go @@ -16,15 +16,14 @@ import ( "golang.org/x/sync/errgroup" ) -type folderPermissions struct{} +type folderPermissions struct{ bundle.RO } -// Apply implements bundle.ReadOnlyMutator. -func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) diag.Diagnostics { - if len(b.Config().Permissions) == 0 { +func (f *folderPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if len(b.Config.Permissions) == 0 { return nil } - bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config().Workspace) + bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config.Workspace) var diags diag.Diagnostics g, ctx := errgroup.WithContext(ctx) @@ -48,7 +47,7 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) return diags } -func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderPath string) diag.Diagnostics { +func checkFolderPermission(ctx context.Context, b *bundle.Bundle, folderPath string) diag.Diagnostics { // If the folder is shared, then we don't need to check permissions as it was already checked in the other mutator before. if libraries.IsWorkspaceSharedPath(folderPath) { return nil @@ -69,7 +68,7 @@ func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderP } p := permissions.ObjectAclToResourcePermissions(folderPath, objPermissions.AccessControlList) - return p.Compare(b.Config().Permissions) + return p.Compare(b.Config.Permissions) } func getClosestExistingObject(ctx context.Context, w workspace.WorkspaceInterface, folderPath string) (*workspace.ObjectInfo, error) { diff --git a/bundle/config/validate/folder_permissions_test.go b/bundle/config/validate/folder_permissions_test.go index 8e68c9fbf..40906ce6f 100644 --- a/bundle/config/validate/folder_permissions_test.go +++ b/bundle/config/validate/folder_permissions_test.go @@ -69,9 +69,7 @@ func TestFolderPermissionsInheritedWhenRootPathDoesNotExist(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - rb := bundle.ReadOnly(b) - - diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + diags := ValidateFolderPermissions().Apply(context.Background(), b) require.Empty(t, diags) } @@ -118,9 +116,7 @@ func TestValidateFolderPermissionsFailsOnMissingBundlePermission(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - rb := bundle.ReadOnly(b) - - diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + diags := ValidateFolderPermissions().Apply(context.Background(), b) require.Len(t, diags, 1) require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary) require.Equal(t, diag.Warning, diags[0].Severity) @@ -164,9 +160,7 @@ func TestValidateFolderPermissionsFailsOnPermissionMismatch(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - rb := bundle.ReadOnly(b) - - diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + diags := ValidateFolderPermissions().Apply(context.Background(), b) require.Len(t, diags, 1) require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary) require.Equal(t, diag.Warning, diags[0].Severity) @@ -199,9 +193,7 @@ func TestValidateFolderPermissionsFailsOnNoRootFolder(t *testing.T) { }) b.SetWorkpaceClient(m.WorkspaceClient) - rb := bundle.ReadOnly(b) - - diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + diags := ValidateFolderPermissions().Apply(context.Background(), b) require.Len(t, diags, 1) require.Equal(t, "folder / and its parent folders do not exist", diags[0].Summary) require.Equal(t, diag.Error, diags[0].Severity) diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index 83d6c6ae1..1a5b64a26 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -24,7 +24,10 @@ func (f *noInterpolationInAuthConfig) Name() string { func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { authFields := []string{ // Generic attributes. - "host", "profile", "auth_type", "metadata_service_url", + "host", + "profile", + "auth_type", + "metadata_service_url", // OAuth specific attributes. "client_id", @@ -33,8 +36,12 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl "google_service_account", // Azure specific attributes. - "azure_resource_id", "azure_use_msi", "azure_client_id", "azure_tenant_id", - "azure_environment", "azure_login_app_id", + "azure_resource_id", + "azure_use_msi", + "azure_client_id", + "azure_tenant_id", + "azure_environment", + "azure_login_app_id", } diags := diag.Diagnostics{} @@ -49,12 +56,11 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl return diag.FromErr(err) } - if v.Kind() == dyn.KindInvalid || v.Kind() == dyn.KindNil { + vv, ok := v.AsString() + if !ok { continue } - vv := v.MustString() - // Check if the field contains interpolation. if dynvar.ContainsVariableReference(vv) { envVar, ok := auth.GetEnvFor(fieldName) diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index c3a1ab3df..5ae2f5437 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -13,16 +13,16 @@ func JobClusterKeyDefined() bundle.ReadOnlyMutator { return &jobClusterKeyDefined{} } -type jobClusterKeyDefined struct{} +type jobClusterKeyDefined struct{ bundle.RO } func (v *jobClusterKeyDefined) Name() string { return "validate:job_cluster_key_defined" } -func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { +func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - for k, job := range rb.Config().Resources.Jobs { + for k, job := range b.Config.Resources.Jobs { jobClusterKeys := make(map[string]bool) for _, cluster := range job.JobClusters { if cluster.JobClusterKey != "" { @@ -33,10 +33,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund for index, task := range job.Tasks { if task.JobClusterKey != "" { if _, ok := jobClusterKeys[task.JobClusterKey]; !ok { - loc := location{ - path: fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index), - rb: rb, - } + path := fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index) diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, @@ -44,8 +41,8 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund // Show only the location where the job_cluster_key is defined. // Other associated locations are not relevant since they are // overridden during merging. - Locations: []dyn.Location{loc.Location()}, - Paths: []dyn.Path{loc.Path()}, + Locations: b.Config.GetLocations(path), + Paths: []dyn.Path{dyn.MustPathFromString(path)}, }) } } diff --git a/bundle/config/validate/job_cluster_key_defined_test.go b/bundle/config/validate/job_cluster_key_defined_test.go index 2cbdb7c6a..559bd1c46 100644 --- a/bundle/config/validate/job_cluster_key_defined_test.go +++ b/bundle/config/validate/job_cluster_key_defined_test.go @@ -33,7 +33,7 @@ func TestJobClusterKeyDefined(t *testing.T) { }, } - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) + diags := JobClusterKeyDefined().Apply(context.Background(), b) require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -56,7 +56,7 @@ func TestJobClusterKeyNotDefined(t *testing.T) { }, } - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) + diags := JobClusterKeyDefined().Apply(context.Background(), b) require.Len(t, diags, 1) require.NoError(t, diags.Error()) require.Equal(t, diag.Warning, diags[0].Severity) @@ -89,7 +89,7 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { }, } - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobClusterKeyDefined()) + diags := JobClusterKeyDefined().Apply(context.Background(), b) require.Len(t, diags, 1) require.NoError(t, diags.Error()) require.Equal(t, diag.Warning, diags[0].Severity) diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 5f532acfe..79672be63 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -17,31 +17,31 @@ func JobTaskClusterSpec() bundle.ReadOnlyMutator { return &jobTaskClusterSpec{} } -type jobTaskClusterSpec struct{} +type jobTaskClusterSpec struct{ bundle.RO } func (v *jobTaskClusterSpec) Name() string { return "validate:job_task_cluster_spec" } -func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { +func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) - for resourceName, job := range rb.Config().Resources.Jobs { + for resourceName, job := range b.Config.Resources.Jobs { resourcePath := jobsPath.Append(dyn.Key(resourceName)) for taskIndex, task := range job.Tasks { taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) - diags = diags.Extend(validateJobTask(rb, task, taskPath)) + diags = diags.Extend(validateJobTask(b, task, taskPath)) } } return diags } -func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { +func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { diags := diag.Diagnostics{} var specified []string @@ -74,7 +74,7 @@ func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path if task.ForEachTask != nil { forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task")) - diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath)) + diags = diags.Extend(validateJobTask(b, task.ForEachTask.Task, forEachTaskPath)) } if isComputeTask(task) && len(specified) == 0 { @@ -92,7 +92,7 @@ func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path Severity: diag.Error, Summary: "Missing required cluster or environment settings", Detail: detail, - Locations: rb.Config().GetLocations(taskPath.String()), + Locations: b.Config.GetLocations(taskPath.String()), Paths: []dyn.Path{taskPath}, }) } diff --git a/bundle/config/validate/job_task_cluster_spec_test.go b/bundle/config/validate/job_task_cluster_spec_test.go index a3a7ccf25..fd316d61f 100644 --- a/bundle/config/validate/job_task_cluster_spec_test.go +++ b/bundle/config/validate/job_task_cluster_spec_test.go @@ -174,7 +174,7 @@ Specify one of the following fields: job_cluster_key, environment_key, existing_ } b := createBundle(map[string]*resources.Job{"job1": job}) - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec()) + diags := JobTaskClusterSpec().Apply(context.Background(), b) if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" { assert.Len(t, diags, 1) diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index 7c159f61a..acc5f444d 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -16,7 +16,7 @@ func SingleNodeCluster() bundle.ReadOnlyMutator { return &singleNodeCluster{} } -type singleNodeCluster struct{} +type singleNodeCluster struct{ bundle.RO } func (m *singleNodeCluster) Name() string { return "validate:SingleNodeCluster" @@ -98,7 +98,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return false } -func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { +func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} patterns := []dyn.Pattern{ @@ -115,7 +115,7 @@ func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) } for _, p := range patterns { - _, err := dyn.MapByPattern(rb.Config().Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + _, err := dyn.MapByPattern(b.Config.Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { warning := diag.Diagnostic{ Severity: diag.Warning, Summary: singleNodeWarningSummary, diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index be93420c6..c6d5f8ca9 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -116,7 +116,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0)) }) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -165,7 +165,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) }) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := SingleNodeCluster().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -214,7 +214,7 @@ func TestValidateSingleNodeClusterFailForJobTaskClusters(t *testing.T) { return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0)) }) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -260,7 +260,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) { return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0)) }) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -313,7 +313,7 @@ func TestValidateSingleNodeClusterFailForJobForEachTaskCluster(t *testing.T) { return dyn.Set(v, "resources.jobs.foo.tasks[0].for_each_task.task.new_cluster.num_workers", dyn.V(0)) }) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -397,7 +397,7 @@ func TestValidateSingleNodeClusterPassInteractiveClusters(t *testing.T) { }) } - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) }) } @@ -437,7 +437,7 @@ func TestValidateSingleNodeClusterPassJobClusters(t *testing.T) { }) } - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) }) } @@ -477,7 +477,7 @@ func TestValidateSingleNodeClusterPassJobTaskClusters(t *testing.T) { }) } - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) }) } @@ -514,7 +514,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) { }) } - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) }) } @@ -558,7 +558,7 @@ func TestValidateSingleNodeClusterPassJobForEachTaskCluster(t *testing.T) { }) } - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster()) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Empty(t, diags) }) } diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index 8fdd704ab..83d17337d 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -5,46 +5,16 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) -type validate struct{} - -type location struct { - path string - rb bundle.ReadOnlyBundle -} - -func (l location) Location() dyn.Location { - return l.rb.Config().GetLocation(l.path) -} - -func (l location) Locations() []dyn.Location { - return l.rb.Config().GetLocations(l.path) -} - -func (l location) Path() dyn.Path { - return dyn.MustPathFromString(l.path) -} - -// Apply implements bundle.Mutator. -func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - return bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), bundle.Parallel( - FastValidateReadonly(), +func Validate(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + return bundle.ApplyParallel(ctx, b, + FastValidate(), // Slow mutators that require network or file i/o. These are only // run in the `bundle validate` command. FilesToSync(), ValidateFolderPermissions(), ValidateSyncPatterns(), - )) -} - -// Name implements bundle.Mutator. -func (v *validate) Name() string { - return "validate" -} - -func Validate() bundle.Mutator { - return &validate{} + ) } diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index aa4492670..78536d4bd 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -16,7 +16,7 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" ) -type validateArtifactPath struct{} +type validateArtifactPath struct{ bundle.RO } func ValidateArtifactPath() bundle.ReadOnlyMutator { return &validateArtifactPath{} @@ -74,9 +74,9 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin return nil, nil, false } -func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { +func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We only validate UC Volumes paths right now. - if !libraries.IsVolumesPath(rb.Config().Workspace.ArtifactPath) { + if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) { return nil } @@ -85,25 +85,25 @@ func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBund { Summary: s, Severity: diag.Error, - Locations: rb.Config().GetLocations("workspace.artifact_path"), + Locations: b.Config.GetLocations("workspace.artifact_path"), Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, }, } } - catalogName, schemaName, volumeName, err := extractVolumeFromPath(rb.Config().Workspace.ArtifactPath) + catalogName, schemaName, volumeName, err := extractVolumeFromPath(b.Config.Workspace.ArtifactPath) if err != nil { return wrapErrorMsg(err.Error()) } volumeFullName := fmt.Sprintf("%s.%s.%s", catalogName, schemaName, volumeName) - w := rb.WorkspaceClient() + w := b.WorkspaceClient() _, err = w.Volumes.ReadByName(ctx, volumeFullName) if errors.Is(err, apierr.ErrPermissionDenied) { return wrapErrorMsg(fmt.Sprintf("cannot access volume %s: %s", volumeFullName, err)) } if errors.Is(err, apierr.ErrNotFound) { - path, locations, ok := findVolumeInBundle(rb.Config(), catalogName, schemaName, volumeName) + path, locations, ok := findVolumeInBundle(b.Config, catalogName, schemaName, volumeName) if !ok { return wrapErrorMsg(fmt.Sprintf("volume %s does not exist", volumeFullName)) } @@ -117,7 +117,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, rb bundle.ReadOnlyBund this bundle but which has not been deployed yet. Please first deploy the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, - Locations: slices.Concat(rb.Config().GetLocations("workspace.artifact_path"), locations), + Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), }} diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index e1ae6af34..3f998567b 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -49,7 +49,7 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { }) b.SetWorkpaceClient(m.WorkspaceClient) - diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), ValidateArtifactPath()) + diags := ValidateArtifactPath().Apply(ctx, b) assert.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "volume catalogN.schemaN.volumeN does not exist", @@ -88,7 +88,6 @@ func TestValidateArtifactPath(t *testing.T) { }}, diags) } - rb := bundle.ReadOnly(b) ctx := context.Background() tcases := []struct { @@ -123,7 +122,7 @@ func TestValidateArtifactPath(t *testing.T) { api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err) b.SetWorkpaceClient(m.WorkspaceClient) - diags := bundle.ApplyReadOnly(ctx, rb, ValidateArtifactPath()) + diags := ValidateArtifactPath().Apply(ctx, b) assertDiags(t, diags, tc.expectedSummary) } } @@ -167,7 +166,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), ValidateArtifactPath()) + diags := ValidateArtifactPath().Apply(context.Background(), b) require.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 04acd28ab..04df36e4f 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -17,24 +17,24 @@ func ValidateSyncPatterns() bundle.ReadOnlyMutator { return &validateSyncPatterns{} } -type validateSyncPatterns struct{} +type validateSyncPatterns struct{ bundle.RO } func (v *validateSyncPatterns) Name() string { return "validate:validate_sync_patterns" } -func (v *validateSyncPatterns) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { - s := rb.Config().Sync +func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + s := b.Config.Sync if len(s.Exclude) == 0 && len(s.Include) == 0 { return nil } - diags, err := checkPatterns(s.Exclude, "sync.exclude", rb) + diags, err := checkPatterns(s.Exclude, "sync.exclude", b) if err != nil { return diag.FromErr(err) } - includeDiags, err := checkPatterns(s.Include, "sync.include", rb) + includeDiags, err := checkPatterns(s.Include, "sync.include", b) if err != nil { return diag.FromErr(err) } @@ -42,7 +42,7 @@ func (v *validateSyncPatterns) Apply(ctx context.Context, rb bundle.ReadOnlyBund return diags.Extend(includeDiags) } -func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (diag.Diagnostics, error) { +func checkPatterns(patterns []string, path string, b *bundle.Bundle) (diag.Diagnostics, error) { var mu sync.Mutex var errs errgroup.Group var diags diag.Diagnostics @@ -55,7 +55,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di // it means: do not include these files into result set p := strings.TrimPrefix(pattern, "!") errs.Go(func() error { - fs, err := fileset.NewGlobSet(rb.BundleRoot(), []string{p}) + fs, err := fileset.NewGlobSet(b.BundleRoot, []string{p}) if err != nil { return err } @@ -66,13 +66,13 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di } if len(all) == 0 { - loc := location{path: fmt.Sprintf("%s[%d]", path, index), rb: rb} + path := fmt.Sprintf("%s[%d]", path, index) mu.Lock() diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", pattern), - Locations: []dyn.Location{loc.Location()}, - Paths: []dyn.Path{loc.Path()}, + Locations: b.Config.GetLocations(path), + Paths: []dyn.Path{dyn.MustPathFromString(path)}, }) mu.Unlock() } diff --git a/bundle/deploy/filer.go b/bundle/deploy/filer.go index c0fd839ef..b65f08a67 100644 --- a/bundle/deploy/filer.go +++ b/bundle/deploy/filer.go @@ -1,14 +1,94 @@ package deploy import ( + "bytes" + "context" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "strings" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/client" ) // FilerFactory is a function that returns a filer.Filer. type FilerFactory func(b *bundle.Bundle) (filer.Filer, error) -// StateFiler returns a filer.Filer that can be used to read/write state files. -func StateFiler(b *bundle.Bundle) (filer.Filer, error) { - return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.StatePath) +type stateFiler struct { + filer filer.Filer + + apiClient *client.DatabricksClient + root filer.WorkspaceRootPath +} + +func (s stateFiler) Delete(ctx context.Context, path string, mode ...filer.DeleteMode) error { + return s.filer.Delete(ctx, path, mode...) +} + +// Mkdir implements filer.Filer. +func (s stateFiler) Mkdir(ctx context.Context, path string) error { + return s.filer.Mkdir(ctx, path) +} + +func (s stateFiler) Read(ctx context.Context, path string) (io.ReadCloser, error) { + absPath, err := s.root.Join(path) + if err != nil { + return nil, err + } + + stat, err := s.Stat(ctx, path) + if err != nil { + return nil, err + } + if stat.IsDir() { + return nil, fmt.Errorf("not a file: %s", absPath) + } + + var buf bytes.Buffer + urlPath := "/api/2.0/workspace-files/" + url.PathEscape(strings.TrimLeft(absPath, "/")) + err = s.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, nil, &buf) + if err != nil { + return nil, err + } + + return io.NopCloser(&buf), nil +} + +func (s stateFiler) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) { + return s.filer.ReadDir(ctx, path) +} + +func (s stateFiler) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + return s.filer.Stat(ctx, name) +} + +func (s stateFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { + return s.filer.Write(ctx, path, reader, mode...) +} + +// StateFiler returns a filer.Filer that can be used to read/write state files. +// We use a custom workspace filer which uses workspace-files API to read state files. +// This API has a higher than 10 MB limits and allows to export large state files. +// We don't use the same API for read because it doesn't correct get the file content for notebooks and returns +// "File Not Found" error instead. +func StateFiler(b *bundle.Bundle) (filer.Filer, error) { + f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.StatePath) + if err != nil { + return nil, err + } + + apiClient, err := client.New(b.WorkspaceClient().Config) + if err != nil { + return nil, fmt.Errorf("failed to create API client: %w", err) + } + + return stateFiler{ + filer: f, + root: filer.NewWorkspaceRootPath(b.Config.Workspace.StatePath), + apiClient: apiClient, + }, nil } diff --git a/bundle/deploy/files/delete.go b/bundle/deploy/files/delete.go index bb28c2722..971186d5b 100644 --- a/bundle/deploy/files/delete.go +++ b/bundle/deploy/files/delete.go @@ -40,7 +40,7 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } func deleteSnapshotFile(ctx context.Context, b *bundle.Bundle) error { - opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b)) + opts, err := GetSyncOptions(ctx, b) if err != nil { return fmt.Errorf("cannot get sync options: %w", err) } diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index e3abc5fef..a3ead13a4 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -8,43 +8,43 @@ import ( "github.com/databricks/cli/libs/sync" ) -func GetSync(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.Sync, error) { - opts, err := GetSyncOptions(ctx, rb) +func GetSync(ctx context.Context, b *bundle.Bundle) (*sync.Sync, error) { + opts, err := GetSyncOptions(ctx, b) if err != nil { return nil, fmt.Errorf("cannot get sync options: %w", err) } return sync.New(ctx, *opts) } -func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOptions, error) { - cacheDir, err := rb.CacheDir(ctx) +func GetSyncOptions(ctx context.Context, b *bundle.Bundle) (*sync.SyncOptions, error) { + cacheDir, err := b.CacheDir(ctx) if err != nil { return nil, fmt.Errorf("cannot get bundle cache directory: %w", err) } - includes, err := rb.GetSyncIncludePatterns(ctx) + includes, err := b.GetSyncIncludePatterns(ctx) if err != nil { return nil, fmt.Errorf("cannot get list of sync includes: %w", err) } opts := &sync.SyncOptions{ - WorktreeRoot: rb.WorktreeRoot(), - LocalRoot: rb.SyncRoot(), - Paths: rb.Config().Sync.Paths, + WorktreeRoot: b.WorktreeRoot, + LocalRoot: b.SyncRoot, + Paths: b.Config.Sync.Paths, Include: includes, - Exclude: rb.Config().Sync.Exclude, + Exclude: b.Config.Sync.Exclude, - RemotePath: rb.Config().Workspace.FilePath, - Host: rb.WorkspaceClient().Config.Host, + RemotePath: b.Config.Workspace.FilePath, + Host: b.WorkspaceClient().Config.Host, Full: false, SnapshotBasePath: cacheDir, - WorkspaceClient: rb.WorkspaceClient(), + WorkspaceClient: b.WorkspaceClient(), } - if rb.Config().Workspace.CurrentUser != nil { - opts.CurrentUser = rb.Config().Workspace.CurrentUser.User + if b.Config.Workspace.CurrentUser != nil { + opts.CurrentUser = b.Config.Workspace.CurrentUser.User } return opts, nil diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index 452850dc4..bb46c97c9 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -30,7 +30,7 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath)) - opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b)) + opts, err := GetSyncOptions(ctx, b) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 8fffca073..844dcb77e 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } // Create a new snapshot based on the deployment state file. - opts, err := files.GetSyncOptions(ctx, bundle.ReadOnly(b)) + opts, err := files.GetSyncOptions(ctx, b) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index f38b71f6b..b3d838fce 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -92,7 +92,7 @@ func testStatePull(t *testing.T, opts statePullOpts) { } if opts.withExistingSnapshot { - opts, err := files.GetSyncOptions(ctx, bundle.ReadOnly(b)) + opts, err := files.GetSyncOptions(ctx, b) require.NoError(t, err) snapshotPath, err := sync.SnapshotPath(opts) @@ -134,7 +134,7 @@ func testStatePull(t *testing.T, opts statePullOpts) { } if opts.expects.snapshotState != nil { - syncOpts, err := files.GetSyncOptions(ctx, bundle.ReadOnly(b)) + syncOpts, err := files.GetSyncOptions(ctx, b) require.NoError(t, err) snapshotPath, err := sync.SnapshotPath(syncOpts) diff --git a/bundle/deploy/state_push.go b/bundle/deploy/state_push.go index 6912414c1..176a907c8 100644 --- a/bundle/deploy/state_push.go +++ b/bundle/deploy/state_push.go @@ -10,8 +10,6 @@ import ( "github.com/databricks/cli/libs/log" ) -const MaxStateFileSize = 10 * 1024 * 1024 // 10MB - type statePush struct { filerFactory FilerFactory } @@ -37,17 +35,6 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() - if !b.Config.Bundle.Force { - state, err := local.Stat() - if err != nil { - return diag.FromErr(err) - } - - if state.Size() > MaxStateFileSize { - return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize) - } - } - log.Infof(ctx, "Writing local deployment state file to remote state directory") err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index 84d8e7670..6cdde1371 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -47,17 +47,6 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() - if !b.Config.Bundle.Force { - state, err := local.Stat() - if err != nil { - return diag.FromErr(err) - } - - if state.Size() > deploy.MaxStateFileSize { - return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize) - } - } - // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") log.Infof(ctx, "Writing local state file to remote state directory") diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index 54e7f621c..e022dee1b 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -3,7 +3,6 @@ package terraform import ( "context" "encoding/json" - "fmt" "io" "testing" @@ -60,29 +59,3 @@ func TestStatePush(t *testing.T) { diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } - -func TestStatePushLargeState(t *testing.T) { - mock := mockfiler.NewMockFiler(t) - m := &statePush{ - identityFiler(mock), - } - - ctx := context.Background() - b := statePushTestBundle(t) - - largeState := map[string]any{} - for i := range 1000000 { - largeState[fmt.Sprintf("field_%d", i)] = i - } - - // Write a stale local state file. - writeLocalState(t, ctx, b, largeState) - diags := bundle.Apply(ctx, b, m) - assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag") - - // Force the write. - b = statePushTestBundle(t) - b.Config.Bundle.Force = true - diags = bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) -} diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index 26c1a0b06..afcc0ce9f 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -7,6 +7,7 @@ type Descriptor struct { Default any `json:"default,omitempty"` Enum []any `json:"enum,omitempty"` MarkdownExamples string `json:"markdown_examples,omitempty"` + DeprecationMessage string `json:"deprecation_message,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index ee3c25ca1..86ffbce73 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -127,6 +127,12 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { if a.Default != nil { s.Default = a.Default } + + if a.DeprecationMessage != "" { + s.Deprecated = true + s.DeprecationMessage = a.DeprecationMessage + } + s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) s.Title = a.Title s.Enum = a.Enum diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e658f6e53..44e10c13a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -61,6 +61,8 @@ github.com/databricks/cli/bundle/config.Experimental: "pydabs": "description": |- The PyDABs configuration. + "deprecation_message": |- + Deprecated: please use python instead "python": "description": |- Configures loading of Python code defined with 'databricks-bundles' package. @@ -220,6 +222,11 @@ github.com/databricks/cli/bundle/config.Root: The bundle attributes when deploying to this target. "markdown_description": |- The bundle attributes when deploying to this target, + "environments": + "description": |- + PLACEHOLDER + "deprecation_message": |- + Deprecated: please use targets instead "experimental": "description": |- Defines attributes for experimental features. @@ -308,6 +315,8 @@ github.com/databricks/cli/bundle/config.Target: "compute_id": "description": |- Deprecated. The ID of the compute to use for this target. + "deprecation_message": |- + Deprecated: please use cluster_id instead "default": "description": |- Whether this target is the default target. diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 669ecb9ed..d65eea64f 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -5,6 +5,7 @@ github.com/databricks/cli/bundle/config/resources.App: The active deployment of the app. A deployment is considered active when it has been deployed to the app compute. "app_status": {} + "budget_policy_id": {} "compute_status": {} "create_time": "description": |- @@ -19,6 +20,7 @@ github.com/databricks/cli/bundle/config/resources.App: "description": "description": |- The description of the app. + "effective_budget_policy_id": {} "id": "description": |- The unique identifier of the app. @@ -118,7 +120,7 @@ github.com/databricks/cli/bundle/config/resources.Cluster: The optional ID of the instance pool to which the cluster belongs. "is_single_node": "description": | - This field can only be used with `kind`. + This field can only be used when `kind = CLASSIC_PREVIEW`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` "kind": {} @@ -175,7 +177,7 @@ github.com/databricks/cli/bundle/config/resources.Cluster: Up to 10 keys can be specified. "use_ml_runtime": "description": | - This field can only be used with `kind`. + This field can only be used when `kind = CLASSIC_PREVIEW`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. "workload_type": {} @@ -311,6 +313,9 @@ github.com/databricks/cli/bundle/config/resources.Job: "description": |- A collection of system notification IDs to notify when runs of this job begin or complete. github.com/databricks/cli/bundle/config/resources.MlflowExperiment: + "_": + "description": |- + An experiment and its metadata. "artifact_location": "description": |- Location where artifacts for the experiment are stored. @@ -1089,7 +1094,7 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: The optional ID of the instance pool to which the cluster belongs. "is_single_node": "description": | - This field can only be used with `kind`. + This field can only be used when `kind = CLASSIC_PREVIEW`. When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers` "kind": {} @@ -1146,7 +1151,7 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: Up to 10 keys can be specified. "use_ml_runtime": "description": | - This field can only be used with `kind`. + This field can only be used when `kind = CLASSIC_PREVIEW`. `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not. "workload_type": {} @@ -1156,7 +1161,7 @@ github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode: Data security mode decides what data governance model to use when accessing data from a cluster. - The following modes can only be used with `kind`. + The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. @@ -1465,6 +1470,19 @@ github.com/databricks/databricks-sdk-go/service/jobs.CleanRoomsNotebookTask: "notebook_name": "description": |- Name of the notebook being run. +github.com/databricks/databricks-sdk-go/service/jobs.ComputeConfig: + "_": + "description": |- + Next field: 4 + "gpu_node_pool_id": + "description": |- + IDof the GPU pool to use. + "gpu_type": + "description": |- + GPU type. + "num_gpus": + "description": |- + Number of GPUs. github.com/databricks/databricks-sdk-go/service/jobs.Condition: "_": "enum": @@ -1579,6 +1597,37 @@ github.com/databricks/databricks-sdk-go/service/jobs.Format: SINGLE_TASK - |- MULTI_TASK +github.com/databricks/databricks-sdk-go/service/jobs.GenAiComputeTask: + "_": + "description": |- + Next field: 9 + "command": + "description": |- + Command launcher to run the actual script, e.g. bash, python etc. + "compute": {} + "dl_runtime_image": + "description": |- + Runtime image + "mlflow_experiment_name": + "description": |- + Optional string containing the name of the MLflow experiment to log the run to. If name is not + found, backend will create the mlflow experiment using the name. + "source": + "description": |- + Optional location type of the training script. When set to `WORKSPACE`, the script will be retrieved from the local Databricks workspace. When set to `GIT`, the script will be retrieved from a Git repository + defined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise. + * `WORKSPACE`: Script is located in Databricks workspace. + * `GIT`: Script is located in cloud Git provider. + "training_script_path": + "description": |- + The training script file path to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required. + "yaml_parameters": + "description": |- + Optional string containing model parameters passed to the training script in yaml format. + If present, then the content in yaml_parameters_file_path will be ignored. + "yaml_parameters_file_path": + "description": |- + Optional path to a YAML file containing model parameters passed to the training script. github.com/databricks/databricks-sdk-go/service/jobs.GitProvider: "_": "enum": @@ -2144,6 +2193,7 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "for_each_task": "description": |- The task executes a nested task for every input provided when the `for_each_task` field is present. + "gen_ai_compute_task": {} "health": {} "job_cluster_key": "description": |- @@ -2296,6 +2346,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications: "description": |- An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property. github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag: + "_": + "description": |- + A tag for an experiment. "key": "description": |- The tag key. @@ -2864,6 +2917,12 @@ github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig: "description": |- The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon. + "instance_profile_arn": + "description": |- + ARN of the instance profile that the external model will use to access AWS resources. + You must authenticate using an instance profile or access keys. + If you prefer to authenticate using access keys, see `aws_access_key_id`, + `aws_access_key_id_plaintext`, `aws_secret_access_key` and `aws_secret_access_key_plaintext`. github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider: "_": "enum": diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index be83af2d1..f985f8c7d 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -5,6 +5,9 @@ github.com/databricks/cli/bundle/config/resources.App: "app_status": "description": |- PLACEHOLDER + "budget_policy_id": + "description": |- + PLACEHOLDER "compute_status": "description": |- PLACEHOLDER @@ -23,6 +26,9 @@ github.com/databricks/cli/bundle/config/resources.App: "description": "description": |- PLACEHOLDER + "effective_budget_policy_id": + "description": |- + PLACEHOLDER "name": "description": |- PLACEHOLDER @@ -506,6 +512,10 @@ github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes: "availability": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/jobs.GenAiComputeTask: + "compute": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/jobs.GitSource: "git_snapshot": "description": |- @@ -530,6 +540,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: "description": |- PLACEHOLDER github.com/databricks/databricks-sdk-go/service/jobs.Task: + "gen_ai_compute_task": + "description": |- + PLACEHOLDER "health": "description": |- PLACEHOLDER diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 46548f3e8..50a52d19e 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.65.1" +const ProviderVersion = "1.68.0" diff --git a/bundle/internal/tf/schema/data_source_app.go b/bundle/internal/tf/schema/data_source_app.go index 9b4ef077e..3c5329e3e 100644 --- a/bundle/internal/tf/schema/data_source_app.go +++ b/bundle/internal/tf/schema/data_source_app.go @@ -90,6 +90,7 @@ type DataSourceAppApp struct { Creator string `json:"creator,omitempty"` DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` Description string `json:"description,omitempty"` + Id string `json:"id,omitempty"` Name string `json:"name"` PendingDeployment *DataSourceAppAppPendingDeployment `json:"pending_deployment,omitempty"` Resources []DataSourceAppAppResources `json:"resources,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_apps.go b/bundle/internal/tf/schema/data_source_apps.go index dd381eabf..6799468f8 100644 --- a/bundle/internal/tf/schema/data_source_apps.go +++ b/bundle/internal/tf/schema/data_source_apps.go @@ -90,6 +90,7 @@ type DataSourceAppsApp struct { Creator string `json:"creator,omitempty"` DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` Description string `json:"description,omitempty"` + Id string `json:"id,omitempty"` Name string `json:"name"` PendingDeployment *DataSourceAppsAppPendingDeployment `json:"pending_deployment,omitempty"` Resources []DataSourceAppsAppResources `json:"resources,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index 38cb534f2..5f139854f 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -46,9 +46,14 @@ type DataSourceClusterClusterInfoClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type DataSourceClusterClusterInfoClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type DataSourceClusterClusterInfoClusterLogConf struct { - Dbfs *DataSourceClusterClusterInfoClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *DataSourceClusterClusterInfoClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *DataSourceClusterClusterInfoClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoClusterLogConfS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoClusterLogConfVolumes `json:"volumes,omitempty"` } type DataSourceClusterClusterInfoClusterLogStatus struct { @@ -191,9 +196,14 @@ type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type DataSourceClusterClusterInfoSpecClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type DataSourceClusterClusterInfoSpecClusterLogConf struct { - Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoSpecClusterLogConfVolumes `json:"volumes,omitempty"` } type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { diff --git a/bundle/internal/tf/schema/data_source_dashboards.go b/bundle/internal/tf/schema/data_source_dashboards.go new file mode 100644 index 000000000..9a047388b --- /dev/null +++ b/bundle/internal/tf/schema/data_source_dashboards.go @@ -0,0 +1,21 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceDashboardsDashboards struct { + CreateTime string `json:"create_time,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Etag string `json:"etag,omitempty"` + LifecycleState string `json:"lifecycle_state,omitempty"` + ParentPath string `json:"parent_path,omitempty"` + Path string `json:"path,omitempty"` + SerializedDashboard string `json:"serialized_dashboard,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + WarehouseId string `json:"warehouse_id,omitempty"` +} + +type DataSourceDashboards struct { + DashboardNameContains string `json:"dashboard_name_contains,omitempty"` + Dashboards []DataSourceDashboardsDashboards `json:"dashboards,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 1880db25a..a1b17d88d 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -18,6 +18,7 @@ type DataSources struct { CurrentConfig map[string]any `json:"databricks_current_config,omitempty"` CurrentMetastore map[string]any `json:"databricks_current_metastore,omitempty"` CurrentUser map[string]any `json:"databricks_current_user,omitempty"` + Dashboards map[string]any `json:"databricks_dashboards,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DbfsFilePaths map[string]any `json:"databricks_dbfs_file_paths,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -83,6 +84,7 @@ func NewDataSources() *DataSources { CurrentConfig: make(map[string]any), CurrentMetastore: make(map[string]any), CurrentUser: make(map[string]any), + Dashboards: make(map[string]any), DbfsFile: make(map[string]any), DbfsFilePaths: make(map[string]any), Directory: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_app.go b/bundle/internal/tf/schema/resource_app.go index cbce5ab0e..7a0e35c17 100644 --- a/bundle/internal/tf/schema/resource_app.go +++ b/bundle/internal/tf/schema/resource_app.go @@ -90,6 +90,7 @@ type ResourceApp struct { Creator string `json:"creator,omitempty"` DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` Description string `json:"description,omitempty"` + Id string `json:"id,omitempty"` Name string `json:"name"` NoCompute bool `json:"no_compute,omitempty"` PendingDeployment *ResourceAppPendingDeployment `json:"pending_deployment,omitempty"` diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go index 50395add9..c553a0b93 100644 --- a/bundle/internal/tf/schema/resource_cluster.go +++ b/bundle/internal/tf/schema/resource_cluster.go @@ -46,9 +46,14 @@ type ResourceClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourceClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourceClusterClusterLogConf struct { - Dbfs *ResourceClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourceClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourceClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourceClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourceClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourceClusterClusterMountInfoNetworkFilesystemInfo struct { diff --git a/bundle/internal/tf/schema/resource_connection.go b/bundle/internal/tf/schema/resource_connection.go index a249a5393..0fc7c05a7 100644 --- a/bundle/internal/tf/schema/resource_connection.go +++ b/bundle/internal/tf/schema/resource_connection.go @@ -3,13 +3,23 @@ package schema type ResourceConnection struct { - Comment string `json:"comment,omitempty"` - ConnectionType string `json:"connection_type"` - Id string `json:"id,omitempty"` - MetastoreId string `json:"metastore_id,omitempty"` - Name string `json:"name"` - Options map[string]string `json:"options"` - Owner string `json:"owner,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - ReadOnly bool `json:"read_only,omitempty"` + Comment string `json:"comment,omitempty"` + ConnectionId string `json:"connection_id,omitempty"` + ConnectionType string `json:"connection_type,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + CredentialType string `json:"credential_type,omitempty"` + FullName string `json:"full_name,omitempty"` + Id string `json:"id,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Options map[string]string `json:"options,omitempty"` + Owner string `json:"owner,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + ProvisioningInfo []any `json:"provisioning_info,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` + SecurableType string `json:"securable_type,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Url string `json:"url,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 2c27f0be7..c07ac8a6b 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -114,9 +114,14 @@ type ResourceJobJobClusterNewClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourceJobJobClusterNewClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourceJobJobClusterNewClusterClusterLogConf struct { - Dbfs *ResourceJobJobClusterNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourceJobJobClusterNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourceJobJobClusterNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourceJobJobClusterNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourceJobJobClusterNewClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourceJobJobClusterNewClusterClusterMountInfoNetworkFilesystemInfo struct { @@ -339,9 +344,14 @@ type ResourceJobNewClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourceJobNewClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourceJobNewClusterClusterLogConf struct { - Dbfs *ResourceJobNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourceJobNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourceJobNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourceJobNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourceJobNewClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourceJobNewClusterClusterMountInfoNetworkFilesystemInfo struct { @@ -708,9 +718,14 @@ type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConf struct { - Dbfs *ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourceJobTaskForEachTaskTaskNewClusterClusterMountInfoNetworkFilesystemInfo struct { @@ -1104,9 +1119,14 @@ type ResourceJobTaskNewClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourceJobTaskNewClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourceJobTaskNewClusterClusterLogConf struct { - Dbfs *ResourceJobTaskNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourceJobTaskNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourceJobTaskNewClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourceJobTaskNewClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourceJobTaskNewClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourceJobTaskNewClusterClusterMountInfoNetworkFilesystemInfo struct { diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 8e260e65c..c38209ece 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -47,9 +47,14 @@ type ResourcePipelineClusterClusterLogConfS3 struct { Region string `json:"region,omitempty"` } +type ResourcePipelineClusterClusterLogConfVolumes struct { + Destination string `json:"destination"` +} + type ResourcePipelineClusterClusterLogConf struct { - Dbfs *ResourcePipelineClusterClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *ResourcePipelineClusterClusterLogConfS3 `json:"s3,omitempty"` + Dbfs *ResourcePipelineClusterClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *ResourcePipelineClusterClusterLogConfS3 `json:"s3,omitempty"` + Volumes *ResourcePipelineClusterClusterLogConfVolumes `json:"volumes,omitempty"` } type ResourcePipelineClusterGcpAttributes struct { diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 816e8e6aa..54c19e7ad 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.65.1" +const ProviderVersion = "1.68.0" func NewRoot() *Root { return &Root{ diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index ab869d3d2..907843193 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -13,8 +13,10 @@ import ( type checkForSameNameLibraries struct{} var patterns = []dyn.Pattern{ - taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), - forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()), + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), } diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go index 700a90d8d..87857022d 100644 --- a/bundle/mutator_read_only.go +++ b/bundle/mutator_read_only.go @@ -2,28 +2,59 @@ package bundle import ( "context" + "sync" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" ) -// ReadOnlyMutator is the interface type that allows access to bundle configuration but does not allow any mutations. type ReadOnlyMutator interface { - // Name returns the mutators name. - Name() string + Mutator - // Apply access the specified read-only bundle object. - Apply(context.Context, ReadOnlyBundle) diag.Diagnostics + // This is just tag, to differentiate this interface from bundle.Mutator + // This prevents non-readonly mutators being passed to ApplyParallel(). + IsRO() } -func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) diag.Diagnostics { - ctx = log.NewContext(ctx, log.GetLogger(ctx).With("mutator (read-only)", m.Name())) +// Helper to mark the mutator as "read-only" +type RO struct{} - log.Debugf(ctx, "ApplyReadOnly") - diags := m.Apply(ctx, rb) - if err := diags.Error(); err != nil { - log.Debugf(ctx, "Error: %s", err) +func (*RO) IsRO() {} + +// Run mutators in parallel. Unlike Apply and ApplySeq, this does not perform sync between +// dynamic and static configuration. +// Warning: none of the mutators involved must modify bundle directly or indirectly. In particular, +// they must not call bundle.Apply or bundle.ApplySeq because those include writes to config even if mutator does not. +// Deprecated: do not use for new use cases. Refactor your parallel task not to depend on bundle at all. +func ApplyParallel(ctx context.Context, b *Bundle, mutators ...ReadOnlyMutator) diag.Diagnostics { + var allDiags diag.Diagnostics + resultsChan := make(chan diag.Diagnostics, len(mutators)) + var wg sync.WaitGroup + + contexts := make([]context.Context, len(mutators)) + + for ind, m := range mutators { + contexts[ind] = log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) + // log right away to have deterministic order of log messages + log.Debug(contexts[ind], "ApplyParallel") } - return diags + for ind, m := range mutators { + wg.Add(1) + go func() { + defer wg.Done() + // We're not using bundle.Apply here because we don't do copy between typed and dynamic values + resultsChan <- m.Apply(contexts[ind], b) + }() + } + + wg.Wait() + close(resultsChan) + + // Collect results into a single slice + for diags := range resultsChan { + allDiags = append(allDiags, diags...) + } + + return allDiags } diff --git a/bundle/parallel.go b/bundle/parallel.go deleted file mode 100644 index ebb91661a..000000000 --- a/bundle/parallel.go +++ /dev/null @@ -1,43 +0,0 @@ -package bundle - -import ( - "context" - "sync" - - "github.com/databricks/cli/libs/diag" -) - -type parallel struct { - mutators []ReadOnlyMutator -} - -func (m *parallel) Name() string { - return "parallel" -} - -func (m *parallel) Apply(ctx context.Context, rb ReadOnlyBundle) diag.Diagnostics { - var wg sync.WaitGroup - var mu sync.Mutex - var diags diag.Diagnostics - - wg.Add(len(m.mutators)) - for _, mutator := range m.mutators { - go func(mutator ReadOnlyMutator) { - defer wg.Done() - d := ApplyReadOnly(ctx, rb, mutator) - - mu.Lock() - diags = diags.Extend(d) - mu.Unlock() - }(mutator) - } - wg.Wait() - return diags -} - -// Parallel runs the given mutators in parallel. -func Parallel(mutators ...ReadOnlyMutator) ReadOnlyMutator { - return ¶llel{ - mutators: mutators, - } -} diff --git a/bundle/parallel_test.go b/bundle/parallel_test.go deleted file mode 100644 index dfc7ddac9..000000000 --- a/bundle/parallel_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package bundle - -import ( - "context" - "sync" - "testing" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" - "github.com/stretchr/testify/require" -) - -type addToContainer struct { - t *testing.T - container *[]int - value int - err bool - - // mu is a mutex that protects container. It is used to ensure that the - // container slice is only modified by one goroutine at a time. - mu *sync.Mutex -} - -func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagnostics { - if m.err { - return diag.Errorf("error") - } - - m.mu.Lock() - *m.container = append(*m.container, m.value) - m.mu.Unlock() - - return nil -} - -func (m *addToContainer) Name() string { - return "addToContainer" -} - -func TestParallelMutatorWork(t *testing.T) { - b := &Bundle{ - Config: config.Root{}, - } - - container := []int{} - var mu sync.Mutex - m1 := &addToContainer{t: t, container: &container, value: 1, mu: &mu} - m2 := &addToContainer{t: t, container: &container, value: 2, mu: &mu} - m3 := &addToContainer{t: t, container: &container, value: 3, mu: &mu} - - m := Parallel(m1, m2, m3) - - // Apply the mutator - diags := ApplyReadOnly(context.Background(), ReadOnly(b), m) - require.Empty(t, diags) - require.Len(t, container, 3) - require.Contains(t, container, 1) - require.Contains(t, container, 2) - require.Contains(t, container, 3) -} - -func TestParallelMutatorWorkWithErrors(t *testing.T) { - b := &Bundle{ - Config: config.Root{}, - } - - container := []int{} - var mu sync.Mutex - m1 := &addToContainer{container: &container, value: 1, mu: &mu} - m2 := &addToContainer{container: &container, err: true, value: 2, mu: &mu} - m3 := &addToContainer{container: &container, value: 3, mu: &mu} - - m := Parallel(m1, m2, m3) - - // Apply the mutator - diags := ApplyReadOnly(context.Background(), ReadOnly(b), m) - require.Len(t, diags, 1) - require.Equal(t, "error", diags[0].Summary) - require.Len(t, container, 2) - require.Contains(t, container, 1) - require.Contains(t, container, 3) -} diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4bfbd62fc..5731f72fd 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -70,6 +70,9 @@ "app_status": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus" }, + "budget_policy_id": { + "$ref": "#/$defs/string" + }, "compute_status": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus" }, @@ -88,6 +91,9 @@ "description": { "$ref": "#/$defs/string" }, + "effective_budget_policy_id": { + "$ref": "#/$defs/string" + }, "id": { "description": "The unique identifier of the app.", "$ref": "#/$defs/string" @@ -210,7 +216,7 @@ "$ref": "#/$defs/string" }, "is_single_node": { - "description": "This field can only be used with `kind`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`\n", + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`\n", "$ref": "#/$defs/bool" }, "kind": { @@ -255,7 +261,7 @@ "$ref": "#/$defs/slice/string" }, "use_ml_runtime": { - "description": "This field can only be used with `kind`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.\n", + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.\n", "$ref": "#/$defs/bool" }, "workload_type": { @@ -465,6 +471,7 @@ "oneOf": [ { "type": "object", + "description": "An experiment and its metadata.", "properties": { "artifact_location": { "description": "Location where artifacts for the experiment are stored.", @@ -1210,7 +1217,9 @@ "properties": { "pydabs": { "description": "The PyDABs configuration.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs", + "deprecationMessage": "Deprecated: please use python instead", + "deprecated": true }, "python": { "description": "Configures loading of Python code defined with 'databricks-bundles' package.", @@ -1499,7 +1508,9 @@ }, "compute_id": { "description": "Deprecated. The ID of the compute to use for this target.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "deprecationMessage": "Deprecated: please use cluster_id instead", + "deprecated": true }, "default": { "description": "Whether this target is the default target.", @@ -2585,7 +2596,7 @@ "$ref": "#/$defs/string" }, "is_single_node": { - "description": "This field can only be used with `kind`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`\n", + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`\n", "$ref": "#/$defs/bool" }, "kind": { @@ -2627,7 +2638,7 @@ "$ref": "#/$defs/slice/string" }, "use_ml_runtime": { - "description": "This field can only be used with `kind`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.\n", + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.\n", "$ref": "#/$defs/bool" }, "workload_type": { @@ -2646,7 +2657,7 @@ "oneOf": [ { "type": "string", - "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used with `kind`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used when `kind = CLASSIC_PREVIEW`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", "enum": [ "DATA_SECURITY_MODE_AUTO", "DATA_SECURITY_MODE_STANDARD", @@ -3230,6 +3241,37 @@ } ] }, + "jobs.ComputeConfig": { + "oneOf": [ + { + "type": "object", + "description": "Next field: 4", + "properties": { + "gpu_node_pool_id": { + "description": "IDof the GPU pool to use.", + "$ref": "#/$defs/string" + }, + "gpu_type": { + "description": "GPU type.", + "$ref": "#/$defs/string" + }, + "num_gpus": { + "description": "Number of GPUs.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "gpu_node_pool_id", + "num_gpus" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.Condition": { "oneOf": [ { @@ -3463,6 +3505,55 @@ } ] }, + "jobs.GenAiComputeTask": { + "oneOf": [ + { + "type": "object", + "description": "Next field: 9", + "properties": { + "command": { + "description": "Command launcher to run the actual script, e.g. bash, python etc.", + "$ref": "#/$defs/string" + }, + "compute": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ComputeConfig" + }, + "dl_runtime_image": { + "description": "Runtime image", + "$ref": "#/$defs/string" + }, + "mlflow_experiment_name": { + "description": "Optional string containing the name of the MLflow experiment to log the run to. If name is not\nfound, backend will create the mlflow experiment using the name.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the training script. When set to `WORKSPACE`, the script will be retrieved from the local Databricks workspace. When set to `GIT`, the script will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Script is located in Databricks workspace.\n* `GIT`: Script is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "training_script_path": { + "description": "The training script file path to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "yaml_parameters": { + "description": "Optional string containing model parameters passed to the training script in yaml format.\nIf present, then the content in yaml_parameters_file_path will be ignored.", + "$ref": "#/$defs/string" + }, + "yaml_parameters_file_path": { + "description": "Optional path to a YAML file containing model parameters passed to the training script.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "dl_runtime_image" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.GitProvider": { "oneOf": [ { @@ -4504,6 +4595,9 @@ "description": "The task executes a nested task for every input provided when the `for_each_task` field is present.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ForEachTask" }, + "gen_ai_compute_task": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GenAiComputeTask" + }, "health": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" }, @@ -4775,6 +4869,7 @@ "oneOf": [ { "type": "object", + "description": "A tag for an experiment.", "properties": { "key": { "description": "The tag key.", @@ -5850,6 +5945,10 @@ "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the external model will use to access AWS resources.\nYou must authenticate using an instance profile or access keys.\nIf you prefer to authenticate using access keys, see `aws_access_key_id`,\n`aws_access_key_id_plaintext`, `aws_secret_access_key` and `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" } }, "additionalProperties": false, @@ -7314,6 +7413,11 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle", "markdownDescription": "The bundle attributes when deploying to this target," }, + "environments": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target", + "deprecationMessage": "Deprecated: please use targets instead", + "deprecated": true + }, "experimental": { "description": "Defines attributes for experimental features.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Experimental" diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index 135ef1917..c22059313 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -9,11 +9,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestEnvironmentKeySupported(t *testing.T) { - _, diags := loadTargetWithDiags("./python_wheel/environment_key", "default") - require.Empty(t, diags) -} - func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) diff --git a/bundle/tests/job_cluster_key_test.go b/bundle/tests/job_cluster_key_test.go index 6a08da89c..0306bf7b5 100644 --- a/bundle/tests/job_cluster_key_test.go +++ b/bundle/tests/job_cluster_key_test.go @@ -13,7 +13,7 @@ import ( func TestJobClusterKeyNotDefinedTest(t *testing.T) { b := loadTarget(t, "./job_cluster_key", "default") - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) + diags := bundle.Apply(context.Background(), b, validate.JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) require.Equal(t, diag.Warning, diags[0].Severity) @@ -23,6 +23,6 @@ func TestJobClusterKeyNotDefinedTest(t *testing.T) { func TestJobClusterKeyDefinedTest(t *testing.T) { b := loadTarget(t, "./job_cluster_key", "development") - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.JobClusterKeyDefined()) + diags := bundle.Apply(context.Background(), b, validate.JobClusterKeyDefined()) require.Empty(t, diags) } diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go deleted file mode 100644 index 22702ec44..000000000 --- a/bundle/tests/python_wheel_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package config_tests - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/phases" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/libs/filer" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestPythonWheelBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildAutoDetect(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelWithDBFSLib(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), - mock.AnythingOfType("*os.File"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - diags = bundle.ApplySeq(ctx, b, - libraries.ExpandGlobReferences(), - libraries.UploadWithClient(mockFiler), - ) - require.NoError(t, diags.Error()) - require.Empty(t, diags) - require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) -} - -func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { - b := loadTarget(t, "./python_wheel/environment_key", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildMultiple(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") - require.NoError(t, err) - require.Len(t, matches, 2) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelNoBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index c206e7471..ed929518a 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -16,7 +16,7 @@ import ( func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { b := loadTarget(t, "./sync/override", "development") - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + diags := bundle.Apply(context.Background(), b, validate.ValidateSyncPatterns()) require.Len(t, diags, 3) require.NoError(t, diags.Error()) @@ -46,7 +46,7 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { func TestSyncIncludeWithNegate(t *testing.T) { b := loadTarget(t, "./sync/negate", "default") - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + diags := bundle.Apply(context.Background(), b, validate.ValidateSyncPatterns()) require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -54,7 +54,7 @@ func TestSyncIncludeWithNegate(t *testing.T) { func TestSyncIncludeWithNegateNoMatches(t *testing.T) { b := loadTarget(t, "./sync/negate", "dev") - diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + diags := bundle.Apply(context.Background(), b, validate.ValidateSyncPatterns()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) diff --git a/cmd/account/budget-policy/budget-policy.go b/cmd/account/budget-policy/budget-policy.go index fb9f8e5a6..8e338f91f 100755 --- a/cmd/account/budget-policy/budget-policy.go +++ b/cmd/account/budget-policy/budget-policy.go @@ -58,8 +58,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - // TODO: array: custom_tags - cmd.Flags().StringVar(&createReq.PolicyName, "policy-name", createReq.PolicyName, `The name of the policy.`) + // TODO: complex arg: policy cmd.Flags().StringVar(&createReq.RequestId, "request-id", createReq.RequestId, `A unique identifier for this request.`) cmd.Use = "create" diff --git a/cmd/account/service-principal-secrets/service-principal-secrets.go b/cmd/account/service-principal-secrets/service-principal-secrets.go index f7dc4e88e..ea6658c17 100755 --- a/cmd/account/service-principal-secrets/service-principal-secrets.go +++ b/cmd/account/service-principal-secrets/service-principal-secrets.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/service/oauth2" "github.com/spf13/cobra" ) @@ -64,8 +65,12 @@ func newCreate() *cobra.Command { cmd := &cobra.Command{} var createReq oauth2.CreateServicePrincipalSecretRequest + var createJson flags.JsonFlag // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createReq.Lifetime, "lifetime", createReq.Lifetime, `The lifetime of the secret in seconds.`) cmd.Use = "create SERVICE_PRINCIPAL_ID" cmd.Short = `Create service principal secret.` @@ -88,6 +93,18 @@ func newCreate() *cobra.Command { ctx := cmd.Context() a := root.AccountClient(ctx) + if cmd.Flags().Changed("json") { + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } _, err = fmt.Sscan(args[0], &createReq.ServicePrincipalId) if err != nil { return fmt.Errorf("invalid SERVICE_PRINCIPAL_ID: %s", args[0]) diff --git a/cmd/bundle/sync.go b/cmd/bundle/sync.go index 3ada07b74..25475206d 100644 --- a/cmd/bundle/sync.go +++ b/cmd/bundle/sync.go @@ -25,7 +25,7 @@ type syncFlags struct { } func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) { - opts, err := files.GetSyncOptions(cmd.Context(), bundle.ReadOnly(b)) + opts, err := files.GetSyncOptions(cmd.Context(), b) if err != nil { return nil, fmt.Errorf("cannot get sync options: %w", err) } diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 0ff9c7867..0a902806f 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -50,7 +50,7 @@ func newValidateCommand() *cobra.Command { } if !diags.HasError() { - diags = diags.Extend(bundle.Apply(ctx, b, validate.Validate())) + diags = diags.Extend(validate.Validate(ctx, b)) } switch root.OutputType(cmd) { diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index dea40f96a..0f4a4aacc 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -33,7 +33,7 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b * return nil, errors.New("SRC and DST are not configurable in the context of a bundle") } - opts, err := files.GetSyncOptions(cmd.Context(), bundle.ReadOnly(b)) + opts, err := files.GetSyncOptions(cmd.Context(), b) if err != nil { return nil, fmt.Errorf("cannot get sync options: %w", err) } diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index 6eb85d873..aa88fe072 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -81,6 +81,7 @@ func newCreate() *cobra.Command { cmd.Flags().BoolVar(&createReq.NoCompute, "no-compute", createReq.NoCompute, `If true, the app will not be started after creation.`) // TODO: complex arg: active_deployment // TODO: complex arg: app_status + cmd.Flags().StringVar(&createReq.App.BudgetPolicyId, "budget-policy-id", createReq.App.BudgetPolicyId, ``) // TODO: complex arg: compute_status cmd.Flags().StringVar(&createReq.App.Description, "description", createReq.App.Description, `The description of the app.`) // TODO: complex arg: pending_deployment @@ -938,6 +939,7 @@ func newUpdate() *cobra.Command { // TODO: complex arg: active_deployment // TODO: complex arg: app_status + cmd.Flags().StringVar(&updateReq.App.BudgetPolicyId, "budget-policy-id", updateReq.App.BudgetPolicyId, ``) // TODO: complex arg: compute_status cmd.Flags().StringVar(&updateReq.App.Description, "description", updateReq.App.Description, `The description of the app.`) // TODO: complex arg: pending_deployment diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index bbb7c578a..343cdf074 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -223,7 +223,7 @@ func newCreate() *cobra.Command { // TODO: complex arg: gcp_attributes // TODO: array: init_scripts cmd.Flags().StringVar(&createReq.InstancePoolId, "instance-pool-id", createReq.InstancePoolId, `The optional ID of the instance pool to which the cluster belongs.`) - cmd.Flags().BoolVar(&createReq.IsSingleNode, "is-single-node", createReq.IsSingleNode, `This field can only be used with kind.`) + cmd.Flags().BoolVar(&createReq.IsSingleNode, "is-single-node", createReq.IsSingleNode, `This field can only be used when kind = CLASSIC_PREVIEW.`) cmd.Flags().Var(&createReq.Kind, "kind", `The kind of compute described by this compute specification. Supported values: [CLASSIC_PREVIEW]`) cmd.Flags().StringVar(&createReq.NodeTypeId, "node-type-id", createReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&createReq.NumWorkers, "num-workers", createReq.NumWorkers, `Number of worker nodes that this cluster should have.`) @@ -233,7 +233,7 @@ func newCreate() *cobra.Command { // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars // TODO: array: ssh_public_keys - cmd.Flags().BoolVar(&createReq.UseMlRuntime, "use-ml-runtime", createReq.UseMlRuntime, `This field can only be used with kind.`) + cmd.Flags().BoolVar(&createReq.UseMlRuntime, "use-ml-runtime", createReq.UseMlRuntime, `This field can only be used when kind = CLASSIC_PREVIEW.`) // TODO: complex arg: workload_type cmd.Use = "create SPARK_VERSION" @@ -493,7 +493,7 @@ func newEdit() *cobra.Command { // TODO: complex arg: gcp_attributes // TODO: array: init_scripts cmd.Flags().StringVar(&editReq.InstancePoolId, "instance-pool-id", editReq.InstancePoolId, `The optional ID of the instance pool to which the cluster belongs.`) - cmd.Flags().BoolVar(&editReq.IsSingleNode, "is-single-node", editReq.IsSingleNode, `This field can only be used with kind.`) + cmd.Flags().BoolVar(&editReq.IsSingleNode, "is-single-node", editReq.IsSingleNode, `This field can only be used when kind = CLASSIC_PREVIEW.`) cmd.Flags().Var(&editReq.Kind, "kind", `The kind of compute described by this compute specification. Supported values: [CLASSIC_PREVIEW]`) cmd.Flags().StringVar(&editReq.NodeTypeId, "node-type-id", editReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&editReq.NumWorkers, "num-workers", editReq.NumWorkers, `Number of worker nodes that this cluster should have.`) @@ -503,7 +503,7 @@ func newEdit() *cobra.Command { // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars // TODO: array: ssh_public_keys - cmd.Flags().BoolVar(&editReq.UseMlRuntime, "use-ml-runtime", editReq.UseMlRuntime, `This field can only be used with kind.`) + cmd.Flags().BoolVar(&editReq.UseMlRuntime, "use-ml-runtime", editReq.UseMlRuntime, `This field can only be used when kind = CLASSIC_PREVIEW.`) // TODO: complex arg: workload_type cmd.Use = "edit CLUSTER_ID SPARK_VERSION" diff --git a/cmd/workspace/credentials/credentials.go b/cmd/workspace/credentials/credentials.go index 672a3aeec..c54737e9a 100755 --- a/cmd/workspace/credentials/credentials.go +++ b/cmd/workspace/credentials/credentials.go @@ -522,6 +522,7 @@ func newValidateCredential() *cobra.Command { // TODO: complex arg: aws_iam_role // TODO: complex arg: azure_managed_identity cmd.Flags().StringVar(&validateCredentialReq.CredentialName, "credential-name", validateCredentialReq.CredentialName, `Required.`) + // TODO: complex arg: databricks_gcp_service_account cmd.Flags().StringVar(&validateCredentialReq.ExternalLocationName, "external-location-name", validateCredentialReq.ExternalLocationName, `The name of an existing external location to validate.`) cmd.Flags().Var(&validateCredentialReq.Purpose, "purpose", `The purpose of the credential. Supported values: [SERVICE, STORAGE]`) cmd.Flags().BoolVar(&validateCredentialReq.ReadOnly, "read-only", validateCredentialReq.ReadOnly, `Whether the credential is only usable for read operations.`) diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index b5173aebf..05a301897 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -105,7 +105,7 @@ func newCreateExperiment() *cobra.Command { already exist and fails if another experiment with the same name already exists. - Throws RESOURCE_ALREADY_EXISTS if a experiment with the given name exists. + Throws RESOURCE_ALREADY_EXISTS if an experiment with the given name exists. Arguments: NAME: Experiment name.` @@ -183,6 +183,7 @@ func newCreateRun() *cobra.Command { cmd.Flags().Var(&createRunJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createRunReq.ExperimentId, "experiment-id", createRunReq.ExperimentId, `ID of the associated experiment.`) + cmd.Flags().StringVar(&createRunReq.RunName, "run-name", createRunReq.RunName, `The name of the run.`) cmd.Flags().Int64Var(&createRunReq.StartTime, "start-time", createRunReq.StartTime, `Unix timestamp in milliseconds of when the run started.`) // TODO: array: tags cmd.Flags().StringVar(&createRunReq.UserId, "user-id", createRunReq.UserId, `ID of the user executing the run.`) @@ -193,7 +194,7 @@ func newCreateRun() *cobra.Command { Creates a new run within an experiment. A run is usually a single execution of a machine learning or data ETL pipeline. MLflow uses runs to track the - mlflowParam, mlflowMetric and mlflowRunTag associated with a single + mlflowParam, mlflowMetric, and mlflowRunTag associated with a single execution.` cmd.Annotations = make(map[string]string) @@ -263,7 +264,7 @@ func newDeleteExperiment() *cobra.Command { cmd.Long = `Delete an experiment. Marks an experiment and associated metadata, runs, metrics, params, and tags - for deletion. If the experiment uses FileStore, artifacts associated with + for deletion. If the experiment uses FileStore, artifacts associated with the experiment are also deleted. Arguments: @@ -431,7 +432,6 @@ func newDeleteRuns() *cobra.Command { Bulk delete runs in an experiment that were created prior to or at the specified timestamp. Deletes at most max_runs per request. To call this API from a Databricks Notebook in Python, you can use the client code snippet on - https://learn.microsoft.com/en-us/azure/databricks/mlflow/runs#bulk-delete. Arguments: EXPERIMENT_ID: The ID of the experiment containing the runs to delete. @@ -518,8 +518,8 @@ func newDeleteTag() *cobra.Command { cmd.Flags().Var(&deleteTagJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "delete-tag RUN_ID KEY" - cmd.Short = `Delete a tag.` - cmd.Long = `Delete a tag. + cmd.Short = `Delete a tag on a run.` + cmd.Long = `Delete a tag on a run. Deletes a tag on a run. Tags are run metadata that can be updated during a run and after a run completes. @@ -602,8 +602,8 @@ func newGetByName() *cobra.Command { // TODO: short flags cmd.Use = "get-by-name EXPERIMENT_NAME" - cmd.Short = `Get metadata.` - cmd.Long = `Get metadata. + cmd.Short = `Get an experiment by name.` + cmd.Long = `Get an experiment by name. Gets metadata for an experiment. @@ -731,8 +731,8 @@ func newGetHistory() *cobra.Command { cmd.Flags().StringVar(&getHistoryReq.RunUuid, "run-uuid", getHistoryReq.RunUuid, `[Deprecated, use run_id instead] ID of the run from which to fetch metric values.`) cmd.Use = "get-history METRIC_KEY" - cmd.Short = `Get history of a given metric within a run.` - cmd.Long = `Get history of a given metric within a run. + cmd.Short = `Get metric history for a run.` + cmd.Long = `Get metric history for a run. Gets a list of all values for the specified metric for a given run. @@ -973,12 +973,11 @@ func newListArtifacts() *cobra.Command { cmd.Flags().StringVar(&listArtifactsReq.RunUuid, "run-uuid", listArtifactsReq.RunUuid, `[Deprecated, use run_id instead] ID of the run whose artifacts to list.`) cmd.Use = "list-artifacts" - cmd.Short = `Get all artifacts.` - cmd.Long = `Get all artifacts. + cmd.Short = `List artifacts.` + cmd.Long = `List artifacts. - List artifacts for a run. Takes an optional artifact_path prefix. If it is - specified, the response contains only artifacts with the specified prefix. - This API does not support pagination when listing artifacts in UC Volumes. A + List artifacts for a run. Takes an optional artifact_path prefix which if + specified, the response contains only artifacts with the specified prefix. A maximum of 1000 artifacts will be retrieved for UC Volumes. Please call /api/2.0/fs/directories{directory_path} for listing artifacts in UC Volumes, which supports pagination. See [List directory contents | Files @@ -1028,9 +1027,9 @@ func newListExperiments() *cobra.Command { // TODO: short flags - cmd.Flags().IntVar(&listExperimentsReq.MaxResults, "max-results", listExperimentsReq.MaxResults, `Maximum number of experiments desired.`) + cmd.Flags().Int64Var(&listExperimentsReq.MaxResults, "max-results", listExperimentsReq.MaxResults, `Maximum number of experiments desired.`) cmd.Flags().StringVar(&listExperimentsReq.PageToken, "page-token", listExperimentsReq.PageToken, `Token indicating the page of experiments to fetch.`) - cmd.Flags().StringVar(&listExperimentsReq.ViewType, "view-type", listExperimentsReq.ViewType, `Qualifier for type of experiments to be returned.`) + cmd.Flags().Var(&listExperimentsReq.ViewType, "view-type", `Qualifier for type of experiments to be returned. Supported values: [ACTIVE_ONLY, ALL, DELETED_ONLY]`) cmd.Use = "list-experiments" cmd.Short = `List experiments.` @@ -1090,8 +1089,8 @@ func newLogBatch() *cobra.Command { // TODO: array: tags cmd.Use = "log-batch" - cmd.Short = `Log a batch.` - cmd.Long = `Log a batch. + cmd.Short = `Log a batch of metrics/params/tags for a run.` + cmd.Long = `Log a batch of metrics/params/tags for a run. Logs a batch of metrics, params, and tags for a run. If any data failed to be persisted, the server will respond with an error (non-200 status code). @@ -1120,8 +1119,13 @@ func newLogBatch() *cobra.Command { Request Limits ------------------------------- A single JSON-serialized API request may be up to 1 MB in size and contain: - * No more than 1000 metrics, params, and tags in total * Up to 1000 metrics * - Up to 100 params * Up to 100 tags + * No more than 1000 metrics, params, and tags in total + + * Up to 1000 metrics + + * Up to 100 params + + * Up to 100 tags For example, a valid request might contain 900 metrics, 50 params, and 50 tags, but logging 900 metrics, 50 params, and 51 tags is invalid. @@ -1129,6 +1133,7 @@ func newLogBatch() *cobra.Command { The following limits also apply to metric, param, and tag keys and values: * Metric keys, param keys, and tag keys can be up to 250 characters in length + * Parameter and tag values can be up to 250 characters in length` cmd.Annotations = make(map[string]string) @@ -1194,19 +1199,30 @@ func newLogInputs() *cobra.Command { cmd.Flags().Var(&logInputsJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: datasets - cmd.Flags().StringVar(&logInputsReq.RunId, "run-id", logInputsReq.RunId, `ID of the run to log under.`) - cmd.Use = "log-inputs" + cmd.Use = "log-inputs RUN_ID" cmd.Short = `Log inputs to a run.` cmd.Long = `Log inputs to a run. **NOTE:** Experimental: This API may change or be removed in a future release - without warning.` + without warning. + + Logs inputs, such as datasets and models, to an MLflow Run. + + Arguments: + RUN_ID: ID of the run to log under` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(0) + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'run_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) return check(cmd, args) } @@ -1227,6 +1243,9 @@ func newLogInputs() *cobra.Command { } } } + if !cmd.Flags().Changed("json") { + logInputsReq.RunId = args[0] + } err = w.Experiments.LogInputs(ctx, logInputsReq) if err != nil { @@ -1270,11 +1289,11 @@ func newLogMetric() *cobra.Command { cmd.Flags().Int64Var(&logMetricReq.Step, "step", logMetricReq.Step, `Step at which to log the metric.`) cmd.Use = "log-metric KEY VALUE TIMESTAMP" - cmd.Short = `Log a metric.` - cmd.Long = `Log a metric. + cmd.Short = `Log a metric for a run.` + cmd.Long = `Log a metric for a run. - Logs a metric for a run. A metric is a key-value pair (string key, float - value) with an associated timestamp. Examples include the various metrics that + Log a metric for a run. A metric is a key-value pair (string key, float value) + with an associated timestamp. Examples include the various metrics that represent ML model accuracy. A metric can be logged multiple times. Arguments: @@ -1442,8 +1461,8 @@ func newLogParam() *cobra.Command { cmd.Flags().StringVar(&logParamReq.RunUuid, "run-uuid", logParamReq.RunUuid, `[Deprecated, use run_id instead] ID of the run under which to log the param.`) cmd.Use = "log-param KEY VALUE" - cmd.Short = `Log a param.` - cmd.Long = `Log a param. + cmd.Short = `Log a param for a run.` + cmd.Long = `Log a param for a run. Logs a param used for a run. A param is a key-value pair (string key, string value). Examples include hyperparameters used for ML model training and @@ -1530,8 +1549,8 @@ func newRestoreExperiment() *cobra.Command { cmd.Flags().Var(&restoreExperimentJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "restore-experiment EXPERIMENT_ID" - cmd.Short = `Restores an experiment.` - cmd.Long = `Restores an experiment. + cmd.Short = `Restore an experiment.` + cmd.Long = `Restore an experiment. Restore an experiment marked for deletion. This also restores associated metadata, runs, metrics, params, and tags. If experiment uses FileStore, @@ -1619,7 +1638,11 @@ func newRestoreRun() *cobra.Command { cmd.Short = `Restore a run.` cmd.Long = `Restore a run. - Restores a deleted run. + Restores a deleted run. This also restores associated metadata, runs, metrics, + params, and tags. + + Throws RESOURCE_DOES_NOT_EXIST if the run was never created or was + permanently deleted. Arguments: RUN_ID: ID of the run to restore.` @@ -1705,7 +1728,6 @@ func newRestoreRuns() *cobra.Command { Bulk restore runs in an experiment that were deleted no earlier than the specified timestamp. Restores at most max_runs per request. To call this API from a Databricks Notebook in Python, you can use the client code snippet on - https://learn.microsoft.com/en-us/azure/databricks/mlflow/runs#bulk-restore. Arguments: EXPERIMENT_ID: The ID of the experiment containing the runs to restore. @@ -1875,7 +1897,7 @@ func newSearchRuns() *cobra.Command { Searches for runs that satisfy expressions. - Search expressions can use mlflowMetric and mlflowParam keys.",` + Search expressions can use mlflowMetric and mlflowParam keys.` cmd.Annotations = make(map[string]string) @@ -1937,18 +1959,16 @@ func newSetExperimentTag() *cobra.Command { cmd.Flags().Var(&setExperimentTagJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "set-experiment-tag EXPERIMENT_ID KEY VALUE" - cmd.Short = `Set a tag.` - cmd.Long = `Set a tag. + cmd.Short = `Set a tag for an experiment.` + cmd.Long = `Set a tag for an experiment. Sets a tag on an experiment. Experiment tags are metadata that can be updated. Arguments: EXPERIMENT_ID: ID of the experiment under which to log the tag. Must be provided. - KEY: Name of the tag. Maximum size depends on storage backend. All storage - backends are guaranteed to support key values up to 250 bytes in size. - VALUE: String value of the tag being logged. Maximum size depends on storage - backend. All storage backends are guaranteed to support key values up to - 5000 bytes in size.` + KEY: Name of the tag. Keys up to 250 bytes in size are supported. + VALUE: String value of the tag being logged. Values up to 64KB in size are + supported.` cmd.Annotations = make(map[string]string) @@ -2108,18 +2128,16 @@ func newSetTag() *cobra.Command { cmd.Flags().StringVar(&setTagReq.RunUuid, "run-uuid", setTagReq.RunUuid, `[Deprecated, use run_id instead] ID of the run under which to log the tag.`) cmd.Use = "set-tag KEY VALUE" - cmd.Short = `Set a tag.` - cmd.Long = `Set a tag. + cmd.Short = `Set a tag for a run.` + cmd.Long = `Set a tag for a run. Sets a tag on a run. Tags are run metadata that can be updated during a run and after a run completes. Arguments: - KEY: Name of the tag. Maximum size depends on storage backend. All storage - backends are guaranteed to support key values up to 250 bytes in size. - VALUE: String value of the tag being logged. Maximum size depends on storage - backend. All storage backends are guaranteed to support key values up to - 5000 bytes in size.` + KEY: Name of the tag. Keys up to 250 bytes in size are supported. + VALUE: String value of the tag being logged. Values up to 64KB in size are + supported.` cmd.Annotations = make(map[string]string) @@ -2356,6 +2374,7 @@ func newUpdateRun() *cobra.Command { cmd.Flags().Int64Var(&updateRunReq.EndTime, "end-time", updateRunReq.EndTime, `Unix timestamp in milliseconds of when the run ended.`) cmd.Flags().StringVar(&updateRunReq.RunId, "run-id", updateRunReq.RunId, `ID of the run to update.`) + cmd.Flags().StringVar(&updateRunReq.RunName, "run-name", updateRunReq.RunName, `Updated name of the run.`) cmd.Flags().StringVar(&updateRunReq.RunUuid, "run-uuid", updateRunReq.RunUuid, `[Deprecated, use run_id instead] ID of the run to update.`) cmd.Flags().Var(&updateRunReq.Status, "status", `Updated status of the run. Supported values: [FAILED, FINISHED, KILLED, RUNNING, SCHEDULED]`) diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index 99841637a..3ede258ea 100755 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -41,6 +41,7 @@ func New() *cobra.Command { cmd.AddCommand(newGetMessage()) cmd.AddCommand(newGetMessageQueryResult()) cmd.AddCommand(newGetMessageQueryResultByAttachment()) + cmd.AddCommand(newGetSpace()) cmd.AddCommand(newStartConversation()) // Apply optional overrides to this command. @@ -78,8 +79,9 @@ func newCreateMessage() *cobra.Command { cmd.Short = `Create conversation message.` cmd.Long = `Create conversation message. - Create new message in [conversation](:method:genie/startconversation). The AI - response uses all previously created messages in the conversation to respond. + Create new message in a [conversation](:method:genie/startconversation). The + AI response uses all previously created messages in the conversation to + respond. Arguments: SPACE_ID: The ID associated with the Genie space where the conversation is started. @@ -298,8 +300,8 @@ func newGetMessageQueryResult() *cobra.Command { // TODO: short flags cmd.Use = "get-message-query-result SPACE_ID CONVERSATION_ID MESSAGE_ID" - cmd.Short = `Get conversation message SQL query result.` - cmd.Long = `Get conversation message SQL query result. + cmd.Short = `[Deprecated] Get conversation message SQL query result.` + cmd.Long = `[Deprecated] Get conversation message SQL query result. Get the result of SQL query if the message has a query attachment. This is only available if a message has a query attachment and the message status is @@ -362,11 +364,12 @@ func newGetMessageQueryResultByAttachment() *cobra.Command { // TODO: short flags cmd.Use = "get-message-query-result-by-attachment SPACE_ID CONVERSATION_ID MESSAGE_ID ATTACHMENT_ID" - cmd.Short = `Get conversation message SQL query result by attachment id.` - cmd.Long = `Get conversation message SQL query result by attachment id. + cmd.Short = `Get conversation message SQL query result.` + cmd.Long = `Get conversation message SQL query result. - Get the result of SQL query by attachment id This is only available if a - message has a query attachment and the message status is EXECUTING_QUERY. + Get the result of SQL query if the message has a query attachment. This is + only available if a message has a query attachment and the message status is + EXECUTING_QUERY OR COMPLETED. Arguments: SPACE_ID: Genie space ID @@ -410,6 +413,64 @@ func newGetMessageQueryResultByAttachment() *cobra.Command { return cmd } +// start get-space command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getSpaceOverrides []func( + *cobra.Command, + *dashboards.GenieGetSpaceRequest, +) + +func newGetSpace() *cobra.Command { + cmd := &cobra.Command{} + + var getSpaceReq dashboards.GenieGetSpaceRequest + + // TODO: short flags + + cmd.Use = "get-space SPACE_ID" + cmd.Short = `Get details of a Genie Space.` + cmd.Long = `Get details of a Genie Space. + + Get a Genie Space. + + Arguments: + SPACE_ID: The ID associated with the Genie space` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getSpaceReq.SpaceId = args[0] + + response, err := w.Genie.GetSpace(ctx, getSpaceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getSpaceOverrides { + fn(cmd, &getSpaceReq) + } + + return cmd +} + // start start-conversation command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index 4d6262cff..4edef6a0e 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -34,6 +34,7 @@ func New() *cobra.Command { cmd.AddCommand(newDelete()) cmd.AddCommand(newGet()) cmd.AddCommand(newList()) + cmd.AddCommand(newListProviderShareAssets()) cmd.AddCommand(newListShares()) cmd.AddCommand(newUpdate()) @@ -337,6 +338,72 @@ func newList() *cobra.Command { return cmd } +// start list-provider-share-assets command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listProviderShareAssetsOverrides []func( + *cobra.Command, + *sharing.ListProviderShareAssetsRequest, +) + +func newListProviderShareAssets() *cobra.Command { + cmd := &cobra.Command{} + + var listProviderShareAssetsReq sharing.ListProviderShareAssetsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listProviderShareAssetsReq.FunctionMaxResults, "function-max-results", listProviderShareAssetsReq.FunctionMaxResults, `Maximum number of functions to return.`) + cmd.Flags().IntVar(&listProviderShareAssetsReq.NotebookMaxResults, "notebook-max-results", listProviderShareAssetsReq.NotebookMaxResults, `Maximum number of notebooks to return.`) + cmd.Flags().IntVar(&listProviderShareAssetsReq.TableMaxResults, "table-max-results", listProviderShareAssetsReq.TableMaxResults, `Maximum number of tables to return.`) + cmd.Flags().IntVar(&listProviderShareAssetsReq.VolumeMaxResults, "volume-max-results", listProviderShareAssetsReq.VolumeMaxResults, `Maximum number of volumes to return.`) + + cmd.Use = "list-provider-share-assets PROVIDER_NAME SHARE_NAME" + cmd.Short = `List assets by provider share.` + cmd.Long = `List assets by provider share. + + Get arrays of assets associated with a specified provider's share. The caller + is the recipient of the share. + + Arguments: + PROVIDER_NAME: The name of the provider who owns the share. + SHARE_NAME: The name of the share.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + listProviderShareAssetsReq.ProviderName = args[0] + listProviderShareAssetsReq.ShareName = args[1] + + response, err := w.Providers.ListProviderShareAssets(ctx, listProviderShareAssetsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listProviderShareAssetsOverrides { + fn(cmd, &listProviderShareAssetsReq) + } + + return cmd +} + // start list-shares command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index 62c3407f4..59167224b 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -484,8 +484,6 @@ func newUpdatePermissions() *cobra.Command { cmd.Flags().Var(&updatePermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: changes - cmd.Flags().IntVar(&updatePermissionsReq.MaxResults, "max-results", updatePermissionsReq.MaxResults, `Maximum number of permissions to return.`) - cmd.Flags().StringVar(&updatePermissionsReq.PageToken, "page-token", updatePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "update-permissions NAME" cmd.Short = `Update permissions.` @@ -494,8 +492,8 @@ func newUpdatePermissions() *cobra.Command { Updates the permissions for a data share in the metastore. The caller must be a metastore admin or an owner of the share. - For new recipient grants, the user must also be the owner of the recipients. - recipient revocations do not require additional privileges. + For new recipient grants, the user must also be the recipient owner or + metastore admin. recipient revocations do not require additional privileges. Arguments: NAME: The name of the share.` @@ -526,11 +524,11 @@ func newUpdatePermissions() *cobra.Command { } updatePermissionsReq.Name = args[0] - err = w.Shares.UpdatePermissions(ctx, updatePermissionsReq) + response, err := w.Shares.UpdatePermissions(ctx, updatePermissionsReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go index 832f4a6dc..74d724a0a 100755 --- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go +++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go @@ -426,6 +426,7 @@ func newQueryIndex() *cobra.Command { // TODO: short flags cmd.Flags().Var(&queryIndexJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: columns_to_rerank cmd.Flags().StringVar(&queryIndexReq.FiltersJson, "filters-json", queryIndexReq.FiltersJson, `JSON string representing query filters.`) cmd.Flags().IntVar(&queryIndexReq.NumResults, "num-results", queryIndexReq.NumResults, `Number of results to return.`) cmd.Flags().StringVar(&queryIndexReq.QueryText, "query-text", queryIndexReq.QueryText, `Query text.`) diff --git a/cmd/workspace/workspace/workspace.go b/cmd/workspace/workspace/workspace.go index 61e1437a1..7326f9bea 100755 --- a/cmd/workspace/workspace/workspace.go +++ b/cmd/workspace/workspace/workspace.go @@ -177,6 +177,7 @@ func newExport() *cobra.Command { DBC, HTML, JUPYTER, + RAW, R_MARKDOWN, SOURCE, ]`) @@ -539,7 +540,7 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().IntVar(&listReq.NotebooksModifiedAfter, "notebooks-modified-after", listReq.NotebooksModifiedAfter, `UTC timestamp in milliseconds.`) + cmd.Flags().Int64Var(&listReq.NotebooksModifiedAfter, "notebooks-modified-after", listReq.NotebooksModifiedAfter, `UTC timestamp in milliseconds.`) cmd.Use = "list PATH" cmd.Short = `List contents.` diff --git a/go.mod b/go.mod index 6e3c51e79..9ce6f400c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // MIT github.com/Masterminds/semver/v3 v3.3.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.58.1 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.59.0 // Apache 2.0 github.com/fatih/color v1.18.0 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/gorilla/mux v1.8.1 // BSD 3-Clause diff --git a/go.sum b/go.sum index 2caabeb95..0e9c2a058 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.58.1 h1:dUs9ZmFi7hYiL3NwLSAbxqQu66E3BzwM8EU/wcCTJ10= -github.com/databricks/databricks-sdk-go v0.58.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= +github.com/databricks/databricks-sdk-go v0.59.0 h1:m87rbnoeO7A6+QKo4QzwyPE5AzEeGvopEaavn3F5y/o= +github.com/databricks/databricks-sdk-go v0.59.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/integration/bundle/bind_resource_test.go b/integration/bundle/bind_resource_test.go index ba10190aa..b182150ad 100644 --- a/integration/bundle/bind_resource_test.go +++ b/integration/bundle/bind_resource_test.go @@ -15,8 +15,62 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sdk-go/service/catalog" ) +func TestBindSchemaToExistingSchema(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + + // create a pre-defined schema: + uniqueId := uuid.New().String() + predefinedSchema, err := wt.W.Schemas.Create(ctx, catalog.CreateSchema{ + CatalogName: "main", + Name: "test-schema-" + uniqueId, + }) + require.NoError(t, err) + t.Cleanup(func() { + err := wt.W.Schemas.DeleteByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + }) + + // setup the bundle: + bundleRoot := initTestTemplate(t, ctx, "uc_schema_only", map[string]any{ + "unique_id": uniqueId, + }) + ctx = env.Set(ctx, "BUNDLE_ROOT", bundleRoot) + + // run the bind command: + c := testcli.NewRunner(t, ctx, "bundle", "deployment", "bind", "schema1", predefinedSchema.FullName, "--auto-approve") + _, _, err = c.Run() + require.NoError(t, err) + + // deploy the bundle: + deployBundle(t, ctx, bundleRoot) + + // Check that predefinedSchema is updated with config from bundle + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + updatedSchema, err := w.Schemas.GetByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + require.Equal(t, updatedSchema.SchemaId, predefinedSchema.SchemaId) + require.Equal(t, "This schema was created from DABs", updatedSchema.Comment) + + // unbind the schema: + c = testcli.NewRunner(t, ctx, "bundle", "deployment", "unbind", "schema1") + _, _, err = c.Run() + require.NoError(t, err) + + // destroy the bundle: + destroyBundle(t, ctx, bundleRoot) + + // Check that schema is unbound and exists after bundle is destroyed + postDestroySchema, err := w.Schemas.GetByFullName(ctx, predefinedSchema.FullName) + require.NoError(t, err) + require.Equal(t, postDestroySchema.SchemaId, predefinedSchema.SchemaId) +} + func TestBindJobToExistingJob(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) gt := &generateJobTest{T: wt, w: wt.W} diff --git a/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json b/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json new file mode 100644 index 000000000..1d3cdec85 --- /dev/null +++ b/integration/bundle/bundles/uc_schema_only/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema name" + } + } +} diff --git a/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl b/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl new file mode 100644 index 000000000..39c64f12b --- /dev/null +++ b/integration/bundle/bundles/uc_schema_only/template/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: uc-schema-only + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +resources: + schemas: + schema1: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created from DABs + diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go index ca66491ab..9ac8f47bd 100644 --- a/integration/bundle/init_default_python_test.go +++ b/integration/bundle/init_default_python_test.go @@ -31,8 +31,8 @@ var pythonVersionsShort = []string{ } var extraInstalls = map[string][]string{ - "3.12": {"setuptools"}, - "3.13": {"setuptools"}, + "3.12": {"setuptools==75.8.2"}, + "3.13": {"setuptools==75.8.2"}, } func TestDefaultPython(t *testing.T) { diff --git a/integration/bundle/testdata/apps/bundle_deploy.txt b/integration/bundle/testdata/apps/bundle_deploy.txt index 437a55596..55b8dedc6 100644 --- a/integration/bundle/testdata/apps/bundle_deploy.txt +++ b/integration/bundle/testdata/apps/bundle_deploy.txt @@ -2,3 +2,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ/files. Deploying resources... Updating deployment state... Deployment complete! +Warning: App config section detected + +remove 'config' from app resource 'test_app' section and use app.yml file in the root of this app instead + diff --git a/integration/bundle/testdata/apps/bundle_validate.txt b/integration/bundle/testdata/apps/bundle_validate.txt index 567fafd24..cb37fc4bd 100644 --- a/integration/bundle/testdata/apps/bundle_validate.txt +++ b/integration/bundle/testdata/apps/bundle_validate.txt @@ -1,7 +1,11 @@ +Warning: App config section detected + +remove 'config' from app resource 'test_app' section and use app.yml file in the root of this app instead + Name: basic Target: default Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/$UNIQUE_PRJ -Validation OK! +Found 1 warning diff --git a/integration/cmd/auth/describe_test.go b/integration/cmd/auth/describe_test.go index f592bc276..3d0a9b17d 100644 --- a/integration/cmd/auth/describe_test.go +++ b/integration/cmd/auth/describe_test.go @@ -2,16 +2,21 @@ package auth_test import ( "context" + "path/filepath" + "regexp" + "strings" "testing" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/cli/internal/testcli" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/require" ) func TestAuthDescribeSuccess(t *testing.T) { - t.Skipf("Skipping because of https://github.com/databricks/cli/issues/2010") - ctx := context.Background() stdout, _ := testcli.RequireSuccessfulRun(t, ctx, "auth", "describe") outStr := stdout.String() @@ -20,7 +25,9 @@ func TestAuthDescribeSuccess(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, outStr) - require.Contains(t, outStr, "Host: "+w.Config.Host) + + hostWithoutPrefix := strings.TrimPrefix(w.Config.Host, "https://") + require.Regexp(t, "Host: (?:https://)?"+regexp.QuoteMeta(hostWithoutPrefix), outStr) me, err := w.CurrentUser.Me(context.Background()) require.NoError(t, err) @@ -32,8 +39,19 @@ func TestAuthDescribeSuccess(t *testing.T) { } func TestAuthDescribeFailure(t *testing.T) { - t.Skipf("Skipping because of https://github.com/databricks/cli/issues/2010") + testutil.CleanupEnvironment(t) + // set up a custom config file: + home := t.TempDir() + cfg := &config.Config{ + ConfigFile: filepath.Join(home, "customcfg"), + Profile: "profile1", + } + err := databrickscfg.SaveToProfile(context.Background(), cfg) + require.NoError(t, err) + t.Setenv("DATABRICKS_CONFIG_FILE", filepath.Join(home, "customcfg")) + + // run the command: ctx := context.Background() stdout, _ := testcli.RequireSuccessfulRun(t, ctx, "auth", "describe", "--profile", "nonexistent") outStr := stdout.String() @@ -42,10 +60,5 @@ func TestAuthDescribeFailure(t *testing.T) { require.Contains(t, outStr, "Unable to authenticate: resolve") require.Contains(t, outStr, "has no nonexistent profile configured") require.Contains(t, outStr, "Current configuration:") - - w, err := databricks.NewWorkspaceClient(&databricks.Config{}) - require.NoError(t, err) - - require.Contains(t, outStr, "✓ host: "+w.Config.Host) require.Contains(t, outStr, "✓ profile: nonexistent (from --profile flag)") } diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index 88e6ed89a..337db8fca 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -224,22 +224,21 @@ func (a *syncTest) snapshotContains(files []string) { _, ok := s.LastModifiedTimes[filePath] assert.True(a.t, ok, "%s not in snapshot file: %v", filePath, s.LastModifiedTimes) } - assert.Equal(a.t, len(files), len(s.LastModifiedTimes)) + assert.Equal(a.t, len(files), len(s.LastModifiedTimes), "files=%s s.LastModifiedTimes=%s", files, s.LastModifiedTimes) } func TestSyncFullFileSync(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--full", "--watch") - // .gitignore is created by the sync process to enforce .databricks is not synced assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) // New file localFilePath := filepath.Join(assertSync.localRoot, "foo.txt") f := testfile.CreateFile(t, localFilePath) defer f.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt")) assertSync.remoteFileContent(ctx, "foo.txt", "") // Write to file @@ -255,24 +254,23 @@ func TestSyncFullFileSync(t *testing.T) { // delete f.Remove(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) } func TestSyncIncrementalFileSync(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--watch") - // .gitignore is created by the sync process to enforce .databricks is not synced assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) // New file localFilePath := filepath.Join(assertSync.localRoot, "foo.txt") f := testfile.CreateFile(t, localFilePath) defer f.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt")) assertSync.remoteFileContent(ctx, "foo.txt", "") - assertSync.snapshotContains(append(repoFiles, "foo.txt", ".gitignore")) + assertSync.snapshotContains(append(repoFiles, "foo.txt")) // Write to file f.Overwrite(t, `{"statement": "Mi Gente"}`) @@ -287,16 +285,15 @@ func TestSyncIncrementalFileSync(t *testing.T) { // delete f.Remove(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) - assertSync.snapshotContains(append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) + assertSync.snapshotContains(repoFiles) } func TestSyncNestedFolderSync(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--watch") - // .gitignore is created by the sync process to enforce .databricks is not synced assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) // New file localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt") @@ -305,25 +302,24 @@ func TestSyncNestedFolderSync(t *testing.T) { f := testfile.CreateFile(t, localFilePath) defer f.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "dir1")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1")) assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"}) assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"}) assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/dir2/dir3/foo.txt")) + assertSync.snapshotContains(append(repoFiles, "dir1/dir2/dir3/foo.txt")) // delete f.Remove(t) assertSync.waitForCompletionMarker() assertSync.remoteNotExist(ctx, "dir1") - assertSync.snapshotContains(append(repoFiles, ".gitignore")) + assertSync.snapshotContains(repoFiles) } func TestSyncNestedFolderDoesntFailOnNonEmptyDirectory(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--watch") - // .gitignore is created by the sync process to enforce .databricks is not synced assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) // New file localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt") @@ -353,9 +349,8 @@ func TestSyncNestedFolderDoesntFailOnNonEmptyDirectory(t *testing.T) { func TestSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--watch") - // .gitignore is created by the sync process to enforce .databricks is not synced assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) // New file localFilePath := filepath.Join(assertSync.localRoot, "dir1/a b+c/c+d e/e+f g#i.txt") @@ -364,17 +359,17 @@ func TestSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) { f := testfile.CreateFile(t, localFilePath) defer f.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "dir1")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1")) assertSync.remoteDirContent(ctx, "dir1", []string{"a b+c"}) assertSync.remoteDirContent(ctx, "dir1/a b+c", []string{"c+d e"}) assertSync.remoteDirContent(ctx, "dir1/a b+c/c+d e", []string{"e+f g#i.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/a b+c/c+d e/e+f g#i.txt")) + assertSync.snapshotContains(append(repoFiles, "dir1/a b+c/c+d e/e+f g#i.txt")) // delete f.Remove(t) assertSync.waitForCompletionMarker() assertSync.remoteNotExist(ctx, "dir1/a b+c/c+d e") - assertSync.snapshotContains(append(repoFiles, ".gitignore")) + assertSync.snapshotContains(repoFiles) } // This is a check for the edge case when a user does the following: @@ -395,23 +390,23 @@ func TestSyncIncrementalFileOverwritesFolder(t *testing.T) { f := testfile.CreateFile(t, localFilePath) defer f.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo/bar.txt")) + assertSync.snapshotContains(append(repoFiles, "foo/bar.txt")) // delete foo/bar.txt f.Remove(t) os.Remove(filepath.Join(assertSync.localRoot, "foo")) assertSync.waitForCompletionMarker() assertSync.remoteNotExist(ctx, "foo") - assertSync.snapshotContains(append(repoFiles, ".gitignore")) + assertSync.snapshotContains(repoFiles) f2 := testfile.CreateFile(t, filepath.Join(assertSync.localRoot, "foo")) defer f2.Close(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) assertSync.objectType(ctx, "foo", "FILE") - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo")) + assertSync.snapshotContains(append(repoFiles, "foo")) } func TestSyncIncrementalSyncPythonNotebookToFile(t *testing.T) { @@ -425,23 +420,23 @@ func TestSyncIncrementalSyncPythonNotebookToFile(t *testing.T) { // notebook was uploaded properly assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.language(ctx, "foo", "PYTHON") - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py")) + assertSync.snapshotContains(append(repoFiles, "foo.py")) // convert to vanilla python file f.Overwrite(t, "# No longer a python notebook") assertSync.waitForCompletionMarker() assertSync.objectType(ctx, "foo.py", "FILE") - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py")) - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py")) + assertSync.snapshotContains(append(repoFiles, "foo.py")) // delete the vanilla python file f.Remove(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) - assertSync.snapshotContains(append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) + assertSync.snapshotContains(repoFiles) } func TestSyncIncrementalSyncFileToPythonNotebook(t *testing.T) { @@ -454,17 +449,17 @@ func TestSyncIncrementalSyncFileToPythonNotebook(t *testing.T) { assertSync.waitForCompletionMarker() // assert file upload - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py")) assertSync.objectType(ctx, "foo.py", "FILE") - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py")) + assertSync.snapshotContains(append(repoFiles, "foo.py")) // convert to notebook f.Overwrite(t, "# Databricks notebook source") assertSync.waitForCompletionMarker() assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.language(ctx, "foo", "PYTHON") - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) - assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) + assertSync.snapshotContains(append(repoFiles, "foo.py")) } func TestSyncIncrementalSyncPythonNotebookDelete(t *testing.T) { @@ -478,14 +473,14 @@ func TestSyncIncrementalSyncPythonNotebookDelete(t *testing.T) { assertSync.waitForCompletionMarker() // notebook was uploaded properly - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) + assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.language(ctx, "foo", "PYTHON") // Delete notebook f.Remove(t) assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore")) + assertSync.remoteDirContent(ctx, "", repoFiles) } func TestSyncEnsureRemotePathIsUsableIfRepoDoesntExist(t *testing.T) { diff --git a/internal/testutil/env.go b/internal/testutil/env.go index 598229655..1ecbe6485 100644 --- a/internal/testutil/env.go +++ b/internal/testutil/env.go @@ -13,19 +13,11 @@ import ( // The original environment is restored upon test completion. // Note: use of this function is incompatible with parallel execution. func CleanupEnvironment(t TestingT) { - // Restore environment when test finishes. - environ := os.Environ() - t.Cleanup(func() { - // Restore original environment. - for _, kv := range environ { - kvs := strings.SplitN(kv, "=", 2) - os.Setenv(kvs[0], kvs[1]) - } - }) - path := os.Getenv("PATH") pwd := os.Getenv("PWD") - os.Clearenv() + + // Clear all environment variables. + NullEnvironment(t) // We use t.Setenv instead of os.Setenv because the former actively // prevents a test being run with t.Parallel. Modifying the environment @@ -38,6 +30,23 @@ func CleanupEnvironment(t TestingT) { } } +// NullEnvironment sets up an empty environment with absolutely no environment variables set. +// The original environment is restored upon test completion. +// Note: use of this function is incompatible with parallel execution +func NullEnvironment(t TestingT) { + // Restore environment when test finishes. + environ := os.Environ() + t.Cleanup(func() { + // Restore original environment. + for _, kv := range environ { + kvs := strings.SplitN(kv, "=", 2) + os.Setenv(kvs[0], kvs[1]) + } + }) + + os.Clearenv() +} + // Changes into specified directory for the duration of the test. // Returns the current working directory. func Chdir(t TestingT, dir string) string { diff --git a/libs/auth/env.go b/libs/auth/env.go index db91c503b..541b03aa2 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -1,6 +1,13 @@ package auth -import "github.com/databricks/databricks-sdk-go/config" +import ( + "fmt" + "os" + "slices" + "strings" + + "github.com/databricks/databricks-sdk-go/config" +) // Env generates the authentication environment variables we need to set for // downstream applications from the CLI to work correctly. @@ -52,3 +59,71 @@ func GetEnvFor(name string) (string, bool) { return "", false } + +// EnvVars returns the list of environment variables that the SDK reads to configure +// authentication. +// This is useful for spawning subprocesses since you can unset all auth environment +// variables to clean up the environment before configuring authentication for the +// child process. +func envVars() []string { + out := []string{} + + for _, attr := range config.ConfigAttributes { + if len(attr.EnvVars) == 0 { + continue + } + + out = append(out, attr.EnvVars...) + } + + return out +} + +// ProcessEnv generates the environment variables that should be set to authenticate +// downstream processes to use the same auth credentials as in cfg. +func ProcessEnv(cfg *config.Config) []string { + // We want child processes to inherit environment variables like $HOME or $HTTPS_PROXY + // because they influence auth resolution. + base := os.Environ() + + out := []string{} + authEnvVars := envVars() + + // Remove any existing auth environment variables. This is done because + // the CLI offers multiple modalities of configuring authentication like + // `--profile` or `DATABRICKS_CONFIG_PROFILE` or `profile: ` in the + // bundle config file. + // + // Each of these modalities have different priorities and thus we don't want + // any auth configuration to piggyback into the child process environment. + // + // This is a precaution to avoid conflicting auth configurations being passed + // to the child telemetry process. + // + // Normally this should be unnecessary because the SDK should error if multiple + // authentication methods have been configured. But there is no harm in doing this + // as a precaution. + for _, v := range base { + k, _, found := strings.Cut(v, "=") + if !found { + continue + } + if slices.Contains(authEnvVars, k) { + continue + } + out = append(out, v) + } + + // Now add the necessary authentication environment variables. + newEnv := Env(cfg) + for k, v := range newEnv { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + + // Sort the environment variables so that the output is deterministic. + // Keeping the output deterministic helps with reproducibility and keeping the + // behavior consistent incase there are any issues. + slices.Sort(out) + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index b0ab1fbaf..fcc071af6 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -116,3 +116,51 @@ func TestGetEnvFor(t *testing.T) { assert.False(t, ok) assert.Empty(t, out) } + +func TestAuthEnvVars(t *testing.T) { + // Few common environment variables that we expect the SDK to support. + contains := []string{ + // Generic attributes. + "DATABRICKS_HOST", + "DATABRICKS_CONFIG_PROFILE", + "DATABRICKS_AUTH_TYPE", + "DATABRICKS_METADATA_SERVICE_URL", + "DATABRICKS_CONFIG_FILE", + + // OAuth specific attributes. + "DATABRICKS_CLIENT_ID", + "DATABRICKS_CLIENT_SECRET", + "DATABRICKS_CLI_PATH", + + // Google specific attributes. + "DATABRICKS_GOOGLE_SERVICE_ACCOUNT", + "GOOGLE_CREDENTIALS", + + // Personal access token specific attributes. + "DATABRICKS_TOKEN", + + // Databricks password specific attributes. + "DATABRICKS_USERNAME", + "DATABRICKS_PASSWORD", + + // Account authentication attributes. + "DATABRICKS_ACCOUNT_ID", + + // Azure attributes + "DATABRICKS_AZURE_RESOURCE_ID", + "ARM_USE_MSI", + "ARM_CLIENT_SECRET", + "ARM_CLIENT_ID", + "ARM_TENANT_ID", + "ARM_ENVIRONMENT", + + // Github attributes + "ACTIONS_ID_TOKEN_REQUEST_URL", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + } + + out := envVars() + for _, v := range contains { + assert.Contains(t, out, v) + } +} diff --git a/libs/cmdio/logger.go b/libs/cmdio/logger.go index 48b76ce42..7d369a8a0 100644 --- a/libs/cmdio/logger.go +++ b/libs/cmdio/logger.go @@ -9,6 +9,7 @@ import ( "io" "os" "strings" + "sync" "github.com/databricks/cli/libs/flags" "github.com/manifoldco/promptui" @@ -29,6 +30,8 @@ type Logger struct { // If true, indicates no events have been printed by the logger yet. Used // by inplace logging for formatting isFirstEvent bool + + mutex sync.Mutex } func NewLogger(mode flags.ProgressLogFormat) *Logger { @@ -216,6 +219,8 @@ func (l *Logger) writeInplace(event Event) { } func (l *Logger) Log(event Event) { + l.mutex.Lock() + defer l.mutex.Unlock() switch l.Mode { case flags.ModeInplace: if event.IsInplaceSupported() { diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 7102b6e29..edf6bcfc4 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -257,8 +257,9 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { // The directory delete API returns a 400 if the directory is not empty if aerr.StatusCode == http.StatusBadRequest { reasons := []string{} - for _, detail := range aerr.Details { - reasons = append(reasons, detail.Reason) + details := aerr.ErrorDetails() + if details.ErrorInfo != nil { + reasons = append(reasons, details.ErrorInfo.Reason) } // Error code 400 is generic and can be returned for other reasons. Make // sure one of the reasons for the error is that the directory is not empty. diff --git a/libs/git/fileset.go b/libs/git/fileset.go index 8391548c9..7c0c372c9 100644 --- a/libs/git/fileset.go +++ b/libs/git/fileset.go @@ -43,7 +43,3 @@ func (f *FileSet) Files() ([]fileset.File, error) { f.view.repo.taintIgnoreRules() return f.fileset.Files() } - -func (f *FileSet) EnsureValidGitIgnoreExists() error { - return f.view.EnsureValidGitIgnoreExists() -} diff --git a/libs/git/fileset_test.go b/libs/git/fileset_test.go index 6d239edf5..cd85ee810 100644 --- a/libs/git/fileset_test.go +++ b/libs/git/fileset_test.go @@ -1,10 +1,8 @@ package git import ( - "os" "path" "path/filepath" - "strings" "testing" "github.com/databricks/cli/libs/vfs" @@ -51,34 +49,3 @@ func TestFileSetNonCleanRoot(t *testing.T) { require.NoError(t, err) assert.Len(t, files, 3) } - -func TestFileSetAddsCacheDirToGitIgnore(t *testing.T) { - projectDir := t.TempDir() - fileSet, err := NewFileSetAtRoot(vfs.MustNew(projectDir)) - require.NoError(t, err) - err = fileSet.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - gitIgnorePath := filepath.Join(projectDir, ".gitignore") - assert.FileExists(t, gitIgnorePath) - fileBytes, err := os.ReadFile(gitIgnorePath) - assert.NoError(t, err) - assert.Contains(t, string(fileBytes), ".databricks") -} - -func TestFileSetDoesNotCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) { - projectDir := t.TempDir() - gitIgnorePath := filepath.Join(projectDir, ".gitignore") - - fileSet, err := NewFileSetAtRoot(vfs.MustNew(projectDir)) - require.NoError(t, err) - err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644) - require.NoError(t, err) - - err = fileSet.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - b, err := os.ReadFile(gitIgnorePath) - require.NoError(t, err) - assert.Equal(t, 1, strings.Count(string(b), ".databricks")) -} diff --git a/libs/git/view.go b/libs/git/view.go index db22dfc5d..142cc4947 100644 --- a/libs/git/view.go +++ b/libs/git/view.go @@ -90,46 +90,25 @@ func NewView(worktreeRoot, root vfs.Path) (*View, error) { target = strings.TrimPrefix(target, string(os.PathSeparator)) target = path.Clean(filepath.ToSlash(target)) - return &View{ + result := &View{ repo: repo, targetPath: target, - }, nil + } + + result.SetupDefaults() + return result, nil } func NewViewAtRoot(root vfs.Path) (*View, error) { return NewView(root, root) } -func (v *View) EnsureValidGitIgnoreExists() error { - ign, err := v.IgnoreDirectory(".databricks") - if err != nil { - return err - } - - // return early if .databricks is already being ignored - if ign { - return nil - } - - // Create .gitignore with .databricks entry - gitIgnorePath := filepath.Join(v.repo.Root(), v.targetPath, ".gitignore") - file, err := os.OpenFile(gitIgnorePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) - if err != nil { - return err - } - defer file.Close() - +func (v *View) SetupDefaults() { // Hard code .databricks ignore pattern so that we never sync it (irrespective) // of .gitignore patterns v.repo.addIgnoreRule(newStringIgnoreRules([]string{ ".databricks", })) - _, err = file.WriteString("\n.databricks\n") - if err != nil { - return err - } - v.repo.taintIgnoreRules() - return nil } diff --git a/libs/git/view_test.go b/libs/git/view_test.go index 96881fdee..9d7a4cdec 100644 --- a/libs/git/view_test.go +++ b/libs/git/view_test.go @@ -209,100 +209,12 @@ func TestViewABInTempDir(t *testing.T) { assert.False(t, tv.Ignore("newfile")) } -func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredAtRoot(t *testing.T) { - expected, err := os.ReadFile("./testdata_view_ignore/.gitignore") - require.NoError(t, err) - - repoPath := createFakeRepo(t, "testdata_view_ignore") - - // Since root .gitignore already has .databricks, there should be no edits - // to root .gitignore - v, err := NewViewAtRoot(vfs.MustNew(repoPath)) - require.NoError(t, err) - - err = v.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - actual, err := os.ReadFile(filepath.Join(repoPath, ".gitignore")) - require.NoError(t, err) - - assert.Equal(t, string(expected), string(actual)) -} - -func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredInSubdir(t *testing.T) { - expected, err := os.ReadFile("./testdata_view_ignore/a/.gitignore") - require.NoError(t, err) - - repoPath := createFakeRepo(t, "testdata_view_ignore") - - // Since root .gitignore already has .databricks, there should be no edits - // to a/.gitignore - v, err := NewView(vfs.MustNew(repoPath), vfs.MustNew(filepath.Join(repoPath, "a"))) - require.NoError(t, err) - - err = v.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - actual, err := os.ReadFile(filepath.Join(repoPath, v.targetPath, ".gitignore")) - require.NoError(t, err) - - assert.Equal(t, string(expected), string(actual)) -} - -func TestViewAddsGitignoreWithCacheDir(t *testing.T) { - repoPath := createFakeRepo(t, "testdata") - err := os.Remove(filepath.Join(repoPath, ".gitignore")) - assert.NoError(t, err) - - // Since root .gitignore was deleted, new view adds .databricks to root .gitignore - v, err := NewViewAtRoot(vfs.MustNew(repoPath)) - require.NoError(t, err) - - err = v.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - actual, err := os.ReadFile(filepath.Join(repoPath, ".gitignore")) - require.NoError(t, err) - - assert.Contains(t, string(actual), "\n.databricks\n") -} - -func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) { - repoPath := createFakeRepo(t, "testdata") - err := os.Remove(filepath.Join(repoPath, ".gitignore")) - require.NoError(t, err) - - // Since root .gitignore was deleted, new view adds .databricks to a/.gitignore - v, err := NewView(vfs.MustNew(repoPath), vfs.MustNew(filepath.Join(repoPath, "a"))) - require.NoError(t, err) - - err = v.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - actual, err := os.ReadFile(filepath.Join(repoPath, v.targetPath, ".gitignore")) - require.NoError(t, err) - - // created .gitignore has cache dir listed - assert.Contains(t, string(actual), "\n.databricks\n") - assert.NoFileExists(t, filepath.Join(repoPath, ".gitignore")) -} - func TestViewAlwaysIgnoresCacheDir(t *testing.T) { repoPath := createFakeRepo(t, "testdata") v, err := NewViewAtRoot(vfs.MustNew(repoPath)) require.NoError(t, err) - err = v.EnsureValidGitIgnoreExists() - require.NoError(t, err) - - // Delete root .gitignore which contains .databricks entry - err = os.Remove(filepath.Join(repoPath, ".gitignore")) - require.NoError(t, err) - - // taint rules to reload .gitignore - v.repo.taintIgnoreRules() - // assert .databricks is still being ignored ign1, err := v.IgnoreDirectory(".databricks") require.NoError(t, err) diff --git a/libs/jsonschema/extension.go b/libs/jsonschema/extension.go index 9badf86a5..e3cae1a7f 100644 --- a/libs/jsonschema/extension.go +++ b/libs/jsonschema/extension.go @@ -40,4 +40,8 @@ type Extension struct { // https://code.visualstudio.com/docs/languages/json#_use-rich-formatting-in-hovers // Also it can be used in documentation generation. MarkdownDescription string `json:"markdownDescription,omitempty"` + + // This field is not in the JSON schema spec, but it is supported in VSCode + // It is used to provide a warning for deprectated fields + DeprecationMessage string `json:"deprecationMessage,omitempty"` } diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index ce25cb023..6e0ba7b26 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -18,10 +18,6 @@ var skipTags = []string{ // Annotation for internal bundle fields that should not be exposed to customers. // Fields can be tagged as "internal" to remove them from the generated schema. "internal", - - // Annotation for bundle fields that have been deprecated. - // Fields tagged as "deprecated" are omitted from the generated schema. - "deprecated", } type constructor struct { @@ -259,8 +255,8 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { structFields := getStructFields(typ) for _, structField := range structFields { bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") - // Fields marked as "readonly", "internal" or "deprecated" are skipped - // while generating the schema + // Fields marked as "readonly" or "internal" are skipped while generating + // the schema skip := false for _, tag := range skipTags { if slices.Contains(bundleTags, tag) { diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index cdfdcfd10..ae9ff54b1 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -17,11 +17,10 @@ func TestFromTypeBasic(t *testing.T) { TriplePointer ***int `json:"triple_pointer,omitempty"` // These fields should be ignored in the resulting schema. - NotAnnotated string - DashedTag string `json:"-"` - InternalTagged string `json:"internal_tagged" bundle:"internal"` - DeprecatedTagged string `json:"deprecated_tagged" bundle:"deprecated"` - ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"` + NotAnnotated string + DashedTag string `json:"-"` + InternalTagged string `json:"internal_tagged" bundle:"internal"` + ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"` } strRef := "#/$defs/string" diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 85f6a0328..6abeacfa4 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -80,6 +80,11 @@ type Schema struct { // Examples of the value for properties in the schema. // https://json-schema.org/understanding-json-schema/reference/annotations Examples any `json:"examples,omitempty"` + + // A boolean that indicates the field should not be used and may be removed + // in the future. + // https://json-schema.org/understanding-json-schema/reference/annotations + Deprecated bool `json:"deprecated,omitempty"` } // Default value defined in a JSON Schema, represented as a string. diff --git a/libs/sync/gitignore.go b/libs/sync/gitignore.go new file mode 100644 index 000000000..b3888a0cc --- /dev/null +++ b/libs/sync/gitignore.go @@ -0,0 +1,26 @@ +package sync + +import ( + "context" + "os" + "path/filepath" + + "github.com/databricks/cli/libs/log" +) + +func WriteGitIgnore(ctx context.Context, dir string) { + gitignorePath := filepath.Join(dir, ".databricks", ".gitignore") + file, err := os.OpenFile(gitignorePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + if os.IsExist(err) { + return + } + log.Debugf(ctx, "Failed to create %s: %s", gitignorePath, err) + } + + defer file.Close() + _, err = file.WriteString("*\n") + if err != nil { + log.Debugf(ctx, "Error writing to %s: %s", gitignorePath, err) + } +} diff --git a/libs/sync/sync.go b/libs/sync/sync.go index f13fa934a..4d14f745a 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -69,10 +69,7 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } - err = fileSet.EnsureValidGitIgnoreExists() - if err != nil { - return nil, err - } + WriteGitIgnore(ctx, opts.LocalRoot.Native()) includeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Include) if err != nil { diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index f30431770..1b5498275 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -40,9 +40,6 @@ func TestGetFileSet(t *testing.T) { fileSet, err := git.NewFileSetAtRoot(root) require.NoError(t, err) - err = fileSet.EnsureValidGitIgnoreExists() - require.NoError(t, err) - inc, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -59,7 +56,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Len(t, fileList, 10) + require.Len(t, fileList, 9) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -77,7 +74,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Len(t, fileList, 2) + require.Len(t, fileList, 1) inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) @@ -95,7 +92,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Len(t, fileList, 11) + require.Len(t, fileList, 10) } func TestRecursiveExclude(t *testing.T) { @@ -106,9 +103,6 @@ func TestRecursiveExclude(t *testing.T) { fileSet, err := git.NewFileSetAtRoot(root) require.NoError(t, err) - err = fileSet.EnsureValidGitIgnoreExists() - require.NoError(t, err) - inc, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -125,7 +119,7 @@ func TestRecursiveExclude(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Len(t, fileList, 7) + require.Len(t, fileList, 6) } func TestNegateExclude(t *testing.T) { @@ -136,9 +130,6 @@ func TestNegateExclude(t *testing.T) { fileSet, err := git.NewFileSetAtRoot(root) require.NoError(t, err) - err = fileSet.EnsureValidGitIgnoreExists() - require.NoError(t, err) - inc, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 97030324b..f9306ca31 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -91,6 +91,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri }) b.Tagging = tags.ForCloud(w.Config) + b.SetWorkpaceClient(w) b.WorkspaceClient() diags = phases.Initialize(ctx, b) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 4e943f828..80e88941d 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -83,6 +83,13 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) { s.files[path] = body } +func (s *FakeWorkspace) WorkspaceFilesExportFile(path string) []byte { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return s.files[path] +} + func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) Response { jobId := s.nextJobId s.nextJobId++