Merge remote-tracking branch 'origin' into implement-async-logger

This commit is contained in:
Shreyas Goenka 2025-02-03 00:21:02 +01:00
commit 5d75c3f098
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
21 changed files with 416 additions and 45 deletions

View File

@ -1,5 +1,25 @@
# Version changelog
## [Release] Release v0.240.0
Bundles:
* Added support for double underscore variable references ([#2203](https://github.com/databricks/cli/pull/2203)).
* Do not wait for app compute to start on `bundle deploy` ([#2144](https://github.com/databricks/cli/pull/2144)).
* Remove bundle.git.inferred ([#2258](https://github.com/databricks/cli/pull/2258)).
* libs/python: Remove DetectInterpreters ([#2234](https://github.com/databricks/cli/pull/2234)).
API Changes:
* Added `databricks access-control` command group.
* Added `databricks serving-endpoints http-request` command.
* Changed `databricks serving-endpoints create` command with new required argument order.
* Changed `databricks serving-endpoints get-open-api` command return type to become non-empty.
* Changed `databricks recipients update` command return type to become non-empty.
OpenAPI commit 0be1b914249781b5e903b7676fd02255755bc851 (2025-01-22)
Dependency updates:
* Bump github.com/databricks/databricks-sdk-go from 0.55.0 to 0.56.1 ([#2238](https://github.com/databricks/cli/pull/2238)).
* Upgrade TF provider to 1.64.1 ([#2247](https://github.com/databricks/cli/pull/2247)).
## [Release] Release v0.239.1
CLI:

1
acceptance/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
build

View File

@ -2,10 +2,12 @@ package acceptance_test
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
@ -25,7 +27,10 @@ import (
"github.com/stretchr/testify/require"
)
var KeepTmp bool
var (
KeepTmp bool
NoRepl bool
)
// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
// Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
@ -40,6 +45,7 @@ var InprocessMode bool
func init() {
flag.BoolVar(&InprocessMode, "inprocess", SingleTest != "", "Run CLI in the same process as test (for debugging)")
flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
flag.BoolVar(&NoRepl, "norepl", false, "Do not apply any replacements (for debugging)")
}
const (
@ -71,6 +77,11 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
cwd, err := os.Getwd()
require.NoError(t, err)
buildDir := filepath.Join(cwd, "build", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
// Download terraform and provider and create config; this also creates build directory.
RunCommand(t, []string{"python3", filepath.Join(cwd, "install_terraform.py"), "--targetdir", buildDir}, ".")
coverDir := os.Getenv("CLI_GOCOVERDIR")
if coverDir != "" {
@ -87,7 +98,7 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
t.Setenv("CMD_SERVER_URL", cmdServer.URL)
execPath = filepath.Join(cwd, "bin", "callserver.py")
} else {
execPath = BuildCLI(t, cwd, coverDir)
execPath = BuildCLI(t, buildDir, coverDir)
}
t.Setenv("CLI", execPath)
@ -108,20 +119,33 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
cloudEnv := os.Getenv("CLOUD_ENV")
if cloudEnv == "" {
server := testserver.New(t)
AddHandlers(server)
defaultServer := testserver.New(t)
AddHandlers(defaultServer)
// Redirect API access to local server:
t.Setenv("DATABRICKS_HOST", server.URL)
t.Setenv("DATABRICKS_HOST", defaultServer.URL)
t.Setenv("DATABRICKS_TOKEN", "dapi1234")
homeDir := t.TempDir()
// Do not read user's ~/.databrickscfg
t.Setenv(env.HomeEnvVar(), homeDir)
// Prevent CLI from downloading terraform in each test:
t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
}
terraformrcPath := filepath.Join(buildDir, ".terraformrc")
t.Setenv("TF_CLI_CONFIG_FILE", terraformrcPath)
t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath)
repls.SetPath(terraformrcPath, "$DATABRICKS_TF_CLI_CONFIG_FILE")
terraformExecPath := filepath.Join(buildDir, "terraform")
if runtime.GOOS == "windows" {
terraformExecPath += ".exe"
}
t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath)
t.Setenv("TERRAFORM", terraformExecPath)
repls.SetPath(terraformExecPath, "$TERRAFORM")
// do it last so that full paths match first:
repls.SetPath(buildDir, "$BUILD_DIR")
workspaceClient, err := databricks.NewWorkspaceClient()
require.NoError(t, err)
@ -210,6 +234,35 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
args := []string{"bash", "-euo", "pipefail", EntryPointScript}
cmd := exec.Command(args[0], args[1:]...)
cmd.Env = os.Environ()
// Start a new server with a custom configuration if the acceptance test
// specifies a custom server stubs.
var server *testserver.Server
// Start a new server for this test if either:
// 1. A custom server spec is defined in the test configuration.
// 2. The test is configured to record requests and assert on them. We need
// a duplicate of the default server to record requests because the default
// server otherwise is a shared resource.
if len(config.Server) > 0 || config.RecordRequests {
server = testserver.New(t)
server.RecordRequests = config.RecordRequests
// If no custom server stubs are defined, add the default handlers.
if len(config.Server) == 0 {
AddHandlers(server)
}
for _, stub := range config.Server {
require.NotEmpty(t, stub.Pattern)
server.Handle(stub.Pattern, func(req *http.Request) (resp any, err error) {
return stub.Response.Body, nil
})
}
cmd.Env = append(cmd.Env, "DATABRICKS_HOST="+server.URL)
}
if coverDir != "" {
// Creating individual coverage directory for each test, because writing to the same one
// results in sporadic failures like this one (only if tests are running in parallel):
@ -217,7 +270,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
err := os.MkdirAll(coverDir, os.ModePerm)
require.NoError(t, err)
cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
}
// Write combined output to a file
@ -228,6 +281,25 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
cmd.Dir = tmpDir
err = cmd.Run()
// Write the requests made to the server to a output file if the test is
// configured to record requests.
if config.RecordRequests {
f, err := os.OpenFile(filepath.Join(tmpDir, "out.requests.txt"), os.O_CREATE|os.O_WRONLY, 0o644)
require.NoError(t, err)
for _, req := range server.Requests {
reqJson, err := json.Marshal(req)
require.NoError(t, err)
line := fmt.Sprintf("%s\n", reqJson)
_, err = f.WriteString(line)
require.NoError(t, err)
}
err = f.Close()
require.NoError(t, err)
}
// Include exit code in output (if non-zero)
formatOutput(out, err)
require.NoError(t, out.Close())
@ -272,7 +344,9 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
// Apply replacements to the new value only.
// The reference value is stored after applying replacements.
valueNew = repls.Replace(valueNew)
if !NoRepl {
valueNew = repls.Replace(valueNew)
}
// The test did not produce an expected output file.
if okRef && !okNew {
@ -350,13 +424,12 @@ func readMergedScriptContents(t *testing.T, dir string) string {
return strings.Join(prepares, "\n")
}
func BuildCLI(t *testing.T, cwd, coverDir string) string {
execPath := filepath.Join(cwd, "build", "databricks")
func BuildCLI(t *testing.T, buildDir, coverDir string) string {
execPath := filepath.Join(buildDir, "databricks")
if runtime.GOOS == "windows" {
execPath += ".exe"
}
start := time.Now()
args := []string{
"go", "build",
"-mod", "vendor",
@ -374,20 +447,8 @@ func BuildCLI(t *testing.T, cwd, coverDir string) string {
args = append(args, "-buildvcs=false")
}
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = ".."
out, err := cmd.CombinedOutput()
elapsed := time.Since(start)
t.Logf("%s took %s", args, elapsed)
require.NoError(t, err, "go build failed: %s: %s\n%s", args, err, out)
if len(out) > 0 {
t.Logf("go build output: %s: %s", args, out)
}
// Quick check + warm up cache:
cmd = exec.Command(execPath, "--version")
out, err = cmd.CombinedOutput()
require.NoError(t, err, "%s --version failed: %s\n%s", execPath, err, out)
RunCommand(t, args, "..")
RunCommand(t, []string{execPath, "--version"}, ".")
return execPath
}
@ -525,3 +586,17 @@ func getUVDefaultCacheDir(t *testing.T) string {
return cacheDir + "/uv"
}
}
func RunCommand(t *testing.T, args []string, dir string) {
start := time.Now()
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
elapsed := time.Since(start)
t.Logf("%s took %s", args, elapsed)
require.NoError(t, err, "%s failed: %s\n%s", args, err, out)
if len(out) > 0 {
t.Logf("%s output: %s", args, out)
}
}

View File

@ -1 +0,0 @@
databricks

View File

@ -11,7 +11,7 @@
"name": "git",
"target": "prod",
"terraform": {
"exec_path": "$TMPHOME"
"exec_path": "$TERRAFORM"
}
},
"sync": {
@ -29,6 +29,7 @@
"workspace": {
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
"current_user": {
"id": "$USER.Id",
"short_name": "$USERNAME",
"userName": "$USERNAME"
},
@ -60,7 +61,7 @@ Validation OK!
"name": "git",
"target": "dev",
"terraform": {
"exec_path": "$TMPHOME"
"exec_path": "$TERRAFORM"
}
},
"sync": {
@ -78,6 +79,7 @@ Validation OK!
"workspace": {
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
"current_user": {
"id": "$USER.Id",
"short_name": "$USERNAME",
"userName": "$USERNAME"
},

View File

@ -7,7 +7,7 @@
},
"target": "dev",
"terraform": {
"exec_path": "$TMPHOME"
"exec_path": "$TERRAFORM"
}
},
"resources": {
@ -54,6 +54,7 @@
"workspace": {
"artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
"current_user": {
"id": "$USER.Id",
"short_name": "$USERNAME",
"userName": "$USERNAME"
},

View File

@ -1,6 +1,7 @@
{
"artifact_path": "TestResolveVariableReferences/bar/artifacts",
"current_user": {
"id": "$USER.Id",
"short_name": "$USERNAME",
"userName": "$USERNAME"
},

View File

@ -1,6 +1,7 @@
{
"artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts",
"current_user": {
"id": "$USER.Id",
"short_name": "$USERNAME",
"userName": "$USERNAME"
},

View File

@ -13,7 +13,8 @@ import (
)
func StartCmdServer(t *testing.T) *testserver.Server {
server := StartServer(t)
server := testserver.New(t)
server.Handle("/", func(r *http.Request) (any, error) {
q := r.URL.Query()
args := strings.Split(q.Get("args"), " ")

View File

@ -29,6 +29,33 @@ type TestConfig struct {
// List of additional replacements to apply on this test.
// Old is a regexp, New is a replacement expression.
Repls []testdiff.Replacement
// List of server stubs to load. Example configuration:
//
// [[Server]]
// Pattern = "POST /api/2.1/jobs/create"
// Response.Body = '''
// {
// "job_id": 1111
// }
// '''
Server []ServerStub
// Record the requests made to the server and write them as output to
// out.requests.txt
RecordRequests bool
}
type ServerStub struct {
// The HTTP method and path to match. Examples:
// 1. /api/2.0/clusters/list (matches all methods)
// 2. GET /api/2.0/clusters/list
Pattern string
// The response body to return.
Response struct {
Body string
}
}
// FindConfig finds the closest config file.

122
acceptance/install_terraform.py Executable file
View File

@ -0,0 +1,122 @@
#!/usr/bin/env python3
"""
Script to set up terraform and databricks terraform provider in a local directory:
- Download terraform.
- Download databricks provider.
- Write a .terraformrc config file that uses this directory.
- The config file contains env vars that need to be set so that databricks CLI uses this terraform and provider.
"""
import os
import platform
import zipfile
import argparse
import json
from pathlib import Path
from urllib.request import urlretrieve
os_name = platform.system().lower()
arch = platform.machine().lower()
arch = {"x86_64": "amd64"}.get(arch, arch)
if os_name == "windows" and arch not in ("386", "amd64"):
# terraform 1.5.5 only has builds for these two.
arch = "amd64"
terraform_version = "1.5.5"
terraform_file = f"terraform_{terraform_version}_{os_name}_{arch}.zip"
terraform_url = f"https://releases.hashicorp.com/terraform/{terraform_version}/{terraform_file}"
terraform_binary = "terraform.exe" if os_name == "windows" else "terraform"
def retrieve(url, path):
if not path.exists():
print(f"Downloading {url} -> {path}")
urlretrieve(url, path)
def read_version(path):
for line in path.open():
if "ProviderVersion" in line:
# Expecting 'const ProviderVersion = "1.64.1"'
items = line.strip().split()
assert len(items) >= 3, items
assert items[-3:-1] == ["ProviderVersion", "="], items
version = items[-1].strip('"')
assert version, items
return version
raise SystemExit(f"Could not find ProviderVersion in {path}")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--targetdir", default="build", type=Path)
parser.add_argument("--provider-version")
args = parser.parse_args()
target = args.targetdir
if not args.provider_version:
version_file = Path(__file__).parent.parent / "bundle/internal/tf/codegen/schema/version.go"
assert version_file.exists(), version_file
terraform_provider_version = read_version(version_file)
print(f"Read version {terraform_provider_version} from {version_file}")
else:
terraform_provider_version = args.provider_version
terraform_provider_file = f"terraform-provider-databricks_{terraform_provider_version}_{os_name}_{arch}.zip"
terraform_provider_url = (
f"https://github.com/databricks/terraform-provider-databricks/releases/download/v{terraform_provider_version}/{terraform_provider_file}"
)
target.mkdir(exist_ok=True, parents=True)
zip_path = target / terraform_file
terraform_path = target / terraform_binary
terraform_provider_path = target / terraform_provider_file
retrieve(terraform_url, zip_path)
retrieve(terraform_provider_url, terraform_provider_path)
if not terraform_path.exists():
print(f"Extracting {zip_path} -> {terraform_path}")
with zipfile.ZipFile(zip_path, "r") as zip_ref:
zip_ref.extractall(target)
terraform_path.chmod(0o755)
tfplugins_path = target / "tfplugins"
provider_dir = Path(tfplugins_path / f"registry.terraform.io/databricks/databricks/{terraform_provider_version}/{os_name}_{arch}")
if not provider_dir.exists():
print(f"Extracting {terraform_provider_path} -> {provider_dir}")
os.makedirs(provider_dir, exist_ok=True)
with zipfile.ZipFile(terraform_provider_path, "r") as zip_ref:
zip_ref.extractall(provider_dir)
files = list(provider_dir.iterdir())
assert files, provider_dir
for f in files:
f.chmod(0o755)
terraformrc_path = target / ".terraformrc"
if not terraformrc_path.exists():
path = json.dumps(str(tfplugins_path.absolute()))
text = f"""# Set these env variables before running databricks cli:
# export DATABRICKS_TF_CLI_CONFIG_FILE={terraformrc_path.absolute()}
# export DATABRICKS_TF_EXEC_PATH={terraform_path.absolute()}
provider_installation {{
filesystem_mirror {{
path = {path}
include = ["registry.terraform.io/databricks/databricks"]
}}
}}
"""
print(f"Writing {terraformrc_path}:\n{text}")
terraformrc_path.write_text(text)
if __name__ == "__main__":
main()

View File

@ -2,7 +2,6 @@ package acceptance_test
import (
"net/http"
"testing"
"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go/service/catalog"
@ -11,14 +10,6 @@ import (
"github.com/databricks/databricks-sdk-go/service/workspace"
)
func StartServer(t *testing.T) *testserver.Server {
server := testserver.New(t)
t.Cleanup(func() {
server.Close()
})
return server
}
func AddHandlers(server *testserver.Server) {
server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, error) {
return compute.ListPoliciesResponse{
@ -63,6 +54,7 @@ func AddHandlers(server *testserver.Server) {
server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, error) {
return iam.User{
Id: "1000012345",
UserName: "tester@databricks.com",
}, nil
})

View File

@ -0,0 +1,25 @@
terraform {
required_providers {
databricks = {
source = "databricks/databricks"
version = "1.64.1"
}
}
required_version = "= 1.5.5"
}
provider "databricks" {
# Optionally, specify the Databricks host and token
# host = "https://<your-databricks-instance>"
# token = "<YOUR_PERSONAL_ACCESS_TOKEN>"
}
data "databricks_current_user" "me" {
# Retrieves the current user's information
}
output "username" {
description = "Username"
value = "${data.databricks_current_user.me.user_name}"
}

View File

@ -0,0 +1,51 @@
>>> $TERRAFORM init -no-color -get=false
Initializing the backend...
Initializing provider plugins...
- Finding databricks/databricks versions matching "1.64.1"...
- Installing databricks/databricks v1.64.1...
- Installed databricks/databricks v1.64.1 (unauthenticated)
Terraform has created a lock file .terraform.lock.hcl to record the provider
selections it made above. Include this file in your version control repository
so that Terraform can guarantee to make the same selections by default when
you run "terraform init" in the future.
Warning: Incomplete lock file information for providers
Due to your customized provider installation methods, Terraform was forced to
calculate lock file checksums locally for the following providers:
- databricks/databricks
To calculate additional checksums for another platform, run:
terraform providers lock -platform=linux_amd64
(where linux_amd64 is the platform to generate)
Terraform has been successfully initialized!
You may now begin working with Terraform. Try running "terraform plan" to see
any changes that are required for your infrastructure. All Terraform commands
should now work.
If you ever set or change modules or backend configuration for Terraform,
rerun this command to reinitialize your working directory. If you forget, other
commands will detect it and remind you to do so if necessary.
>>> $TERRAFORM plan -no-color
data.databricks_current_user.me: Reading...
data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
Changes to Outputs:
+ username = "$USERNAME"
You can apply this plan to save these new output values to the Terraform
state, without changing any real infrastructure.
─────────────────────────────────────────────────────────────────────────────
Note: You didn't use the -out option to save this plan, so Terraform can't
guarantee to take exactly these actions if you run "terraform apply" now.

View File

@ -0,0 +1,14 @@
# Want to filter out these message:
# Mac:
# The current .terraform.lock.hcl file only includes checksums for
# darwin_arm64, so Terraform running on another platform will fail to install
# these providers.
#
# Linux:
# The current .terraform.lock.hcl file only includes checksums for linux_amd64,
# so Terraform running on another platform will fail to install these
# providers.
trace $TERRAFORM init -no-color -get=false | grep -v 'includes checksums for' | grep -v 'so Terraform running on another' | grep -v 'providers\.'
trace $TERRAFORM plan -no-color
rm -fr .terraform.lock.hcl .terraform

View File

@ -0,0 +1 @@
{"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}

View File

@ -0,0 +1,5 @@
>>> $CLI jobs create --json {"name":"abc"}
{
"job_id":1111
}

View File

@ -0,0 +1 @@
trace $CLI jobs create --json '{"name":"abc"}'

View File

@ -0,0 +1,9 @@
RecordRequests = true
[[Server]]
Pattern = "POST /api/2.1/jobs/create"
Response.Body = '''
{
"job_id": 1111
}
'''

View File

@ -2,9 +2,12 @@ package testserver
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"github.com/stretchr/testify/assert"
"github.com/databricks/cli/internal/testutil"
)
@ -13,11 +16,22 @@ type Server struct {
Mux *http.ServeMux
t testutil.TestingT
RecordRequests bool
Requests []Request
}
type Request struct {
Method string `json:"method"`
Path string `json:"path"`
Body any `json:"body"`
}
func New(t testutil.TestingT) *Server {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
t.Cleanup(server.Close)
return &Server{
Server: server,
@ -28,10 +42,6 @@ func New(t testutil.TestingT) *Server {
type HandlerFunc func(req *http.Request) (resp any, err error)
func (s *Server) Close() {
s.Server.Close()
}
func (s *Server) Handle(pattern string, handler HandlerFunc) {
s.Mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
resp, err := handler(r)
@ -40,6 +50,18 @@ func (s *Server) Handle(pattern string, handler HandlerFunc) {
return
}
if s.RecordRequests {
body, err := io.ReadAll(r.Body)
assert.NoError(s.t, err)
s.Requests = append(s.Requests, Request{
Method: r.Method,
Path: r.URL.Path,
Body: json.RawMessage(body),
})
}
w.Header().Set("Content-Type", "application/json")
var respBytes []byte

1
ruff.toml Normal file
View File

@ -0,0 +1 @@
line-length = 150