Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls

This commit is contained in:
Lennart Kats 2024-10-17 10:25:25 +02:00
commit 2765a41cf2
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
6 changed files with 104 additions and 22 deletions

View File

@ -11,6 +11,7 @@
"toolchain": { "toolchain": {
"required": ["go"], "required": ["go"],
"post_generate": [ "post_generate": [
"go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
"go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json", "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
"echo 'go.sum linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes",

View File

@ -0,0 +1,73 @@
package build
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/mod/modfile"
)
// This test ensures that the OpenAPI SHA the CLI is being generated from matches
// the OpenAPI SHA of the Go SDK version used in the CLI. We should always upgrade
// the Go SDK version before generating the CLI because downstream generated assets
// like the bundle schema depend on the Go SDK itself.
func TestConsistentDatabricksSdkVersion(t *testing.T) {
// Read the go.mod file
b, err := os.ReadFile("../../go.mod")
require.NoError(t, err)
// Parse the go.mod file to get the databricks-sdk version
modFile, err := modfile.Parse("../../go.mod", b, nil)
require.NoError(t, err)
modulePath := "github.com/databricks/databricks-sdk-go"
var version string
for _, r := range modFile.Require {
if r.Mod.Path == modulePath {
version = r.Mod.Version
}
}
require.NotEmpty(t, version)
// Full path of the package. For example: github.com/databricks/databricks-sdk-go@v0.47.1-0.20241002195128-6cecc224cbf7
fullPath := fmt.Sprintf("%s@%s", modulePath, version)
type goListResponse struct {
Origin struct {
Hash string
}
}
// Using the go CLI query for the git hash corresponding to the databricks-sdk-go version
cmd := exec.Command("go", "list", "-m", "-json", "-mod=readonly", fullPath)
out, err := cmd.Output()
require.NoError(t, err)
parsedOutput := new(goListResponse)
err = json.Unmarshal(out, parsedOutput)
require.NoError(t, err)
hash := parsedOutput.Origin.Hash
require.NotEmpty(t, hash)
// Read the OpenAPI SHA from the Go SDK.
url := fmt.Sprintf("https://raw.githubusercontent.com/databricks/databricks-sdk-go/%s/.codegen/_openapi_sha", hash)
resp, err := http.Get(url)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
sdkSha, err := io.ReadAll(resp.Body)
require.NoError(t, err)
cliSha, err := os.ReadFile("../../.codegen/_openapi_sha")
require.NoError(t, err)
assert.Equal(t, strings.TrimSpace(string(cliSha)), strings.TrimSpace(string(sdkSha)), "please update the SDK version before generating the CLI")
}

View File

@ -112,8 +112,8 @@ func TestAccFsMkdirWhenFileExistsAtPath(t *testing.T) {
// assert mkdir fails // assert mkdir fails
_, _, err = RequireErrorRun(t, "fs", "mkdir", path.Join(tmpDir, "hello")) _, _, err = RequireErrorRun(t, "fs", "mkdir", path.Join(tmpDir, "hello"))
// Different cloud providers return different errors. // Different cloud providers or cloud configurations return different errors.
regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$`) regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$|(^|: )"The specified path already exists.".*$`)
assert.Regexp(t, regex, err.Error()) assert.Regexp(t, regex, err.Error())
}) })

View File

@ -20,6 +20,7 @@ import (
"time" "time"
"github.com/databricks/cli/cmd/root" "github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/internal/acc"
"github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/cmd" "github.com/databricks/cli/cmd"
@ -591,13 +592,10 @@ func setupWsfsExtensionsFiler(t *testing.T) (filer.Filer, string) {
} }
func setupDbfsFiler(t *testing.T) (filer.Filer, string) { func setupDbfsFiler(t *testing.T) (filer.Filer, string) {
t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) _, wt := acc.WorkspaceTest(t)
w, err := databricks.NewWorkspaceClient() tmpDir := TemporaryDbfsDir(t, wt.W)
require.NoError(t, err) f, err := filer.NewDbfsClient(wt.W, tmpDir)
tmpDir := TemporaryDbfsDir(t, w)
f, err := filer.NewDbfsClient(w, tmpDir)
require.NoError(t, err) require.NoError(t, err)
return f, path.Join("dbfs:/", tmpDir) return f, path.Join("dbfs:/", tmpDir)

View File

@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"reflect"
"github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/convert"
@ -63,12 +64,25 @@ func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics {
return diags.Extend(diag.FromErr(err)) return diags.Extend(diag.FromErr(err))
} }
kind := reflect.ValueOf(v).Kind()
if kind == reflect.Ptr {
kind = reflect.ValueOf(v).Elem().Kind()
}
if kind == reflect.Struct {
// Finally unmarshal the normalized data to the output. // Finally unmarshal the normalized data to the output.
// It will fill in the ForceSendFields field if the struct contains it. // It will fill in the ForceSendFields field if the struct contains it.
err = marshal.Unmarshal(data, v) err = marshal.Unmarshal(data, v)
if err != nil { if err != nil {
return diags.Extend(diag.FromErr(err)) return diags.Extend(diag.FromErr(err))
} }
} else {
// If the output is not a struct, just unmarshal the data to the output.
err = json.Unmarshal(data, v)
if err != nil {
return diags.Extend(diag.FromErr(err))
}
}
return diags return diags
} }

View File

@ -13,10 +13,6 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
type requestType struct {
Foo string `json:"foo"`
}
func TestJsonFlagEmpty(t *testing.T) { func TestJsonFlagEmpty(t *testing.T) {
var body JsonFlag var body JsonFlag
@ -35,13 +31,13 @@ func TestJsonFlagInline(t *testing.T) {
err := body.Set(`{"foo": "bar"}`) err := body.Set(`{"foo": "bar"}`)
assert.NoError(t, err) assert.NoError(t, err)
var request requestType var request any
diags := body.Unmarshal(&request) diags := body.Unmarshal(&request)
assert.NoError(t, diags.Error()) assert.NoError(t, diags.Error())
assert.Empty(t, diags) assert.Empty(t, diags)
assert.Equal(t, "JSON (14 bytes)", body.String()) assert.Equal(t, "JSON (14 bytes)", body.String())
assert.Equal(t, requestType{"bar"}, request) assert.Equal(t, map[string]any{"foo": "bar"}, request)
} }
func TestJsonFlagError(t *testing.T) { func TestJsonFlagError(t *testing.T) {
@ -50,7 +46,7 @@ func TestJsonFlagError(t *testing.T) {
err := body.Set(`{"foo":`) err := body.Set(`{"foo":`)
assert.NoError(t, err) assert.NoError(t, err)
var request requestType var request any
diags := body.Unmarshal(&request) diags := body.Unmarshal(&request)
assert.EqualError(t, diags.Error(), "error decoding JSON at (inline):1:8: unexpected end of JSON input") assert.EqualError(t, diags.Error(), "error decoding JSON at (inline):1:8: unexpected end of JSON input")
assert.Equal(t, "JSON (7 bytes)", body.String()) assert.Equal(t, "JSON (7 bytes)", body.String())
@ -58,7 +54,7 @@ func TestJsonFlagError(t *testing.T) {
func TestJsonFlagFile(t *testing.T) { func TestJsonFlagFile(t *testing.T) {
var body JsonFlag var body JsonFlag
var request requestType var request any
var fpath string var fpath string
var payload = []byte(`{"foo": "bar"}`) var payload = []byte(`{"foo": "bar"}`)
@ -78,7 +74,7 @@ func TestJsonFlagFile(t *testing.T) {
assert.NoError(t, diags.Error()) assert.NoError(t, diags.Error())
assert.Empty(t, diags) assert.Empty(t, diags)
assert.Equal(t, requestType{"bar"}, request) assert.Equal(t, map[string]any{"foo": "bar"}, request)
} }
const jsonData = ` const jsonData = `