Merge remote-tracking branch 'origin' into improve/json-schema

This commit is contained in:
Shreyas Goenka 2024-08-22 17:27:49 +02:00
commit 08133126ae
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
52 changed files with 1356 additions and 233 deletions

View File

@ -1,5 +1,31 @@
# Version changelog
## [Release] Release v0.227.0
CLI:
* Added filtering flags for cluster list commands ([#1703](https://github.com/databricks/cli/pull/1703)).
Bundles:
* Allow users to configure paths (including outside of the bundle root) to synchronize to the workspace. ([#1694](https://github.com/databricks/cli/pull/1694)).
* Add configurable presets for name prefixes, tags, etc. ([#1490](https://github.com/databricks/cli/pull/1490)).
* Add support for requirements libraries in Job Tasks ([#1543](https://github.com/databricks/cli/pull/1543)).
* Remove reference to "dbt" in the default-sql template ([#1696](https://github.com/databricks/cli/pull/1696)).
* Pause continuous pipelines when 'mode: development' is used ([#1590](https://github.com/databricks/cli/pull/1590)).
* Report all empty resources present in error diagnostic ([#1685](https://github.com/databricks/cli/pull/1685)).
* Improves detection of PyPI package names in environment dependencies ([#1699](https://github.com/databricks/cli/pull/1699)).
Internal:
* Add `import` option for PyDABs ([#1693](https://github.com/databricks/cli/pull/1693)).
* Make fileset take optional list of paths to list ([#1684](https://github.com/databricks/cli/pull/1684)).
* Pass through paths argument to libs/sync ([#1689](https://github.com/databricks/cli/pull/1689)).
* Correctly mark package names with versions as remote libraries ([#1697](https://github.com/databricks/cli/pull/1697)).
* Share test initializer in common helper function ([#1695](https://github.com/databricks/cli/pull/1695)).
* Make `pydabs/venv_path` optional ([#1687](https://github.com/databricks/cli/pull/1687)).
* Use API mocks for duplicate path errors in workspace files extensions client ([#1690](https://github.com/databricks/cli/pull/1690)).
* Fix prefix preset used for UC schemas ([#1704](https://github.com/databricks/cli/pull/1704)).
## [Release] Release v0.226.0
CLI:

View File

@ -15,6 +15,8 @@ type infer struct {
func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
artifact := b.Config.Artifacts[m.name]
// TODO use python.DetectVEnvExecutable once bundle has a way to specify venv path
py, err := python.DetectExecutable(ctx)
if err != nil {
return diag.FromErr(err)

View File

@ -39,6 +39,14 @@ type Bundle struct {
// Exclusively use this field for filesystem operations.
BundleRoot vfs.Path
// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
SyncRoot vfs.Path
// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
SyncRootPath string
Config config.Root
// Metadata about the bundle deployment. This is the interface Databricks services

View File

@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path {
return r.b.BundleRoot
}
func (r ReadOnlyBundle) SyncRoot() vfs.Path {
return r.b.SyncRoot
}
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
return r.b.WorkspaceClient()
}

View File

@ -36,8 +36,8 @@ type PyDABs struct {
// VEnvPath is path to the virtual environment.
//
// Required if PyDABs is enabled. PyDABs will load the code in the specified
// environment.
// If enabled, PyDABs will execute code within this environment. If disabled,
// it defaults to using the Python interpreter available in the current shell.
VEnvPath string `json:"venv_path,omitempty"`
// Import contains a list Python packages with PyDABs code.

View File

@ -155,8 +155,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
// Schemas: Prefix
for i := range r.Schemas {
prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_"
r.Schemas[i].Name = prefix + r.Schemas[i].Name
r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name
// HTTP API for schemas doesn't yet support tags. It's only supported in
// the Databricks UI and via the SQL API.
}

View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/require"
)
@ -68,6 +69,62 @@ func TestApplyPresetsPrefix(t *testing.T) {
}
}
func TestApplyPresetsPrefixForUcSchema(t *testing.T) {
tests := []struct {
name string
prefix string
schema *resources.Schema
want string
}{
{
name: "add prefix to schema",
prefix: "[prefix]",
schema: &resources.Schema{
CreateSchema: &catalog.CreateSchema{
Name: "schema1",
},
},
want: "prefix_schema1",
},
{
name: "add empty prefix to schema",
prefix: "",
schema: &resources.Schema{
CreateSchema: &catalog.CreateSchema{
Name: "schema1",
},
},
want: "schema1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Schemas: map[string]*resources.Schema{
"schema1": tt.schema,
},
},
Presets: config.Presets{
NamePrefix: tt.prefix,
},
},
}
ctx := context.Background()
diag := bundle.Apply(ctx, b, mutator.ApplyPresets())
if diag.HasError() {
t.Fatalf("unexpected error: %v", diag)
}
require.Equal(t, tt.want, b.Config.Resources.Schemas["schema1"].Name)
})
}
}
func TestApplyPresetsTags(t *testing.T) {
tests := []struct {
name string

View File

@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string {
}
func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
root := b.BundleRoot.Native()
root := b.SyncRoot.Native()
// The bundle root must be located in /Workspace/
if !strings.HasPrefix(root, "/Workspace/") {
@ -45,6 +45,6 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
return diag.FromErr(err)
}
b.BundleRoot = p
b.SyncRoot = p
return nil
}

View File

@ -7,8 +7,8 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/databricks/cli/libs/python"
"github.com/databricks/databricks-sdk-go/logger"
"github.com/databricks/cli/bundle/env"
@ -86,23 +86,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
return nil
}
if experimental.PyDABs.VEnvPath == "" {
return diag.Errorf("\"experimental.pydabs.enabled\" can only be used when \"experimental.pydabs.venv_path\" is set")
}
// mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics'
var mutateDiags diag.Diagnostics
var mutateDiagsHasError = errors.New("unexpected error")
err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
pythonPath := interpreterPath(experimental.PyDABs.VEnvPath)
pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
if _, err := os.Stat(pythonPath); err != nil {
if os.IsNotExist(err) {
return dyn.InvalidValue, fmt.Errorf("can't find %q, check if venv is created", pythonPath)
} else {
return dyn.InvalidValue, fmt.Errorf("can't find %q: %w", pythonPath, err)
}
if err != nil {
return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
}
cacheDir, err := createCacheDir(ctx)
@ -423,11 +415,16 @@ func isOmitemptyDelete(left dyn.Value) bool {
}
}
// interpreterPath returns platform-specific path to Python interpreter in the virtual environment.
func interpreterPath(venvPath string) string {
if runtime.GOOS == "windows" {
return filepath.Join(venvPath, "Scripts", "python3.exe")
} else {
return filepath.Join(venvPath, "bin", "python3")
// detectExecutable lookups Python interpreter in virtual environment, or if not set, in PATH.
func detectExecutable(ctx context.Context, venvPath string) (string, error) {
if venvPath == "" {
interpreter, err := python.DetectExecutable(ctx)
if err != nil {
return "", err
}
return interpreter, nil
}
return python.DetectVEnvExecutable(venvPath)
}

View File

@ -282,7 +282,7 @@ func TestPythonMutator_venvRequired(t *testing.T) {
}
func TestPythonMutator_venvNotFound(t *testing.T) {
expectedError := fmt.Sprintf("can't find %q, check if venv is created", interpreterPath("bad_path"))
expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path"))
b := loadYaml("databricks.yml", `
experimental:
@ -596,9 +596,7 @@ func loadYaml(name string, content string) *bundle.Bundle {
}
}
func withFakeVEnv(t *testing.T, path string) {
interpreterPath := interpreterPath(path)
func withFakeVEnv(t *testing.T, venvPath string) {
cwd, err := os.Getwd()
if err != nil {
panic(err)
@ -608,6 +606,8 @@ func withFakeVEnv(t *testing.T, path string) {
panic(err)
}
interpreterPath := interpreterPath(venvPath)
err = os.MkdirAll(filepath.Dir(interpreterPath), 0755)
if err != nil {
panic(err)
@ -618,9 +618,22 @@ func withFakeVEnv(t *testing.T, path string) {
panic(err)
}
err = os.WriteFile(filepath.Join(venvPath, "pyvenv.cfg"), []byte(""), 0755)
if err != nil {
panic(err)
}
t.Cleanup(func() {
if err := os.Chdir(cwd); err != nil {
panic(err)
}
})
}
func interpreterPath(venvPath string) string {
if runtime.GOOS == "windows" {
return filepath.Join(venvPath, "Scripts", "python3.exe")
} else {
return filepath.Join(venvPath, "bin", "python3")
}
}

View File

@ -45,6 +45,10 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
if err != nil {
return dyn.InvalidValue, err
}
v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
if err != nil {
return dyn.InvalidValue, err

View File

@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
RootPath: ".",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
"../common",
},
Include: []string{
"foo",
"bar",
@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
},
}
bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])
@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
RootPath: "/tmp/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
"../common",
},
Include: []string{
"foo",
"bar",
@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
},
}
bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])

View File

@ -0,0 +1,48 @@
package mutator
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
)
type syncDefaultPath struct{}
// SyncDefaultPath configures the default sync path to be equal to the bundle root.
func SyncDefaultPath() bundle.Mutator {
return &syncDefaultPath{}
}
func (m *syncDefaultPath) Name() string {
return "SyncDefaultPath"
}
func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
isset := false
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
pv, _ := dyn.Get(v, "sync.paths")
// If the sync paths field is already set, do nothing.
// We know it is set if its value is either a nil or a sequence (empty or not).
switch pv.Kind() {
case dyn.KindNil, dyn.KindSequence:
isset = true
}
return v, nil
})
if err != nil {
return diag.FromErr(err)
}
// If the sync paths field is already set, do nothing.
if isset {
return nil
}
// Set the sync paths to the default value.
b.Config.Sync.Paths = []string{"."}
return nil
}

View File

@ -0,0 +1,82 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath())
require.NoError(t, diags.Error())
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
}
func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
tcases := []struct {
name string
paths dyn.Value
expect []string
}{
{
name: "nil",
paths: dyn.V(nil),
expect: nil,
},
{
name: "empty sequence",
paths: dyn.V([]dyn.Value{}),
expect: []string{},
},
{
name: "non-empty sequence",
paths: dyn.V([]dyn.Value{dyn.V("something")}),
expect: []string{"something"},
},
}
for _, tcase := range tcases {
t.Run(tcase.name, func(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{},
}
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping()))
if err != nil {
return dyn.InvalidValue, err
}
v, err = dyn.Set(v, "sync.paths", tcase.paths)
if err != nil {
return dyn.InvalidValue, err
}
return v, nil
})
return diag.FromErr(err)
})
require.NoError(t, diags.Error())
ctx := context.Background()
diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath())
require.NoError(t, diags.Error())
// If the sync paths field is already set, do nothing.
assert.Equal(t, tcase.expect, b.Config.Sync.Paths)
})
}
}

View File

@ -0,0 +1,120 @@
package mutator
import (
"context"
"fmt"
"path/filepath"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"
)
type syncInferRoot struct{}
// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the
// paths in the sync configuration. The sync root may be different from the bundle root
// when the user intends to synchronize files outside the bundle root.
//
// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant.
// That is, the sync root must contain the bundle root.
//
// This mutator requires all sync-related paths and patterns to be relative to the bundle root path.
// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator.
func SyncInferRoot() bundle.Mutator {
return &syncInferRoot{}
}
func (m *syncInferRoot) Name() string {
return "SyncInferRoot"
}
// computeRoot finds the innermost path that contains the specified path.
// It traverses up the root path until it finds the innermost path.
// If the path does not exist, it returns an empty string.
//
// See "sync_infer_root_internal_test.go" for examples.
func (m *syncInferRoot) computeRoot(path string, root string) string {
for !filepath.IsLocal(path) {
// Break if we have reached the root of the filesystem.
dir := filepath.Dir(root)
if dir == root {
return ""
}
// Update the sync path as we navigate up the directory tree.
path = filepath.Join(filepath.Base(root), path)
// Move up the directory tree.
root = dir
}
return filepath.Clean(root)
}
func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
var diags diag.Diagnostics
// Use the bundle root path as the starting point for inferring the sync root path.
bundleRootPath := filepath.Clean(b.RootPath)
// Infer the sync root path by looking at each one of the sync paths.
// Every sync path must be a descendant of the final sync root path.
syncRootPath := bundleRootPath
for _, path := range b.Config.Sync.Paths {
computedPath := m.computeRoot(path, bundleRootPath)
if computedPath == "" {
continue
}
// Update sync root path if the computed root path is an ancestor of the current sync root path.
if len(computedPath) < len(syncRootPath) {
syncRootPath = computedPath
}
}
// The new sync root path can only be an ancestor of the previous root path.
// Compute the relative path from the sync root to the bundle root.
rel, err := filepath.Rel(syncRootPath, bundleRootPath)
if err != nil {
return diag.FromErr(err)
}
// If during computation of the sync root path we hit the root of the filesystem,
// then one or more of the sync paths are outside the filesystem.
// Check if this happened by verifying that none of the paths escape the root
// when joined with the sync root path.
for i, path := range b.Config.Sync.Paths {
if filepath.IsLocal(filepath.Join(rel, path)) {
continue
}
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("invalid sync path %q", path),
Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)),
Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))},
})
}
if diags.HasError() {
return diags
}
// Update all paths in the sync configuration to be relative to the sync root.
for i, p := range b.Config.Sync.Paths {
b.Config.Sync.Paths[i] = filepath.Join(rel, p)
}
for i, p := range b.Config.Sync.Include {
b.Config.Sync.Include[i] = filepath.Join(rel, p)
}
for i, p := range b.Config.Sync.Exclude {
b.Config.Sync.Exclude[i] = filepath.Join(rel, p)
}
// Configure the sync root path.
b.SyncRoot = vfs.MustNew(syncRootPath)
b.SyncRootPath = syncRootPath
return nil
}

View File

@ -0,0 +1,72 @@
package mutator
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSyncInferRootInternal_ComputeRoot(t *testing.T) {
s := syncInferRoot{}
tcases := []struct {
path string
root string
out string
}{
{
// Test that "." doesn't change the root.
path: ".",
root: "/tmp/some/dir",
out: "/tmp/some/dir",
},
{
// Test that a subdirectory doesn't change the root.
path: "sub",
root: "/tmp/some/dir",
out: "/tmp/some/dir",
},
{
// Test that a parent directory changes the root.
path: "../common",
root: "/tmp/some/dir",
out: "/tmp/some",
},
{
// Test that a deeply nested parent directory changes the root.
path: "../../../../../../common",
root: "/tmp/some/dir/that/is/very/deeply/nested",
out: "/tmp/some",
},
{
// Test that a parent directory changes the root at the filesystem root boundary.
path: "../common",
root: "/tmp",
out: "/",
},
{
// Test that an invalid parent directory doesn't change the root and returns an empty string.
path: "../common",
root: "/",
out: "",
},
{
// Test that the returned path is cleaned even if the root doesn't change.
path: "sub",
root: "/tmp/some/../dir",
out: "/tmp/dir",
},
{
// Test that a relative root path also works.
path: "../common",
root: "foo/bar",
out: "foo",
},
}
for _, tc := range tcases {
out := s.computeRoot(tc.path, tc.root)
assert.Equal(t, tc.out, filepath.ToSlash(out))
}
}

View File

@ -0,0 +1,198 @@
package mutator_test
import (
"context"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some/dir"), b.SyncRootPath)
// Check that the paths are unchanged.
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_NominalRelative(t *testing.T) {
b := &bundle.Bundle{
RootPath: "./some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("some/dir"), b.SyncRootPath)
// Check that the paths are unchanged.
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_ParentDirectory(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../common",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
assert.Equal(t, []string{filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/bar")}, b.Config.Sync.Include)
assert.Equal(t, []string{filepath.FromSlash("dir/baz"), filepath.FromSlash("dir/qux")}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir/that/is/very/deeply/nested",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../../../../../../common",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
assert.Equal(t, []string{
filepath.FromSlash("dir/that/is/very/deeply/nested/foo"),
filepath.FromSlash("dir/that/is/very/deeply/nested/bar"),
}, b.Config.Sync.Include)
assert.Equal(t, []string{
filepath.FromSlash("dir/that/is/very/deeply/nested/baz"),
filepath.FromSlash("dir/that/is/very/deeply/nested/qux"),
}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_MultiplePaths(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/bundle/root",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"./foo",
"../common",
"./bar",
"../../baz",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, filepath.FromSlash("bundle/root/foo"), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.FromSlash("bundle/common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.FromSlash("bundle/root/bar"), b.Config.Sync.Paths[2])
assert.Equal(t, filepath.FromSlash("baz"), b.Config.Sync.Paths[3])
}
func TestSyncInferRoot_Error(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../../../../error",
"../../../thisworks",
"../../../../../error",
},
},
},
}
bundletest.SetLocation(b, "sync.paths", "databricks.yml")
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
require.Len(t, diags, 2)
assert.Equal(t, `invalid sync path "../../../../error"`, diags[0].Summary)
assert.Equal(t, "databricks.yml:0:0", diags[0].Locations[0].String())
assert.Equal(t, "sync.paths[0]", diags[0].Paths[0].String())
assert.Equal(t, `invalid sync path "../../../../../error"`, diags[1].Summary)
assert.Equal(t, "databricks.yml:0:0", diags[1].Locations[0].String())
assert.Equal(t, "sync.paths[2]", diags[1].Paths[0].String())
}

View File

@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund
return err
}
internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
internalDirRel, err := filepath.Rel(b.SyncRootPath, internalDir)
if err != nil {
return err
}

View File

@ -56,8 +56,12 @@ func TestGenerateTrampoline(t *testing.T) {
}
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Bundle: config.Bundle{
Target: "development",
},
@ -89,6 +93,6 @@ func TestGenerateTrampoline(t *testing.T) {
require.Equal(t, "Hello from Trampoline", string(bytes))
task := b.Config.Resources.Jobs["test"].Tasks[0]
require.Equal(t, task.NotebookTask.NotebookPath, ".databricks/bundle/development/.internal/notebook_test_to_trampoline")
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_test_to_trampoline", task.NotebookTask.NotebookPath)
require.Nil(t, task.PythonWheelTask)
}

View File

@ -93,14 +93,14 @@ func (t *translateContext) rewritePath(
return nil
}
// Local path must be contained in the bundle root.
// Local path must be contained in the sync root.
// If it isn't, it won't be synchronized into the workspace.
localRelPath, err := filepath.Rel(t.b.RootPath, localPath)
localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath)
if err != nil {
return err
}
if strings.HasPrefix(localRelPath, "..") {
return fmt.Errorf("path %s is not contained in bundle root path", localPath)
return fmt.Errorf("path %s is not contained in sync root path", localPath)
}
// Prefix remote path with its remote root path.
@ -118,7 +118,7 @@ func (t *translateContext) rewritePath(
}
func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("notebook %s not found", literal)
}
@ -134,7 +134,7 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe
}
func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("file %s not found", literal)
}
@ -148,7 +148,7 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat
}
func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
info, err := t.b.BundleRoot.Stat(filepath.ToSlash(localRelPath))
info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
if err != nil {
return "", err
}

View File

@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern
t.translateNoOp,
noSkipRewrite,
},
{
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
t.translateFilePath,
noSkipRewrite,
},
}
}

View File

@ -41,8 +41,8 @@ func touchEmptyFile(t *testing.T, path string) {
func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -110,10 +110,11 @@ func TestTranslatePaths(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -140,6 +141,9 @@ func TestTranslatePaths(t *testing.T) {
NotebookTask: &jobs.NotebookTask{
NotebookPath: "./my_job_notebook.py",
},
Libraries: []compute.Library{
{Requirements: "./requirements.txt"},
},
},
{
PythonWheelTask: &jobs.PythonWheelTask{
@ -232,6 +236,11 @@ func TestTranslatePaths(t *testing.T) {
"/bundle/my_job_notebook",
b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath,
)
assert.Equal(
t,
"/bundle/requirements.txt",
b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
)
assert.Equal(
t,
"/bundle/my_python_file.py",
@ -280,8 +289,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -371,12 +380,12 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
)
}
func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
func TestTranslatePathsOutsideSyncRoot(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -402,15 +411,15 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml"))
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
assert.ErrorContains(t, diags.Error(), "is not contained in bundle root")
assert.ErrorContains(t, diags.Error(), "is not contained in sync root path")
}
func TestJobNotebookDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -440,8 +449,8 @@ func TestJobFileDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -471,8 +480,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
@ -502,8 +511,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
@ -534,8 +543,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -569,8 +578,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -604,8 +613,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -639,8 +648,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -675,8 +684,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "env2.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -715,8 +724,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
func TestTranslatePathWithComplexVariables(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Variables: map[string]*variable.Variable{
"cluster_libraries": {

View File

@ -1,6 +1,10 @@
package config
type Sync struct {
// Paths contains a list of paths to synchronize relative to the bundle root path.
// If not configured, this defaults to synchronizing everything in the bundle root path (i.e. `.`).
Paths []string `json:"paths,omitempty"`
// Include contains a list of globs evaluated relative to the bundle root path
// to explicitly include files that were excluded by the user's gitignore.
Include []string `json:"include,omitempty"`

View File

@ -28,8 +28,8 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp
}
opts := &sync.SyncOptions{
LocalRoot: rb.BundleRoot(),
Paths: []string{"."},
LocalRoot: rb.SyncRoot(),
Paths: rb.Config().Sync.Paths,
Include: includes,
Exclude: rb.Config().Sync.Exclude,

View File

@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
}
log.Infof(ctx, "Creating new snapshot")
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.BundleRoot), opts)
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.SyncRoot), opts)
if err != nil {
return diag.FromErr(err)
}

View File

@ -64,6 +64,10 @@ func testStatePull(t *testing.T, opts statePullOpts) {
b := &bundle.Bundle{
RootPath: tmpDir,
BundleRoot: vfs.MustNew(tmpDir),
SyncRootPath: tmpDir,
SyncRoot: vfs.MustNew(tmpDir),
Config: config.Root{
Bundle: config.Bundle{
Target: "default",
@ -81,11 +85,11 @@ func testStatePull(t *testing.T, opts statePullOpts) {
ctx := context.Background()
for _, file := range opts.localFiles {
testutil.Touch(t, b.RootPath, "bar", file)
testutil.Touch(t, b.SyncRootPath, "bar", file)
}
for _, file := range opts.localNotebooks {
testutil.TouchNotebook(t, b.RootPath, "bar", file)
testutil.TouchNotebook(t, b.SyncRootPath, "bar", file)
}
if opts.withExistingSnapshot {

View File

@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string {
if library.Egg != "" {
return library.Egg
}
if library.Requirements != "" {
return library.Requirements
}
return ""
}

View File

@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) {
assert.Equal(t, path, libraryPath(&compute.Library{Whl: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Jar: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Egg: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path}))
assert.Equal(t, "", libraryPath(&compute.Library{}))
}

View File

@ -3,6 +3,7 @@ package libraries
import (
"net/url"
"path"
"regexp"
"strings"
)
@ -65,9 +66,27 @@ func IsLibraryLocal(dep string) bool {
return IsLocalPath(dep)
}
// ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_).
// \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security].
// ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?)?: Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1).
// Spec for package name and version specifier: https://pip.pypa.io/en/stable/reference/requirement-specifiers/
var packageRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+\s?(\[.*\])?\s?((==|!=|<=|>=|~=|==|>|<)\s?\d+(\.\d+){0,2}(\.\*)?)?$`)
func isPackage(name string) bool {
// If the dependency has no extension, it's a PyPi package name
return path.Ext(name) == ""
if packageRegex.MatchString(name) {
return true
}
return isUrlBasedLookup(name)
}
func isUrlBasedLookup(name string) bool {
parts := strings.Split(name, " @ ")
if len(parts) != 2 {
return false
}
return packageRegex.MatchString(parts[0]) && isRemoteStorageScheme(parts[1])
}
func isRemoteStorageScheme(path string) bool {

View File

@ -54,6 +54,16 @@ func TestIsLibraryLocal(t *testing.T) {
{path: "-r /Workspace/my_project/requirements.txt", expected: false},
{path: "s3://mybucket/path/to/package", expected: false},
{path: "dbfs:/mnt/path/to/package", expected: false},
{path: "beautifulsoup4", expected: false},
{path: "beautifulsoup4==4.12.3", expected: false},
{path: "beautifulsoup4 >= 4.12.3", expected: false},
{path: "beautifulsoup4 < 4.12.3", expected: false},
{path: "beautifulsoup4 ~= 4.12.3", expected: false},
{path: "beautifulsoup4[security, tests]", expected: false},
{path: "beautifulsoup4[security, tests] ~= 4.12.3", expected: false},
{path: "https://github.com/pypa/pip/archive/22.0.2.zip", expected: false},
{path: "pip @ https://github.com/pypa/pip/archive/22.0.2.zip", expected: false},
{path: "requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip", expected: false},
}
for i, tc := range testCases {

View File

@ -21,7 +21,18 @@ func Initialize() bundle.Mutator {
"initialize",
[]bundle.Mutator{
validate.AllResourcesHaveValues(),
// Update all path fields in the sync block to be relative to the bundle root path.
mutator.RewriteSyncPaths(),
// Configure the default sync path to equal the bundle root if not explicitly configured.
// By default, this means all files in the bundle root directory are synchronized.
mutator.SyncDefaultPath(),
// Figure out if the sync root path is identical or an ancestor of the bundle root path.
// If it is an ancestor, this updates all paths to be relative to the sync root path.
mutator.SyncInferRoot(),
mutator.MergeJobClusters(),
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),

View File

@ -2,7 +2,6 @@ package python
import (
"context"
"path"
"path/filepath"
"testing"
@ -18,11 +17,15 @@ func TestNoTransformByDefault(t *testing.T) {
tmpDir := t.TempDir()
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Bundle: config.Bundle{
Target: "development",
},
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
@ -63,11 +66,15 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
tmpDir := t.TempDir()
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Bundle: config.Bundle{
Target: "development",
},
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
@ -102,14 +109,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
task := b.Config.Resources.Jobs["job1"].Tasks[0]
require.Nil(t, task.PythonWheelTask)
require.NotNil(t, task.NotebookTask)
dir, err := b.InternalDir(context.Background())
require.NoError(t, err)
internalDirRel, err := filepath.Rel(b.RootPath, dir)
require.NoError(t, err)
require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath)
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_job1_key1", task.NotebookTask.NotebookPath)
require.Len(t, task.Libraries, 1)
require.Equal(t, "/Workspace/Users/test@test.com/bundle/dist/test.jar", task.Libraries[0].Jar)

View File

@ -8,6 +8,10 @@ import (
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/phases"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
@ -36,6 +40,8 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
diags := bundle.Apply(ctx, b, bundle.Seq(
phases.LoadNamedTarget(env),
mutator.RewriteSyncPaths(),
mutator.SyncDefaultPath(),
mutator.SyncInferRoot(),
mutator.MergeJobClusters(),
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),
@ -43,3 +49,28 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
))
return b, diags
}
func configureMock(t *testing.T, b *bundle.Bundle) {
// Configure mock workspace client
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &config.Config{
Host: "https://mock.databricks.workspace.com",
}
m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{
UserName: "user@domain.com",
}, nil)
b.SetWorkpaceClient(m.WorkspaceClient)
}
func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diagnostics) {
b := load(t, path)
configureMock(t, b)
ctx := context.Background()
diags := bundle.Apply(ctx, b, bundle.Seq(
mutator.SelectTarget(env),
phases.Initialize(),
))
return b, diags
}

View File

@ -1,33 +1,13 @@
package config_tests
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/phases"
"github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestExpandPipelineGlobPaths(t *testing.T) {
b := loadTarget(t, "./pipeline_glob_paths", "default")
// Configure mock workspace client
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &config.Config{
Host: "https://mock.databricks.workspace.com",
}
m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{
UserName: "user@domain.com",
}, nil)
b.SetWorkpaceClient(m.WorkspaceClient)
ctx := context.Background()
diags := bundle.Apply(ctx, b, phases.Initialize())
b, diags := initializeTarget(t, "./pipeline_glob_paths", "default")
require.NoError(t, diags.Error())
require.Equal(
t,
@ -37,19 +17,6 @@ func TestExpandPipelineGlobPaths(t *testing.T) {
}
func TestExpandPipelineGlobPathsWithNonExistent(t *testing.T) {
b := loadTarget(t, "./pipeline_glob_paths", "error")
// Configure mock workspace client
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &config.Config{
Host: "https://mock.databricks.workspace.com",
}
m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{
UserName: "user@domain.com",
}, nil)
b.SetWorkpaceClient(m.WorkspaceClient)
ctx := context.Background()
diags := bundle.Apply(ctx, b, phases.Initialize())
_, diags := initializeTarget(t, "./pipeline_glob_paths", "error")
require.ErrorContains(t, diags.Error(), "notebook ./non-existent not found")
}

View File

@ -1,36 +1,14 @@
package config_tests
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/phases"
"github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func configureMock(t *testing.T, b *bundle.Bundle) {
// Configure mock workspace client
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &config.Config{
Host: "https://mock.databricks.workspace.com",
}
m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{
UserName: "user@domain.com",
}, nil)
b.SetWorkpaceClient(m.WorkspaceClient)
}
func TestRelativePathTranslationDefault(t *testing.T) {
b := loadTarget(t, "./relative_path_translation", "default")
configureMock(t, b)
diags := bundle.Apply(context.Background(), b, phases.Initialize())
b, diags := initializeTarget(t, "./relative_path_translation", "default")
require.NoError(t, diags.Error())
t0 := b.Config.Resources.Jobs["job"].Tasks[0]
@ -40,10 +18,7 @@ func TestRelativePathTranslationDefault(t *testing.T) {
}
func TestRelativePathTranslationOverride(t *testing.T) {
b := loadTarget(t, "./relative_path_translation", "override")
configureMock(t, b)
diags := bundle.Apply(context.Background(), b, phases.Initialize())
b, diags := initializeTarget(t, "./relative_path_translation", "override")
require.NoError(t, diags.Error())
t0 := b.Config.Resources.Jobs["job"].Tasks[0]

View File

@ -0,0 +1,20 @@
bundle:
name: sync_paths
workspace:
host: https://acme.cloud.databricks.com/
sync:
paths:
- src
targets:
development:
sync:
paths:
- development
staging:
sync:
paths:
- staging

View File

@ -0,0 +1,26 @@
bundle:
name: sync_paths
workspace:
host: https://acme.cloud.databricks.com/
targets:
development:
sync:
paths:
- development
staging:
sync:
paths:
- staging
undefined: ~
nil:
sync:
paths: ~
empty:
sync:
paths: []

View File

@ -0,0 +1,10 @@
bundle:
name: shared_code
workspace:
host: https://acme.cloud.databricks.com/
sync:
paths:
- "../common"
- "."

View File

@ -0,0 +1 @@
Placeholder for files to be deployed as part of multiple bundles.

View File

@ -12,14 +12,20 @@ func TestSyncOverride(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/override", "development")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override", "staging")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override", "prod")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
}
@ -28,14 +34,20 @@ func TestSyncOverrideNoRootSync(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/override_no_root", "development")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override_no_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override_no_root", "prod")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
}
@ -44,10 +56,14 @@ func TestSyncNil(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/nil", "development")
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Nil(t, b.Config.Sync.Include)
assert.Nil(t, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/nil", "staging")
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
}
@ -56,10 +72,59 @@ func TestSyncNilRoot(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/nil_root", "development")
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Nil(t, b.Config.Sync.Include)
assert.Nil(t, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/nil_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
}
func TestSyncPaths(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/paths", "development")
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
assert.Equal(t, []string{"src", "development"}, b.Config.Sync.Paths)
b = loadTarget(t, "./sync/paths", "staging")
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
assert.Equal(t, []string{"src", "staging"}, b.Config.Sync.Paths)
}
func TestSyncPathsNoRoot(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/paths_no_root", "development")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"development"}, b.Config.Sync.Paths)
b = loadTarget(t, "./sync/paths_no_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"staging"}, b.Config.Sync.Paths)
// If not set at all, it defaults to "."
b = loadTarget(t, "./sync/paths_no_root", "undefined")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
// If set to nil, it won't sync anything.
b = loadTarget(t, "./sync/paths_no_root", "nil")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Len(t, b.Config.Sync.Paths, 0)
// If set to an empty sequence, it won't sync anything.
b = loadTarget(t, "./sync/paths_no_root", "empty")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Len(t, b.Config.Sync.Paths, 0)
}
func TestSyncSharedCode(t *testing.T) {
b := loadTarget(t, "./sync/shared_code/bundle", "default")
assert.Equal(t, filepath.FromSlash("sync/shared_code"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"common", "bundle"}, b.Config.Sync.Paths)
}

View File

@ -19,6 +19,8 @@ func TestSyncOptionsFromBundle(t *testing.T) {
b := &bundle.Bundle{
RootPath: tempDir,
BundleRoot: vfs.MustNew(tempDir),
SyncRootPath: tempDir,
SyncRoot: vfs.MustNew(tempDir),
Config: config.Root{
Bundle: config.Bundle{
Target: "default",

View File

@ -1,17 +1,83 @@
package clusters
import (
"strings"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/spf13/cobra"
)
func listOverride(listCmd *cobra.Command, _ *compute.ListClustersRequest) {
// Below we add overrides for filter flags for cluster list command to allow for custom filtering
// Auto generating such flags is not yet supported by the CLI generator
func listOverride(listCmd *cobra.Command, listReq *compute.ListClustersRequest) {
listCmd.Annotations["headerTemplate"] = cmdio.Heredoc(`
{{header "ID"}} {{header "Name"}} {{header "State"}}`)
listCmd.Annotations["template"] = cmdio.Heredoc(`
{{range .}}{{.ClusterId | green}} {{.ClusterName | cyan}} {{if eq .State "RUNNING"}}{{green "%s" .State}}{{else if eq .State "TERMINATED"}}{{red "%s" .State}}{{else}}{{blue "%s" .State}}{{end}}
{{end}}`)
listReq.FilterBy = &compute.ListClustersFilterBy{}
listCmd.Flags().BoolVar(&listReq.FilterBy.IsPinned, "is-pinned", false, "Filter clusters by pinned status")
listCmd.Flags().StringVar(&listReq.FilterBy.PolicyId, "policy-id", "", "Filter clusters by policy id")
sources := &clusterSources{source: &listReq.FilterBy.ClusterSources}
listCmd.Flags().Var(sources, "cluster-sources", "Filter clusters by source")
states := &clusterStates{state: &listReq.FilterBy.ClusterStates}
listCmd.Flags().Var(states, "cluster-states", "Filter clusters by states")
}
type clusterSources struct {
source *[]compute.ClusterSource
}
func (c *clusterSources) String() string {
s := make([]string, len(*c.source))
for i, source := range *c.source {
s[i] = string(source)
}
return strings.Join(s, ",")
}
func (c *clusterSources) Set(value string) error {
splits := strings.Split(value, ",")
for _, split := range splits {
*c.source = append(*c.source, compute.ClusterSource(split))
}
return nil
}
func (c *clusterSources) Type() string {
return "[]string"
}
type clusterStates struct {
state *[]compute.State
}
func (c *clusterStates) String() string {
s := make([]string, len(*c.state))
for i, source := range *c.state {
s[i] = string(source)
}
return strings.Join(s, ",")
}
func (c *clusterStates) Set(value string) error {
splits := strings.Split(value, ",")
for _, split := range splits {
*c.state = append(*c.state, compute.State(split))
}
return nil
}
func (c *clusterStates) Type() string {
return "[]string"
}
func listNodeTypesOverride(listNodeTypesCmd *cobra.Command) {

3
go.mod
View File

@ -10,7 +10,7 @@ require (
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/google/uuid v1.6.0 // BSD-3-Clause
github.com/hashicorp/go-version v1.7.0 // MPL 2.0
github.com/hashicorp/hc-install v0.8.0 // MPL 2.0
github.com/hashicorp/hc-install v0.7.0 // MPL 2.0
github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
@ -49,7 +49,6 @@ require (
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect

8
go.sum generated
View File

@ -99,14 +99,10 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw
github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/hc-install v0.8.0 h1:LdpZeXkZYMQhoKPCecJHlKvUkQFixN/nvyR1CdfOLjI=
github.com/hashicorp/hc-install v0.8.0/go.mod h1:+MwJYjDfCruSD/udvBmRB22Nlkwwkwf5sAB6uTIhSaU=
github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk=
github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA=
github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=

View File

@ -13,6 +13,7 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/internal"
"github.com/databricks/cli/internal/acc"
"github.com/databricks/cli/libs/env"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/catalog"
@ -123,3 +124,33 @@ func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) {
assert.EqualError(t, err, root.ErrAlreadyPrinted.Error())
assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed")
}
func TestAccDeployBasicBundleLogs(t *testing.T) {
ctx, wt := acc.WorkspaceTest(t)
nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
uniqueId := uuid.New().String()
root, err := initTestTemplate(t, ctx, "basic", map[string]any{
"unique_id": uniqueId,
"node_type_id": nodeTypeId,
"spark_version": defaultSparkVersion,
})
require.NoError(t, err)
t.Cleanup(func() {
err = destroyBundle(t, ctx, root)
require.NoError(t, err)
})
currentUser, err := wt.W.CurrentUser.Me(ctx)
require.NoError(t, err)
stdout, stderr := blackBoxRun(t, root, "bundle", "deploy")
assert.Equal(t, strings.Join([]string{
fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId),
"Deploying resources...",
"Updating deployment state...",
"Deployment complete!\n",
}, "\n"), stderr)
assert.Equal(t, "", stdout)
}

View File

@ -1,10 +1,12 @@
package bundle
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
@ -15,6 +17,7 @@ import (
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/template"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go"
"github.com/stretchr/testify/require"
)
@ -114,3 +117,29 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique
root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId)
return root
}
func blackBoxRun(t *testing.T, root string, args ...string) (stdout string, stderr string) {
cwd := vfs.MustNew(".")
gitRoot, err := vfs.FindLeafInTree(cwd, ".git")
require.NoError(t, err)
t.Setenv("BUNDLE_ROOT", root)
// Create the command
cmd := exec.Command("go", append([]string{"run", "main.go"}, args...)...)
cmd.Dir = gitRoot.Native()
// Create buffers to capture output
var outBuffer, errBuffer bytes.Buffer
cmd.Stdout = &outBuffer
cmd.Stderr = &errBuffer
// Run the command
err = cmd.Run()
require.NoError(t, err)
// Get the output
stdout = outBuffer.String()
stderr = errBuffer.String()
return
}

View File

@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
@ -722,67 +721,6 @@ func TestAccFilerWorkspaceFilesExtensionsStat(t *testing.T) {
assert.ErrorIs(t, err, fs.ErrNotExist)
}
func TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) {
t.Parallel()
tcases := []struct {
files []struct{ name, content string }
name string
}{
{
name: "python",
files: []struct{ name, content string }{
{"foo.py", "print('foo')"},
{"foo.py", "# Databricks notebook source\nprint('foo')"},
},
},
{
name: "r",
files: []struct{ name, content string }{
{"foo.r", "print('foo')"},
{"foo.r", "# Databricks notebook source\nprint('foo')"},
},
},
{
name: "sql",
files: []struct{ name, content string }{
{"foo.sql", "SELECT 'foo'"},
{"foo.sql", "-- Databricks notebook source\nSELECT 'foo'"},
},
},
{
name: "scala",
files: []struct{ name, content string }{
{"foo.scala", "println('foo')"},
{"foo.scala", "// Databricks notebook source\nprintln('foo')"},
},
},
// We don't need to test this for ipynb notebooks. The import API
// fails when the file extension is .ipynb but the content is not a
// valid juptyer notebook.
}
for i := range tcases {
tc := tcases[i]
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
wf, tmpDir := setupWsfsExtensionsFiler(t)
for _, f := range tc.files {
err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories)
require.NoError(t, err)
}
_, err := wf.ReadDir(ctx, ".")
assert.ErrorAs(t, err, &filer.DuplicatePathError{})
assert.ErrorContains(t, err, fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at %s and FILE at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", path.Join(tmpDir, "foo"), path.Join(tmpDir, tc.files[0].name), tc.files[0].name))
})
}
}
func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) {
t.Parallel()

View File

@ -102,13 +102,21 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) {
return marshal.Marshal(info)
}
// Interface for *client.DatabricksClient from the Databricks Go SDK. Abstracted
// as an interface to allow for mocking in tests.
type apiClient interface {
Do(ctx context.Context, method, path string,
headers map[string]string, request, response any,
visitors ...func(*http.Request) error) error
}
// WorkspaceFilesClient implements the files-in-workspace API.
// NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled.
// It can access any workspace path if files-in-workspace is enabled.
type WorkspaceFilesClient struct {
type workspaceFilesClient struct {
workspaceClient *databricks.WorkspaceClient
apiClient *client.DatabricksClient
apiClient apiClient
// File operations will be relative to this path.
root WorkspaceRootPath
@ -120,7 +128,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer,
return nil, err
}
return &WorkspaceFilesClient{
return &workspaceFilesClient{
workspaceClient: w,
apiClient: apiClient,
@ -128,7 +136,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer,
}, nil
}
func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error {
func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error {
absPath, err := w.root.Join(name)
if err != nil {
return err
@ -198,7 +206,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io
return err
}
func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
func (w *workspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err
@ -222,7 +230,7 @@ func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCl
return w.workspaceClient.Workspace.Download(ctx, absPath)
}
func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error {
func (w *workspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error {
absPath, err := w.root.Join(name)
if err != nil {
return err
@ -266,7 +274,7 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ...
return err
}
func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err
@ -299,7 +307,7 @@ func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D
return wsfsDirEntriesFromObjectInfos(objects), nil
}
func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error {
func (w *workspaceFilesClient) Mkdir(ctx context.Context, name string) error {
dirPath, err := w.root.Join(name)
if err != nil {
return err
@ -309,7 +317,7 @@ func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error {
})
}
func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
func (w *workspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err

View File

@ -133,14 +133,14 @@ func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx con
}, nil
}
type DuplicatePathError struct {
type duplicatePathError struct {
oi1 workspace.ObjectInfo
oi2 workspace.ObjectInfo
commonName string
}
func (e DuplicatePathError) Error() string {
func (e duplicatePathError) Error() string {
return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName)
}
@ -157,7 +157,7 @@ func (e ReadOnlyError) Error() string {
// delete, and stat notebooks (and files in general) in the workspace, using their paths
// with the extension included.
//
// The ReadDir method returns a DuplicatePathError if this traditional file system view is
// The ReadDir method returns a duplicatePathError if this traditional file system view is
// not possible. For example, a Python notebook called foo and a Python file called `foo.py`
// would resolve to the same path `foo.py` in a tradition file system.
//
@ -220,7 +220,7 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin
// Error if we have seen this path before in the current directory.
// If not seen before, add it to the seen paths.
if _, ok := seenPaths[entries[i].Name()]; ok {
return nil, DuplicatePathError{
return nil, duplicatePathError{
oi1: seenPaths[entries[i].Name()],
oi2: sysInfo,
commonName: path.Join(name, entries[i].Name()),

View File

@ -0,0 +1,151 @@
package filer
import (
"context"
"net/http"
"testing"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
// Mocks client.DatabricksClient from the databricks-sdk-go package.
type mockApiClient struct {
mock.Mock
}
func (m *mockApiClient) Do(ctx context.Context, method, path string,
headers map[string]string, request any, response any,
visitors ...func(*http.Request) error) error {
args := m.Called(ctx, method, path, headers, request, response, visitors)
// Set the http response from a value provided in the mock call.
p := response.(*wsfsFileInfo)
*p = args.Get(1).(wsfsFileInfo)
return args.Error(0)
}
func TestFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) {
for _, tc := range []struct {
name string
language workspace.Language
notebookExportFormat workspace.ExportFormat
notebookPath string
filePath string
expectedError string
}{
{
name: "python source notebook and file",
language: workspace.LanguagePython,
notebookExportFormat: workspace.ExportFormatSource,
notebookPath: "/dir/foo",
filePath: "/dir/foo.py",
expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.py resolve to the same name /foo.py. Changing the name of one of these objects will resolve this issue",
},
{
name: "python jupyter notebook and file",
language: workspace.LanguagePython,
notebookExportFormat: workspace.ExportFormatJupyter,
notebookPath: "/dir/foo",
filePath: "/dir/foo.py",
// Jupyter notebooks would correspond to foo.ipynb so an error is not expected.
expectedError: "",
},
{
name: "scala source notebook and file",
language: workspace.LanguageScala,
notebookExportFormat: workspace.ExportFormatSource,
notebookPath: "/dir/foo",
filePath: "/dir/foo.scala",
expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.scala resolve to the same name /foo.scala. Changing the name of one of these objects will resolve this issue",
},
{
name: "r source notebook and file",
language: workspace.LanguageR,
notebookExportFormat: workspace.ExportFormatSource,
notebookPath: "/dir/foo",
filePath: "/dir/foo.r",
expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.r resolve to the same name /foo.r. Changing the name of one of these objects will resolve this issue",
},
{
name: "sql source notebook and file",
language: workspace.LanguageSql,
notebookExportFormat: workspace.ExportFormatSource,
notebookPath: "/dir/foo",
filePath: "/dir/foo.sql",
expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.sql resolve to the same name /foo.sql. Changing the name of one of these objects will resolve this issue",
},
{
name: "python jupyter notebook and file",
language: workspace.LanguagePython,
notebookExportFormat: workspace.ExportFormatJupyter,
notebookPath: "/dir/foo",
filePath: "/dir/foo.ipynb",
expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.ipynb resolve to the same name /foo.ipynb. Changing the name of one of these objects will resolve this issue",
},
} {
t.Run(tc.name, func(t *testing.T) {
mockedWorkspaceClient := mocks.NewMockWorkspaceClient(t)
mockedApiClient := mockApiClient{}
// Mock the workspace API's ListAll method.
workspaceApi := mockedWorkspaceClient.GetMockWorkspaceAPI()
workspaceApi.EXPECT().ListAll(mock.Anything, workspace.ListWorkspaceRequest{
Path: "/dir",
}).Return([]workspace.ObjectInfo{
{
Path: tc.filePath,
Language: tc.language,
ObjectType: workspace.ObjectTypeFile,
},
{
Path: tc.notebookPath,
Language: tc.language,
ObjectType: workspace.ObjectTypeNotebook,
},
}, nil)
// Mock bespoke API calls to /api/2.0/workspace/get-status, that are
// used to figure out the right file extension for the notebook.
statNotebook := wsfsFileInfo{
ObjectInfo: workspace.ObjectInfo{
Path: tc.notebookPath,
Language: tc.language,
ObjectType: workspace.ObjectTypeNotebook,
},
ReposExportFormat: tc.notebookExportFormat,
}
mockedApiClient.On("Do", mock.Anything, http.MethodGet, "/api/2.0/workspace/get-status", map[string]string(nil), map[string]string{
"path": tc.notebookPath,
"return_export_info": "true",
}, mock.AnythingOfType("*filer.wsfsFileInfo"), []func(*http.Request) error(nil)).Return(nil, statNotebook)
workspaceFilesClient := workspaceFilesClient{
workspaceClient: mockedWorkspaceClient.WorkspaceClient,
apiClient: &mockedApiClient,
root: NewWorkspaceRootPath("/dir"),
}
workspaceFilesExtensionsClient := workspaceFilesExtensionsClient{
workspaceClient: mockedWorkspaceClient.WorkspaceClient,
wsfs: &workspaceFilesClient,
}
_, err := workspaceFilesExtensionsClient.ReadDir(context.Background(), "/")
if tc.expectedError == "" {
assert.NoError(t, err)
} else {
assert.ErrorAs(t, err, &duplicatePathError{})
assert.EqualError(t, err, tc.expectedError)
}
// assert the mocked methods were actually called, as a sanity check.
workspaceApi.AssertNumberOfCalls(t, "ListAll", 1)
mockedApiClient.AssertNumberOfCalls(t, "Do", 1)
})
}
}

View File

@ -3,9 +3,23 @@ package python
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"runtime"
)
// DetectExecutable looks up the path to the python3 executable from the PATH
// environment variable.
//
// If virtualenv is activated, executable from the virtualenv is returned,
// because activating virtualenv adds python3 executable on a PATH.
//
// If python3 executable is not found on the PATH, the interpreter with the
// least version that satisfies minimal 3.8 version is returned, e.g.
// python3.10.
func DetectExecutable(ctx context.Context) (string, error) {
// TODO: add a shortcut if .python-version file is detected somewhere in
// the parent directory tree.
@ -32,3 +46,35 @@ func DetectExecutable(ctx context.Context) (string, error) {
}
return interpreter.Path, nil
}
// DetectVEnvExecutable returns the path to the python3 executable inside venvPath,
// that is not necessarily activated.
//
// If virtualenv is not created, or executable doesn't exist, the error is returned.
func DetectVEnvExecutable(venvPath string) (string, error) {
interpreterPath := filepath.Join(venvPath, "bin", "python3")
if runtime.GOOS == "windows" {
interpreterPath = filepath.Join(venvPath, "Scripts", "python3.exe")
}
if _, err := os.Stat(interpreterPath); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("can't find %q, check if virtualenv is created", interpreterPath)
} else {
return "", fmt.Errorf("can't find %q: %w", interpreterPath, err)
}
}
// pyvenv.cfg must be always present in correctly configured virtualenv,
// read more in https://snarky.ca/how-virtual-environments-work/
pyvenvPath := filepath.Join(venvPath, "pyvenv.cfg")
if _, err := os.Stat(pyvenvPath); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("expected %q to be virtualenv, but pyvenv.cfg is missing", venvPath)
} else {
return "", fmt.Errorf("can't find %q: %w", pyvenvPath, err)
}
}
return interpreterPath, nil
}

View File

@ -0,0 +1,46 @@
package python
import (
"os"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDetectVEnvExecutable(t *testing.T) {
dir := t.TempDir()
interpreterPath := interpreterPath(dir)
err := os.Mkdir(filepath.Dir(interpreterPath), 0755)
require.NoError(t, err)
err = os.WriteFile(interpreterPath, []byte(""), 0755)
require.NoError(t, err)
err = os.WriteFile(filepath.Join(dir, "pyvenv.cfg"), []byte(""), 0755)
require.NoError(t, err)
executable, err := DetectVEnvExecutable(dir)
assert.NoError(t, err)
assert.Equal(t, interpreterPath, executable)
}
func TestDetectVEnvExecutable_badLayout(t *testing.T) {
dir := t.TempDir()
_, err := DetectVEnvExecutable(dir)
assert.Errorf(t, err, "can't find %q, check if virtualenv is created", interpreterPath(dir))
}
func interpreterPath(venvPath string) string {
if runtime.GOOS == "windows" {
return filepath.Join(venvPath, "Scripts", "python3.exe")
} else {
return filepath.Join(venvPath, "bin", "python3")
}
}