Compare commits

...

16 Commits

Author SHA1 Message Date
Ilya Kuznetsov b89eff0883
Merge e9b72895dc into 7d732ceba8 2024-11-15 17:15:57 +00:00
Pieter Noordhuis 7d732ceba8
Consolidate test helpers for `io/fs` (#1906)
## Changes

We had a number of copies of test helpers for `io/fs` in the repository.

This change consolidates all of them to use the `libs/fakefs` package.

## Tests

Unit tests pass.
2024-11-15 15:37:21 +00:00
Ilya Kuznetsov e9b72895dc
test: Process target mode 2024-11-15 12:32:44 +01:00
Ilya Kuznetsov 8cd95eb0ea
test: Apply presets unit test 2024-11-15 12:09:53 +01:00
Ilya Kuznetsov 4cf692924c
fix: Missing root variable 2024-11-14 18:46:10 +01:00
Ilya Kuznetsov 51c8ef592c
fix: Applies autoformat 2024-11-14 18:38:31 +01:00
Ilya Kuznetsov 26f24538ee feat: Use dbr package for runtime check 2024-11-14 18:18:17 +01:00
Ilya Kuznetsov 0004ed27a7 Merge branch 'main' of github.com:databricks/cli into tmp/in-place-prototype 2024-11-14 18:12:34 +01:00
Ilya Kuznetsov b2164c0725
fix: Use SyncRootPath to allow using parent directories 2024-11-12 15:16:47 +01:00
Ilya Kuznetsov 6c1230889a
Merge branch 'main' of github.com:databricks/cli into tmp/in-place-prototype 2024-11-12 13:23:15 +01:00
Ilya Kuznetsov 5a22151580 feat: Move FilePath update to apply preset step 2024-11-12 09:58:36 +01:00
Ilya Kuznetsov a43f7ddad0 feat: Use global FilePath instead of locally overriden paths 2024-11-11 19:36:31 +01:00
Ilya Kuznetsov 086dfbc79d Merge branch 'main' of github.com:databricks/cli into tmp/in-place-prototype 2024-11-11 11:20:10 +01:00
Ilya Kuznetsov fbb9be9bdc
fix: Add SyncRoot mock to tests 2024-11-05 19:03:27 +01:00
Ilya Kuznetsov ae26beabfe Merge branch 'main' of github.com:databricks/cli into tmp/in-place-prototype 2024-11-05 18:13:27 +01:00
Ilya Kuznetsov 6b22fdebba Prototype for in-place deployments in DEV mode 2024-11-05 18:12:45 +01:00
15 changed files with 351 additions and 138 deletions

View File

@ -9,6 +9,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/textutil"
@ -221,6 +222,17 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
dashboard.DisplayName = prefix + dashboard.DisplayName
}
if config.IsExplicitlyEnabled((b.Config.Presets.InPlaceDeployment)) {
root := b.SyncRoot.Native()
isInWorkspace := strings.HasPrefix(root, "/Workspace/")
if isInWorkspace && dbr.RunsOnRuntime(ctx) {
b.Config.Workspace.FilePath = b.SyncRootPath
} else {
disabled := false
b.Config.Presets.InPlaceDeployment = &disabled
}
}
return diags
}

View File

@ -8,6 +8,8 @@ import (
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/require"
@ -55,6 +57,7 @@ func TestApplyPresetsPrefix(t *testing.T) {
NamePrefix: tt.prefix,
},
},
SyncRoot: vfs.MustNew(t.TempDir()),
}
ctx := context.Background()
@ -101,6 +104,7 @@ func TestApplyPresetsPrefixForUcSchema(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &bundle.Bundle{
SyncRoot: vfs.MustNew(t.TempDir()),
Config: config.Root{
Resources: config.Resources{
Schemas: map[string]*resources.Schema{
@ -180,6 +184,7 @@ func TestApplyPresetsTags(t *testing.T) {
Tags: tt.tags,
},
},
SyncRoot: vfs.MustNew(t.TempDir()),
}
ctx := context.Background()
@ -239,6 +244,7 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) {
JobsMaxConcurrentRuns: tt.setting,
},
},
SyncRoot: vfs.MustNew(t.TempDir()),
}
ctx := context.Background()
diag := bundle.Apply(ctx, b, mutator.ApplyPresets())
@ -264,6 +270,7 @@ func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) {
NamePrefix: "prefix-",
},
},
SyncRoot: vfs.MustNew(t.TempDir()),
}
ctx := context.Background()
@ -355,6 +362,7 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
TriggerPauseStatus: config.Paused,
},
},
SyncRoot: vfs.MustNew(t.TempDir()),
}
ctx := context.Background()
@ -364,3 +372,86 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
})
}
}
func TestApplyPresetsInPlaceDeployment(t *testing.T) {
testContext := context.Background()
enabled := true
disabled := false
remotePath := "/Users/files"
workspacePath := "/Workspace/user.name@company.com"
tests := []struct {
bundlePath string
ctx context.Context
name string
initialValue *bool
expectedValue *bool
expectedFilePath string
}{
{
name: "preset enabled, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: &enabled,
expectedValue: &enabled,
expectedFilePath: workspacePath,
},
{
name: "preset enabled, bundle not in Workspace, databricks runtime",
bundlePath: "/Users/user.name@company.com",
ctx: dbr.MockRuntime(testContext, true),
initialValue: &enabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset enabled, bundle in Workspace, not databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, false),
initialValue: &enabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset disabled, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: &disabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset nil, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: nil,
expectedValue: nil,
expectedFilePath: remotePath,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Presets: config.Presets{
InPlaceDeployment: tt.initialValue,
},
Workspace: config.Workspace{
FilePath: remotePath,
},
},
SyncRoot: vfs.MustNew(tt.bundlePath),
SyncRootPath: tt.bundlePath,
}
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
if diags.HasError() {
t.Fatalf("unexpected error: %v", diags)
}
require.Equal(t, tt.expectedFilePath, b.Config.Workspace.FilePath)
require.Equal(t, tt.expectedValue, b.Config.Presets.InPlaceDeployment)
})
}
}

View File

@ -57,6 +57,11 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
t.TriggerPauseStatus = config.Paused
}
if !config.IsExplicitlyDisabled(t.InPlaceDeployment) {
enabled := true
t.InPlaceDeployment = &enabled
}
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
enabled := true
t.PipelinesDevelopment = &enabled

View File

@ -9,8 +9,10 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/vfs"
sdkconfig "github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
@ -140,6 +142,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
},
},
},
SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
// Use AWS implementation for testing.
Tagging: tags.ForCloud(&sdkconfig.Config{
Host: "https://company.cloud.databricks.com",
@ -522,3 +525,31 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
}
func TestInPlaceDeploymentEnabled(t *testing.T) {
b, diags := processInPlaceBundle(true)
require.NoError(t, diags.Error())
assert.True(t, *b.Config.Presets.InPlaceDeployment)
assert.Equal(t, b.Config.Workspace.FilePath, b.SyncRootPath)
}
func TestInPlaceDeploymentDisabled(t *testing.T) {
b, diags := processInPlaceBundle(false)
require.NoError(t, diags.Error())
assert.False(t, *b.Config.Presets.InPlaceDeployment)
assert.NotEqual(t, b.Config.Workspace.FilePath, b.SyncRootPath)
}
func processInPlaceBundle(presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
b := mockBundle(config.Development)
workspacePath := "/Workspace/lennart@company.com/"
b.SyncRoot = vfs.MustNew(workspacePath)
b.SyncRootPath = workspacePath
b.Config.Presets.InPlaceDeployment = &presetEnabled
ctx := dbr.MockRuntime(context.Background(), true)
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
diags := bundle.Apply(ctx, b, m)
return b, diags
}

View File

@ -17,6 +17,11 @@ type Presets struct {
// JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs.
JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"`
// InPlaceDeployment indicates whether in-place deployment is enabled. Works only in workspace
// When set to true, resources created during deployment will point to source files in the workspace instead of their workspace copies.
// No resources will be uploaded to workspace
InPlaceDeployment *bool `json:"in_place_deployment,omitempty"`
// Tags to add to all resources.
Tags map[string]string `json:"tags,omitempty"`
}

View File

@ -7,6 +7,7 @@ import (
"io/fs"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
@ -23,6 +24,11 @@ func (m *upload) Name() string {
}
func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
if config.IsExplicitlyEnabled(b.Config.Presets.InPlaceDeployment) {
cmdio.LogString(ctx, "Bundle files uploading skipped: in-place deployment is enabled")
return nil
}
cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath))
opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b))
if err != nil {

View File

@ -7,6 +7,7 @@ import (
"testing"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/fakefs"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/spf13/cobra"
@ -84,7 +85,7 @@ func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceCl
cmd, m := setupCommand(t)
fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) {
fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
"dir": {FakeName: "root", FakeDir: true},
"dir/dirA": {FakeDir: true},
"dir/dirB": {FakeDir: true},

View File

@ -1,18 +1,21 @@
package fakefs
import (
"fmt"
"io/fs"
"time"
)
var ErrNotImplemented = fmt.Errorf("not implemented")
// DirEntry is a fake implementation of [fs.DirEntry].
type DirEntry struct {
FileInfo
fs.FileInfo
}
func (entry DirEntry) Type() fs.FileMode {
typ := fs.ModePerm
if entry.FakeDir {
if entry.IsDir() {
typ |= fs.ModeDir
}
return typ
@ -53,3 +56,32 @@ func (info FileInfo) IsDir() bool {
func (info FileInfo) Sys() any {
return nil
}
// File is a fake implementation of [fs.File].
type File struct {
FileInfo fs.FileInfo
}
func (f File) Close() error {
return nil
}
func (f File) Read(p []byte) (n int, err error) {
return 0, ErrNotImplemented
}
func (f File) Stat() (fs.FileInfo, error) {
return f.FileInfo, nil
}
// FS is a fake implementation of [fs.FS].
type FS map[string]fs.File
func (f FS) Open(name string) (fs.File, error) {
e, ok := f[name]
if !ok {
return nil, fs.ErrNotExist
}
return e, nil
}

View File

@ -0,0 +1,38 @@
package fakefs
import (
"io/fs"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFile(t *testing.T) {
var fakefile fs.File = File{
FileInfo: FileInfo{
FakeName: "file",
},
}
_, err := fakefile.Read([]byte{})
assert.ErrorIs(t, err, ErrNotImplemented)
fi, err := fakefile.Stat()
assert.NoError(t, err)
assert.Equal(t, "file", fi.Name())
err = fakefile.Close()
assert.NoError(t, err)
}
func TestFS(t *testing.T) {
var fakefs fs.FS = FS{
"file": File{},
}
_, err := fakefs.Open("doesntexist")
assert.ErrorIs(t, err, fs.ErrNotExist)
_, err = fakefs.Open("file")
assert.NoError(t, err)
}

View File

@ -6,6 +6,7 @@ import (
"testing"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/fakefs"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/spf13/cobra"
@ -17,7 +18,7 @@ func setupCompleter(t *testing.T, onlyDirs bool) *completer {
// Needed to make type context.valueCtx for mockFilerForPath
ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient)
fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
"dir": {FakeName: "root", FakeDir: true},
"dir/dirA": {FakeDir: true},
"dir/dirB": {FakeDir: true},

View File

@ -8,58 +8,12 @@ import (
"path"
"sort"
"strings"
"time"
"github.com/databricks/cli/libs/fakefs"
)
type FakeDirEntry struct {
FakeFileInfo
}
func (entry FakeDirEntry) Type() fs.FileMode {
typ := fs.ModePerm
if entry.FakeDir {
typ |= fs.ModeDir
}
return typ
}
func (entry FakeDirEntry) Info() (fs.FileInfo, error) {
return entry.FakeFileInfo, nil
}
type FakeFileInfo struct {
FakeName string
FakeSize int64
FakeDir bool
FakeMode fs.FileMode
}
func (info FakeFileInfo) Name() string {
return info.FakeName
}
func (info FakeFileInfo) Size() int64 {
return info.FakeSize
}
func (info FakeFileInfo) Mode() fs.FileMode {
return info.FakeMode
}
func (info FakeFileInfo) ModTime() time.Time {
return time.Now()
}
func (info FakeFileInfo) IsDir() bool {
return info.FakeDir
}
func (info FakeFileInfo) Sys() any {
return nil
}
type FakeFiler struct {
entries map[string]FakeFileInfo
entries map[string]fakefs.FileInfo
}
func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error {
@ -97,7 +51,7 @@ func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error
continue
}
out = append(out, FakeDirEntry{v})
out = append(out, fakefs.DirEntry{FileInfo: v})
}
sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
@ -117,7 +71,11 @@ func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error)
return entry, nil
}
func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler {
// NewFakeFiler creates a new fake [Filer] instance with the given entries.
// It sets the [Name] field of each entry to the base name of the path.
//
// This is meant to be used in tests.
func NewFakeFiler(entries map[string]fakefs.FileInfo) *FakeFiler {
fakeFiler := &FakeFiler{
entries: entries,
}

View File

@ -0,0 +1,98 @@
package filer
import (
"context"
"io"
"io/fs"
"testing"
"github.com/databricks/cli/libs/fakefs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFakeFiler_Read(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"file": {},
})
ctx := context.Background()
r, err := f.Read(ctx, "file")
require.NoError(t, err)
contents, err := io.ReadAll(r)
require.NoError(t, err)
// Contents of every file is "foo".
assert.Equal(t, "foo", string(contents))
}
func TestFakeFiler_Read_NotFound(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"foo": {},
})
ctx := context.Background()
_, err := f.Read(ctx, "bar")
assert.ErrorIs(t, err, fs.ErrNotExist)
}
func TestFakeFiler_ReadDir_NotFound(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"dir1": {FakeDir: true},
})
ctx := context.Background()
_, err := f.ReadDir(ctx, "dir2")
assert.ErrorIs(t, err, fs.ErrNotExist)
}
func TestFakeFiler_ReadDir_NotADirectory(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"file": {},
})
ctx := context.Background()
_, err := f.ReadDir(ctx, "file")
assert.ErrorIs(t, err, fs.ErrInvalid)
}
func TestFakeFiler_ReadDir(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"dir1": {FakeDir: true},
"dir1/file2": {},
"dir1/dir2": {FakeDir: true},
})
ctx := context.Background()
entries, err := f.ReadDir(ctx, "dir1/")
require.NoError(t, err)
require.Len(t, entries, 2)
// The entries are sorted by name.
assert.Equal(t, "dir2", entries[0].Name())
assert.True(t, entries[0].IsDir())
assert.Equal(t, "file2", entries[1].Name())
assert.False(t, entries[1].IsDir())
}
func TestFakeFiler_Stat(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"file": {},
})
ctx := context.Background()
info, err := f.Stat(ctx, "file")
require.NoError(t, err)
assert.Equal(t, "file", info.Name())
}
func TestFakeFiler_Stat_NotFound(t *testing.T) {
f := NewFakeFiler(map[string]fakefs.FileInfo{
"foo": {},
})
ctx := context.Background()
_, err := f.Stat(ctx, "bar")
assert.ErrorIs(t, err, fs.ErrNotExist)
}

View File

@ -6,6 +6,7 @@ import (
"io/fs"
"testing"
"github.com/databricks/cli/libs/fakefs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -35,7 +36,7 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) {
}
func fakeFS() fs.FS {
fakeFiler := NewFakeFiler(map[string]FakeFileInfo{
fakeFiler := NewFakeFiler(map[string]fakefs.FileInfo{
".": {FakeName: "root", FakeDir: true},
"dirA": {FakeDir: true},
"dirB": {FakeDir: true},

View File

@ -7,6 +7,7 @@ import (
"path/filepath"
"testing"
"github.com/databricks/cli/libs/fakefs"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -100,11 +101,21 @@ func TestDetectFileWithLongHeader(t *testing.T) {
assert.False(t, nb)
}
type fileInfoWithWorkspaceInfo struct {
fakefs.FileInfo
oi workspace.ObjectInfo
}
func (f fileInfoWithWorkspaceInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
return f.oi
}
func TestDetectWithObjectInfo(t *testing.T) {
fakeFS := &fakeFS{
fakeFile{
fakeFileInfo{
workspace.ObjectInfo{
fakefs := fakefs.FS{
"file.py": fakefs.File{
FileInfo: fileInfoWithWorkspaceInfo{
oi: workspace.ObjectInfo{
ObjectType: workspace.ObjectTypeNotebook,
Language: workspace.LanguagePython,
},
@ -112,7 +123,7 @@ func TestDetectWithObjectInfo(t *testing.T) {
},
}
nb, lang, err := DetectWithFS(fakeFS, "doesntmatter")
nb, lang, err := DetectWithFS(fakefs, "file.py")
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguagePython, lang)

View File

@ -1,77 +0,0 @@
package notebook
import (
"fmt"
"io/fs"
"time"
"github.com/databricks/databricks-sdk-go/service/workspace"
)
type fakeFS struct {
fakeFile
}
type fakeFile struct {
fakeFileInfo
}
func (f fakeFile) Close() error {
return nil
}
func (f fakeFile) Read(p []byte) (n int, err error) {
return 0, fmt.Errorf("not implemented")
}
func (f fakeFile) Stat() (fs.FileInfo, error) {
return f.fakeFileInfo, nil
}
type fakeFileInfo struct {
oi workspace.ObjectInfo
}
func (f fakeFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
return f.oi
}
func (f fakeFileInfo) Name() string {
return ""
}
func (f fakeFileInfo) Size() int64 {
return 0
}
func (f fakeFileInfo) Mode() fs.FileMode {
return 0
}
func (f fakeFileInfo) ModTime() time.Time {
return time.Time{}
}
func (f fakeFileInfo) IsDir() bool {
return false
}
func (f fakeFileInfo) Sys() any {
return nil
}
func (f fakeFS) Open(name string) (fs.File, error) {
return f.fakeFile, nil
}
func (f fakeFS) Stat(name string) (fs.FileInfo, error) {
panic("not implemented")
}
func (f fakeFS) ReadDir(name string) ([]fs.DirEntry, error) {
panic("not implemented")
}
func (f fakeFS) ReadFile(name string) ([]byte, error) {
panic("not implemented")
}