mirror of https://github.com/databricks/cli.git
Add paths field to bundle sync configuration (#1694)
## Changes This field allows a user to configure paths to synchronize to the workspace. Allowed values are relative paths to files and directories anchored at the directory where the field is set. If one or more values traverse up the directory tree (to an ancestor of the bundle root directory), the CLI will dynamically determine the root path to use to ensure that the file tree structure remains intact. For example, given a `databricks.yml` in `my_bundle` that includes: ```yaml sync: paths: - ../common - . ``` Then upon synchronization, the workspace will look like: ``` . ├── common │ └── lib.py └── my_bundle ├── databricks.yml └── notebook.py ``` If not set behavior remains identical. ## Tests * Newly added unit tests for the mutators and under `bundle/tests`. * Manually confirmed a bundle without this configuration works the same. * Manually confirmed a bundle with this configuration works.
This commit is contained in:
parent
6f345293b1
commit
6e8cd835a3
|
@ -39,6 +39,14 @@ type Bundle struct {
|
|||
// Exclusively use this field for filesystem operations.
|
||||
BundleRoot vfs.Path
|
||||
|
||||
// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
|
||||
// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
|
||||
SyncRoot vfs.Path
|
||||
|
||||
// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
|
||||
// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
|
||||
SyncRootPath string
|
||||
|
||||
Config config.Root
|
||||
|
||||
// Metadata about the bundle deployment. This is the interface Databricks services
|
||||
|
|
|
@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path {
|
|||
return r.b.BundleRoot
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) SyncRoot() vfs.Path {
|
||||
return r.b.SyncRoot
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
|
||||
return r.b.WorkspaceClient()
|
||||
}
|
||||
|
|
|
@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string {
|
|||
}
|
||||
|
||||
func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
root := b.BundleRoot.Native()
|
||||
root := b.SyncRoot.Native()
|
||||
|
||||
// The bundle root must be located in /Workspace/
|
||||
if !strings.HasPrefix(root, "/Workspace/") {
|
||||
|
@ -45,6 +45,6 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
|
|||
return diag.FromErr(err)
|
||||
}
|
||||
|
||||
b.BundleRoot = p
|
||||
b.SyncRoot = p
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -45,6 +45,10 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
|
|||
func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
|
||||
v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
|
|
|
@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
|
|||
RootPath: ".",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
".",
|
||||
"../common",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
|
@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
|
||||
bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
|
||||
|
@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
|
|||
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
|
||||
assert.NoError(t, diags.Error())
|
||||
|
||||
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
|
||||
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
|
||||
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
|
||||
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
|
||||
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])
|
||||
|
@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
|||
RootPath: "/tmp/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
".",
|
||||
"../common",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
|
@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
|
||||
bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
|
||||
|
@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
|||
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
|
||||
assert.NoError(t, diags.Error())
|
||||
|
||||
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
|
||||
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
|
||||
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
|
||||
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
|
||||
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type syncDefaultPath struct{}
|
||||
|
||||
// SyncDefaultPath configures the default sync path to be equal to the bundle root.
|
||||
func SyncDefaultPath() bundle.Mutator {
|
||||
return &syncDefaultPath{}
|
||||
}
|
||||
|
||||
func (m *syncDefaultPath) Name() string {
|
||||
return "SyncDefaultPath"
|
||||
}
|
||||
|
||||
func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
isset := false
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
pv, _ := dyn.Get(v, "sync.paths")
|
||||
|
||||
// If the sync paths field is already set, do nothing.
|
||||
// We know it is set if its value is either a nil or a sequence (empty or not).
|
||||
switch pv.Kind() {
|
||||
case dyn.KindNil, dyn.KindSequence:
|
||||
isset = true
|
||||
}
|
||||
|
||||
return v, nil
|
||||
})
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
||||
// If the sync paths field is already set, do nothing.
|
||||
if isset {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set the sync paths to the default value.
|
||||
b.Config.Sync.Paths = []string{"."}
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,82 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath())
|
||||
require.NoError(t, diags.Error())
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
}
|
||||
|
||||
func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
|
||||
tcases := []struct {
|
||||
name string
|
||||
paths dyn.Value
|
||||
expect []string
|
||||
}{
|
||||
{
|
||||
name: "nil",
|
||||
paths: dyn.V(nil),
|
||||
expect: nil,
|
||||
},
|
||||
{
|
||||
name: "empty sequence",
|
||||
paths: dyn.V([]dyn.Value{}),
|
||||
expect: []string{},
|
||||
},
|
||||
{
|
||||
name: "non-empty sequence",
|
||||
paths: dyn.V([]dyn.Value{dyn.V("something")}),
|
||||
expect: []string{"something"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tcase := range tcases {
|
||||
t.Run(tcase.name, func(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
}
|
||||
|
||||
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping()))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
v, err = dyn.Set(v, "sync.paths", tcase.paths)
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
return v, nil
|
||||
})
|
||||
return diag.FromErr(err)
|
||||
})
|
||||
require.NoError(t, diags.Error())
|
||||
|
||||
ctx := context.Background()
|
||||
diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath())
|
||||
require.NoError(t, diags.Error())
|
||||
|
||||
// If the sync paths field is already set, do nothing.
|
||||
assert.Equal(t, tcase.expect, b.Config.Sync.Paths)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -0,0 +1,120 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/cli/libs/vfs"
|
||||
)
|
||||
|
||||
type syncInferRoot struct{}
|
||||
|
||||
// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the
|
||||
// paths in the sync configuration. The sync root may be different from the bundle root
|
||||
// when the user intends to synchronize files outside the bundle root.
|
||||
//
|
||||
// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant.
|
||||
// That is, the sync root must contain the bundle root.
|
||||
//
|
||||
// This mutator requires all sync-related paths and patterns to be relative to the bundle root path.
|
||||
// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator.
|
||||
func SyncInferRoot() bundle.Mutator {
|
||||
return &syncInferRoot{}
|
||||
}
|
||||
|
||||
func (m *syncInferRoot) Name() string {
|
||||
return "SyncInferRoot"
|
||||
}
|
||||
|
||||
// computeRoot finds the innermost path that contains the specified path.
|
||||
// It traverses up the root path until it finds the innermost path.
|
||||
// If the path does not exist, it returns an empty string.
|
||||
//
|
||||
// See "sync_infer_root_internal_test.go" for examples.
|
||||
func (m *syncInferRoot) computeRoot(path string, root string) string {
|
||||
for !filepath.IsLocal(path) {
|
||||
// Break if we have reached the root of the filesystem.
|
||||
dir := filepath.Dir(root)
|
||||
if dir == root {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Update the sync path as we navigate up the directory tree.
|
||||
path = filepath.Join(filepath.Base(root), path)
|
||||
|
||||
// Move up the directory tree.
|
||||
root = dir
|
||||
}
|
||||
|
||||
return filepath.Clean(root)
|
||||
}
|
||||
|
||||
func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
var diags diag.Diagnostics
|
||||
|
||||
// Use the bundle root path as the starting point for inferring the sync root path.
|
||||
bundleRootPath := filepath.Clean(b.RootPath)
|
||||
|
||||
// Infer the sync root path by looking at each one of the sync paths.
|
||||
// Every sync path must be a descendant of the final sync root path.
|
||||
syncRootPath := bundleRootPath
|
||||
for _, path := range b.Config.Sync.Paths {
|
||||
computedPath := m.computeRoot(path, bundleRootPath)
|
||||
if computedPath == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Update sync root path if the computed root path is an ancestor of the current sync root path.
|
||||
if len(computedPath) < len(syncRootPath) {
|
||||
syncRootPath = computedPath
|
||||
}
|
||||
}
|
||||
|
||||
// The new sync root path can only be an ancestor of the previous root path.
|
||||
// Compute the relative path from the sync root to the bundle root.
|
||||
rel, err := filepath.Rel(syncRootPath, bundleRootPath)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
||||
// If during computation of the sync root path we hit the root of the filesystem,
|
||||
// then one or more of the sync paths are outside the filesystem.
|
||||
// Check if this happened by verifying that none of the paths escape the root
|
||||
// when joined with the sync root path.
|
||||
for i, path := range b.Config.Sync.Paths {
|
||||
if filepath.IsLocal(filepath.Join(rel, path)) {
|
||||
continue
|
||||
}
|
||||
|
||||
diags = append(diags, diag.Diagnostic{
|
||||
Severity: diag.Error,
|
||||
Summary: fmt.Sprintf("invalid sync path %q", path),
|
||||
Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)),
|
||||
Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))},
|
||||
})
|
||||
}
|
||||
|
||||
if diags.HasError() {
|
||||
return diags
|
||||
}
|
||||
|
||||
// Update all paths in the sync configuration to be relative to the sync root.
|
||||
for i, p := range b.Config.Sync.Paths {
|
||||
b.Config.Sync.Paths[i] = filepath.Join(rel, p)
|
||||
}
|
||||
for i, p := range b.Config.Sync.Include {
|
||||
b.Config.Sync.Include[i] = filepath.Join(rel, p)
|
||||
}
|
||||
for i, p := range b.Config.Sync.Exclude {
|
||||
b.Config.Sync.Exclude[i] = filepath.Join(rel, p)
|
||||
}
|
||||
|
||||
// Configure the sync root path.
|
||||
b.SyncRoot = vfs.MustNew(syncRootPath)
|
||||
b.SyncRootPath = syncRootPath
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,72 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSyncInferRootInternal_ComputeRoot(t *testing.T) {
|
||||
s := syncInferRoot{}
|
||||
|
||||
tcases := []struct {
|
||||
path string
|
||||
root string
|
||||
out string
|
||||
}{
|
||||
{
|
||||
// Test that "." doesn't change the root.
|
||||
path: ".",
|
||||
root: "/tmp/some/dir",
|
||||
out: "/tmp/some/dir",
|
||||
},
|
||||
{
|
||||
// Test that a subdirectory doesn't change the root.
|
||||
path: "sub",
|
||||
root: "/tmp/some/dir",
|
||||
out: "/tmp/some/dir",
|
||||
},
|
||||
{
|
||||
// Test that a parent directory changes the root.
|
||||
path: "../common",
|
||||
root: "/tmp/some/dir",
|
||||
out: "/tmp/some",
|
||||
},
|
||||
{
|
||||
// Test that a deeply nested parent directory changes the root.
|
||||
path: "../../../../../../common",
|
||||
root: "/tmp/some/dir/that/is/very/deeply/nested",
|
||||
out: "/tmp/some",
|
||||
},
|
||||
{
|
||||
// Test that a parent directory changes the root at the filesystem root boundary.
|
||||
path: "../common",
|
||||
root: "/tmp",
|
||||
out: "/",
|
||||
},
|
||||
{
|
||||
// Test that an invalid parent directory doesn't change the root and returns an empty string.
|
||||
path: "../common",
|
||||
root: "/",
|
||||
out: "",
|
||||
},
|
||||
{
|
||||
// Test that the returned path is cleaned even if the root doesn't change.
|
||||
path: "sub",
|
||||
root: "/tmp/some/../dir",
|
||||
out: "/tmp/dir",
|
||||
},
|
||||
{
|
||||
// Test that a relative root path also works.
|
||||
path: "../common",
|
||||
root: "foo/bar",
|
||||
out: "foo",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tcases {
|
||||
out := s.computeRoot(tc.path, tc.root)
|
||||
assert.Equal(t, tc.out, filepath.ToSlash(out))
|
||||
}
|
||||
}
|
|
@ -0,0 +1,198 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
".",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
},
|
||||
Exclude: []string{
|
||||
"baz",
|
||||
"qux",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, filepath.FromSlash("/tmp/some/dir"), b.SyncRootPath)
|
||||
|
||||
// Check that the paths are unchanged.
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
|
||||
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
||||
func TestSyncInferRoot_NominalRelative(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "./some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
".",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
},
|
||||
Exclude: []string{
|
||||
"baz",
|
||||
"qux",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, filepath.FromSlash("some/dir"), b.SyncRootPath)
|
||||
|
||||
// Check that the paths are unchanged.
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
|
||||
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
||||
func TestSyncInferRoot_ParentDirectory(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
"../common",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
},
|
||||
Exclude: []string{
|
||||
"baz",
|
||||
"qux",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
|
||||
|
||||
// Check that the paths are updated.
|
||||
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
|
||||
assert.Equal(t, []string{filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/bar")}, b.Config.Sync.Include)
|
||||
assert.Equal(t, []string{filepath.FromSlash("dir/baz"), filepath.FromSlash("dir/qux")}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
||||
func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir/that/is/very/deeply/nested",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
"../../../../../../common",
|
||||
},
|
||||
Include: []string{
|
||||
"foo",
|
||||
"bar",
|
||||
},
|
||||
Exclude: []string{
|
||||
"baz",
|
||||
"qux",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
|
||||
|
||||
// Check that the paths are updated.
|
||||
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
|
||||
assert.Equal(t, []string{
|
||||
filepath.FromSlash("dir/that/is/very/deeply/nested/foo"),
|
||||
filepath.FromSlash("dir/that/is/very/deeply/nested/bar"),
|
||||
}, b.Config.Sync.Include)
|
||||
assert.Equal(t, []string{
|
||||
filepath.FromSlash("dir/that/is/very/deeply/nested/baz"),
|
||||
filepath.FromSlash("dir/that/is/very/deeply/nested/qux"),
|
||||
}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
||||
func TestSyncInferRoot_MultiplePaths(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/bundle/root",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
"./foo",
|
||||
"../common",
|
||||
"./bar",
|
||||
"../../baz",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
|
||||
|
||||
// Check that the paths are updated.
|
||||
assert.Equal(t, filepath.FromSlash("bundle/root/foo"), b.Config.Sync.Paths[0])
|
||||
assert.Equal(t, filepath.FromSlash("bundle/common"), b.Config.Sync.Paths[1])
|
||||
assert.Equal(t, filepath.FromSlash("bundle/root/bar"), b.Config.Sync.Paths[2])
|
||||
assert.Equal(t, filepath.FromSlash("baz"), b.Config.Sync.Paths[3])
|
||||
}
|
||||
|
||||
func TestSyncInferRoot_Error(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
"../../../../error",
|
||||
"../../../thisworks",
|
||||
"../../../../../error",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths", "databricks.yml")
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
require.Len(t, diags, 2)
|
||||
assert.Equal(t, `invalid sync path "../../../../error"`, diags[0].Summary)
|
||||
assert.Equal(t, "databricks.yml:0:0", diags[0].Locations[0].String())
|
||||
assert.Equal(t, "sync.paths[0]", diags[0].Paths[0].String())
|
||||
assert.Equal(t, `invalid sync path "../../../../../error"`, diags[1].Summary)
|
||||
assert.Equal(t, "databricks.yml:0:0", diags[1].Locations[0].String())
|
||||
assert.Equal(t, "sync.paths[2]", diags[1].Paths[0].String())
|
||||
}
|
|
@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund
|
|||
return err
|
||||
}
|
||||
|
||||
internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
|
||||
internalDirRel, err := filepath.Rel(b.SyncRootPath, internalDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -56,8 +56,12 @@ func TestGenerateTrampoline(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: tmpDir,
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/Workspace/files",
|
||||
},
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
||||
},
|
||||
|
@ -89,6 +93,6 @@ func TestGenerateTrampoline(t *testing.T) {
|
|||
require.Equal(t, "Hello from Trampoline", string(bytes))
|
||||
|
||||
task := b.Config.Resources.Jobs["test"].Tasks[0]
|
||||
require.Equal(t, task.NotebookTask.NotebookPath, ".databricks/bundle/development/.internal/notebook_test_to_trampoline")
|
||||
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_test_to_trampoline", task.NotebookTask.NotebookPath)
|
||||
require.Nil(t, task.PythonWheelTask)
|
||||
}
|
||||
|
|
|
@ -93,14 +93,14 @@ func (t *translateContext) rewritePath(
|
|||
return nil
|
||||
}
|
||||
|
||||
// Local path must be contained in the bundle root.
|
||||
// Local path must be contained in the sync root.
|
||||
// If it isn't, it won't be synchronized into the workspace.
|
||||
localRelPath, err := filepath.Rel(t.b.RootPath, localPath)
|
||||
localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(localRelPath, "..") {
|
||||
return fmt.Errorf("path %s is not contained in bundle root path", localPath)
|
||||
return fmt.Errorf("path %s is not contained in sync root path", localPath)
|
||||
}
|
||||
|
||||
// Prefix remote path with its remote root path.
|
||||
|
@ -118,7 +118,7 @@ func (t *translateContext) rewritePath(
|
|||
}
|
||||
|
||||
func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
|
||||
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", fmt.Errorf("notebook %s not found", literal)
|
||||
}
|
||||
|
@ -134,7 +134,7 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe
|
|||
}
|
||||
|
||||
func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
|
||||
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", fmt.Errorf("file %s not found", literal)
|
||||
}
|
||||
|
@ -148,7 +148,7 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat
|
|||
}
|
||||
|
||||
func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||
info, err := t.b.BundleRoot.Stat(filepath.ToSlash(localRelPath))
|
||||
info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
@ -41,8 +41,8 @@ func touchEmptyFile(t *testing.T, path string) {
|
|||
func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -113,8 +113,8 @@ func TestTranslatePaths(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -289,8 +289,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -380,12 +380,12 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
)
|
||||
}
|
||||
|
||||
func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
|
||||
func TestTranslatePathsOutsideSyncRoot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -411,15 +411,15 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
|
|||
bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), "is not contained in bundle root")
|
||||
assert.ErrorContains(t, diags.Error(), "is not contained in sync root path")
|
||||
}
|
||||
|
||||
func TestJobNotebookDoesNotExistError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
|
@ -449,8 +449,8 @@ func TestJobFileDoesNotExistError(t *testing.T) {
|
|||
dir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
|
@ -480,8 +480,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
|
|||
dir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Pipelines: map[string]*resources.Pipeline{
|
||||
|
@ -511,8 +511,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
|
|||
dir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Pipelines: map[string]*resources.Pipeline{
|
||||
|
@ -543,8 +543,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
|
|||
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -578,8 +578,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -613,8 +613,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -648,8 +648,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
|
|||
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/bundle",
|
||||
|
@ -684,8 +684,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "env2.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
|
@ -724,8 +724,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
|
|||
func TestTranslatePathWithComplexVariables(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
BundleRoot: vfs.MustNew(dir),
|
||||
SyncRootPath: dir,
|
||||
SyncRoot: vfs.MustNew(dir),
|
||||
Config: config.Root{
|
||||
Variables: map[string]*variable.Variable{
|
||||
"cluster_libraries": {
|
||||
|
|
|
@ -1,6 +1,10 @@
|
|||
package config
|
||||
|
||||
type Sync struct {
|
||||
// Paths contains a list of paths to synchronize relative to the bundle root path.
|
||||
// If not configured, this defaults to synchronizing everything in the bundle root path (i.e. `.`).
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
|
||||
// Include contains a list of globs evaluated relative to the bundle root path
|
||||
// to explicitly include files that were excluded by the user's gitignore.
|
||||
Include []string `json:"include,omitempty"`
|
||||
|
|
|
@ -28,8 +28,8 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp
|
|||
}
|
||||
|
||||
opts := &sync.SyncOptions{
|
||||
LocalRoot: rb.BundleRoot(),
|
||||
Paths: []string{"."},
|
||||
LocalRoot: rb.SyncRoot(),
|
||||
Paths: rb.Config().Sync.Paths,
|
||||
Include: includes,
|
||||
Exclude: rb.Config().Sync.Exclude,
|
||||
|
||||
|
|
|
@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
|
|||
}
|
||||
|
||||
log.Infof(ctx, "Creating new snapshot")
|
||||
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.BundleRoot), opts)
|
||||
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.SyncRoot), opts)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
|
|
@ -64,6 +64,10 @@ func testStatePull(t *testing.T, opts statePullOpts) {
|
|||
b := &bundle.Bundle{
|
||||
RootPath: tmpDir,
|
||||
BundleRoot: vfs.MustNew(tmpDir),
|
||||
|
||||
SyncRootPath: tmpDir,
|
||||
SyncRoot: vfs.MustNew(tmpDir),
|
||||
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
@ -81,11 +85,11 @@ func testStatePull(t *testing.T, opts statePullOpts) {
|
|||
ctx := context.Background()
|
||||
|
||||
for _, file := range opts.localFiles {
|
||||
testutil.Touch(t, b.RootPath, "bar", file)
|
||||
testutil.Touch(t, b.SyncRootPath, "bar", file)
|
||||
}
|
||||
|
||||
for _, file := range opts.localNotebooks {
|
||||
testutil.TouchNotebook(t, b.RootPath, "bar", file)
|
||||
testutil.TouchNotebook(t, b.SyncRootPath, "bar", file)
|
||||
}
|
||||
|
||||
if opts.withExistingSnapshot {
|
||||
|
|
|
@ -21,7 +21,18 @@ func Initialize() bundle.Mutator {
|
|||
"initialize",
|
||||
[]bundle.Mutator{
|
||||
validate.AllResourcesHaveValues(),
|
||||
|
||||
// Update all path fields in the sync block to be relative to the bundle root path.
|
||||
mutator.RewriteSyncPaths(),
|
||||
|
||||
// Configure the default sync path to equal the bundle root if not explicitly configured.
|
||||
// By default, this means all files in the bundle root directory are synchronized.
|
||||
mutator.SyncDefaultPath(),
|
||||
|
||||
// Figure out if the sync root path is identical or an ancestor of the bundle root path.
|
||||
// If it is an ancestor, this updates all paths to be relative to the sync root path.
|
||||
mutator.SyncInferRoot(),
|
||||
|
||||
mutator.MergeJobClusters(),
|
||||
mutator.MergeJobParameters(),
|
||||
mutator.MergeJobTasks(),
|
||||
|
|
|
@ -2,7 +2,6 @@ package python
|
|||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
|
@ -18,11 +17,15 @@ func TestNoTransformByDefault(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: tmpDir,
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
||||
},
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/Workspace/files",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job1": {
|
||||
|
@ -63,11 +66,15 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: tmpDir,
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
||||
},
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/Workspace/files",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job1": {
|
||||
|
@ -102,14 +109,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
|
|||
task := b.Config.Resources.Jobs["job1"].Tasks[0]
|
||||
require.Nil(t, task.PythonWheelTask)
|
||||
require.NotNil(t, task.NotebookTask)
|
||||
|
||||
dir, err := b.InternalDir(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
internalDirRel, err := filepath.Rel(b.RootPath, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath)
|
||||
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_job1_key1", task.NotebookTask.NotebookPath)
|
||||
|
||||
require.Len(t, task.Libraries, 1)
|
||||
require.Equal(t, "/Workspace/Users/test@test.com/bundle/dist/test.jar", task.Libraries[0].Jar)
|
||||
|
|
|
@ -40,6 +40,8 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
|
|||
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||
phases.LoadNamedTarget(env),
|
||||
mutator.RewriteSyncPaths(),
|
||||
mutator.SyncDefaultPath(),
|
||||
mutator.SyncInferRoot(),
|
||||
mutator.MergeJobClusters(),
|
||||
mutator.MergeJobParameters(),
|
||||
mutator.MergeJobTasks(),
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
bundle:
|
||||
name: sync_paths
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
sync:
|
||||
paths:
|
||||
- src
|
||||
|
||||
targets:
|
||||
development:
|
||||
sync:
|
||||
paths:
|
||||
- development
|
||||
|
||||
staging:
|
||||
sync:
|
||||
paths:
|
||||
- staging
|
|
@ -0,0 +1,26 @@
|
|||
bundle:
|
||||
name: sync_paths
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
targets:
|
||||
development:
|
||||
sync:
|
||||
paths:
|
||||
- development
|
||||
|
||||
staging:
|
||||
sync:
|
||||
paths:
|
||||
- staging
|
||||
|
||||
undefined: ~
|
||||
|
||||
nil:
|
||||
sync:
|
||||
paths: ~
|
||||
|
||||
empty:
|
||||
sync:
|
||||
paths: []
|
|
@ -0,0 +1,10 @@
|
|||
bundle:
|
||||
name: shared_code
|
||||
|
||||
workspace:
|
||||
host: https://acme.cloud.databricks.com/
|
||||
|
||||
sync:
|
||||
paths:
|
||||
- "../common"
|
||||
- "."
|
|
@ -0,0 +1 @@
|
|||
Placeholder for files to be deployed as part of multiple bundles.
|
|
@ -12,14 +12,20 @@ func TestSyncOverride(t *testing.T) {
|
|||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/override", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/override", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/override", "prod")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
@ -28,14 +34,20 @@ func TestSyncOverrideNoRootSync(t *testing.T) {
|
|||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/override_no_root", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/override_no_root", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/override_no_root", "prod")
|
||||
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
@ -44,10 +56,14 @@ func TestSyncNil(t *testing.T) {
|
|||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/nil", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.Nil(t, b.Config.Sync.Include)
|
||||
assert.Nil(t, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/nil", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
@ -56,10 +72,59 @@ func TestSyncNilRoot(t *testing.T) {
|
|||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/nil_root", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.Nil(t, b.Config.Sync.Include)
|
||||
assert.Nil(t, b.Config.Sync.Exclude)
|
||||
|
||||
b = loadTarget(t, "./sync/nil_root", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
|
||||
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
|
||||
}
|
||||
|
||||
func TestSyncPaths(t *testing.T) {
|
||||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/paths", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"src", "development"}, b.Config.Sync.Paths)
|
||||
|
||||
b = loadTarget(t, "./sync/paths", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"src", "staging"}, b.Config.Sync.Paths)
|
||||
}
|
||||
|
||||
func TestSyncPathsNoRoot(t *testing.T) {
|
||||
var b *bundle.Bundle
|
||||
|
||||
b = loadTarget(t, "./sync/paths_no_root", "development")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
|
||||
assert.ElementsMatch(t, []string{"development"}, b.Config.Sync.Paths)
|
||||
|
||||
b = loadTarget(t, "./sync/paths_no_root", "staging")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
|
||||
assert.ElementsMatch(t, []string{"staging"}, b.Config.Sync.Paths)
|
||||
|
||||
// If not set at all, it defaults to "."
|
||||
b = loadTarget(t, "./sync/paths_no_root", "undefined")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
|
||||
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
|
||||
|
||||
// If set to nil, it won't sync anything.
|
||||
b = loadTarget(t, "./sync/paths_no_root", "nil")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
|
||||
assert.Len(t, b.Config.Sync.Paths, 0)
|
||||
|
||||
// If set to an empty sequence, it won't sync anything.
|
||||
b = loadTarget(t, "./sync/paths_no_root", "empty")
|
||||
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
|
||||
assert.Len(t, b.Config.Sync.Paths, 0)
|
||||
}
|
||||
|
||||
func TestSyncSharedCode(t *testing.T) {
|
||||
b := loadTarget(t, "./sync/shared_code/bundle", "default")
|
||||
assert.Equal(t, filepath.FromSlash("sync/shared_code"), b.SyncRootPath)
|
||||
assert.ElementsMatch(t, []string{"common", "bundle"}, b.Config.Sync.Paths)
|
||||
}
|
||||
|
|
|
@ -17,8 +17,10 @@ import (
|
|||
func TestSyncOptionsFromBundle(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
b := &bundle.Bundle{
|
||||
RootPath: tempDir,
|
||||
BundleRoot: vfs.MustNew(tempDir),
|
||||
RootPath: tempDir,
|
||||
BundleRoot: vfs.MustNew(tempDir),
|
||||
SyncRootPath: tempDir,
|
||||
SyncRoot: vfs.MustNew(tempDir),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
Loading…
Reference in New Issue