Add paths field to bundle sync configuration (#1694)

## Changes

This field allows a user to configure paths to synchronize to the
workspace.

Allowed values are relative paths to files and directories anchored at
the directory where the field is set. If one or more values traverse up
the directory tree (to an ancestor of the bundle root directory), the
CLI will dynamically determine the root path to use to ensure that the
file tree structure remains intact.

For example, given a `databricks.yml` in `my_bundle` that includes:

```yaml
sync:
  paths:
    - ../common
    - .
```

Then upon synchronization, the workspace will look like:
```
.
├── common
│   └── lib.py
└── my_bundle
    ├── databricks.yml
    └── notebook.py
```

If not set behavior remains identical.

## Tests

* Newly added unit tests for the mutators and under `bundle/tests`.
* Manually confirmed a bundle without this configuration works the same.
* Manually confirmed a bundle with this configuration works.
This commit is contained in:
Pieter Noordhuis 2024-08-21 17:33:25 +02:00 committed by GitHub
parent 6f345293b1
commit 6e8cd835a3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 760 additions and 59 deletions

View File

@ -39,6 +39,14 @@ type Bundle struct {
// Exclusively use this field for filesystem operations.
BundleRoot vfs.Path
// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
SyncRoot vfs.Path
// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
SyncRootPath string
Config config.Root
// Metadata about the bundle deployment. This is the interface Databricks services

View File

@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path {
return r.b.BundleRoot
}
func (r ReadOnlyBundle) SyncRoot() vfs.Path {
return r.b.SyncRoot
}
func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient {
return r.b.WorkspaceClient()
}

View File

@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string {
}
func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
root := b.BundleRoot.Native()
root := b.SyncRoot.Native()
// The bundle root must be located in /Workspace/
if !strings.HasPrefix(root, "/Workspace/") {
@ -45,6 +45,6 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
return diag.FromErr(err)
}
b.BundleRoot = p
b.SyncRoot = p
return nil
}

View File

@ -45,6 +45,10 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
if err != nil {
return dyn.InvalidValue, err
}
v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
if err != nil {
return dyn.InvalidValue, err

View File

@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
RootPath: ".",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
"../common",
},
Include: []string{
"foo",
"bar",
@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
},
}
bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])
@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
RootPath: "/tmp/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
"../common",
},
Include: []string{
"foo",
"bar",
@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
},
}
bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0])
assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1])
assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0])

View File

@ -0,0 +1,48 @@
package mutator
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
)
type syncDefaultPath struct{}
// SyncDefaultPath configures the default sync path to be equal to the bundle root.
func SyncDefaultPath() bundle.Mutator {
return &syncDefaultPath{}
}
func (m *syncDefaultPath) Name() string {
return "SyncDefaultPath"
}
func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
isset := false
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
pv, _ := dyn.Get(v, "sync.paths")
// If the sync paths field is already set, do nothing.
// We know it is set if its value is either a nil or a sequence (empty or not).
switch pv.Kind() {
case dyn.KindNil, dyn.KindSequence:
isset = true
}
return v, nil
})
if err != nil {
return diag.FromErr(err)
}
// If the sync paths field is already set, do nothing.
if isset {
return nil
}
// Set the sync paths to the default value.
b.Config.Sync.Paths = []string{"."}
return nil
}

View File

@ -0,0 +1,82 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath())
require.NoError(t, diags.Error())
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
}
func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
tcases := []struct {
name string
paths dyn.Value
expect []string
}{
{
name: "nil",
paths: dyn.V(nil),
expect: nil,
},
{
name: "empty sequence",
paths: dyn.V([]dyn.Value{}),
expect: []string{},
},
{
name: "non-empty sequence",
paths: dyn.V([]dyn.Value{dyn.V("something")}),
expect: []string{"something"},
},
}
for _, tcase := range tcases {
t.Run(tcase.name, func(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{},
}
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping()))
if err != nil {
return dyn.InvalidValue, err
}
v, err = dyn.Set(v, "sync.paths", tcase.paths)
if err != nil {
return dyn.InvalidValue, err
}
return v, nil
})
return diag.FromErr(err)
})
require.NoError(t, diags.Error())
ctx := context.Background()
diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath())
require.NoError(t, diags.Error())
// If the sync paths field is already set, do nothing.
assert.Equal(t, tcase.expect, b.Config.Sync.Paths)
})
}
}

View File

@ -0,0 +1,120 @@
package mutator
import (
"context"
"fmt"
"path/filepath"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"
)
type syncInferRoot struct{}
// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the
// paths in the sync configuration. The sync root may be different from the bundle root
// when the user intends to synchronize files outside the bundle root.
//
// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant.
// That is, the sync root must contain the bundle root.
//
// This mutator requires all sync-related paths and patterns to be relative to the bundle root path.
// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator.
func SyncInferRoot() bundle.Mutator {
return &syncInferRoot{}
}
func (m *syncInferRoot) Name() string {
return "SyncInferRoot"
}
// computeRoot finds the innermost path that contains the specified path.
// It traverses up the root path until it finds the innermost path.
// If the path does not exist, it returns an empty string.
//
// See "sync_infer_root_internal_test.go" for examples.
func (m *syncInferRoot) computeRoot(path string, root string) string {
for !filepath.IsLocal(path) {
// Break if we have reached the root of the filesystem.
dir := filepath.Dir(root)
if dir == root {
return ""
}
// Update the sync path as we navigate up the directory tree.
path = filepath.Join(filepath.Base(root), path)
// Move up the directory tree.
root = dir
}
return filepath.Clean(root)
}
func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
var diags diag.Diagnostics
// Use the bundle root path as the starting point for inferring the sync root path.
bundleRootPath := filepath.Clean(b.RootPath)
// Infer the sync root path by looking at each one of the sync paths.
// Every sync path must be a descendant of the final sync root path.
syncRootPath := bundleRootPath
for _, path := range b.Config.Sync.Paths {
computedPath := m.computeRoot(path, bundleRootPath)
if computedPath == "" {
continue
}
// Update sync root path if the computed root path is an ancestor of the current sync root path.
if len(computedPath) < len(syncRootPath) {
syncRootPath = computedPath
}
}
// The new sync root path can only be an ancestor of the previous root path.
// Compute the relative path from the sync root to the bundle root.
rel, err := filepath.Rel(syncRootPath, bundleRootPath)
if err != nil {
return diag.FromErr(err)
}
// If during computation of the sync root path we hit the root of the filesystem,
// then one or more of the sync paths are outside the filesystem.
// Check if this happened by verifying that none of the paths escape the root
// when joined with the sync root path.
for i, path := range b.Config.Sync.Paths {
if filepath.IsLocal(filepath.Join(rel, path)) {
continue
}
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("invalid sync path %q", path),
Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)),
Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))},
})
}
if diags.HasError() {
return diags
}
// Update all paths in the sync configuration to be relative to the sync root.
for i, p := range b.Config.Sync.Paths {
b.Config.Sync.Paths[i] = filepath.Join(rel, p)
}
for i, p := range b.Config.Sync.Include {
b.Config.Sync.Include[i] = filepath.Join(rel, p)
}
for i, p := range b.Config.Sync.Exclude {
b.Config.Sync.Exclude[i] = filepath.Join(rel, p)
}
// Configure the sync root path.
b.SyncRoot = vfs.MustNew(syncRootPath)
b.SyncRootPath = syncRootPath
return nil
}

View File

@ -0,0 +1,72 @@
package mutator
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSyncInferRootInternal_ComputeRoot(t *testing.T) {
s := syncInferRoot{}
tcases := []struct {
path string
root string
out string
}{
{
// Test that "." doesn't change the root.
path: ".",
root: "/tmp/some/dir",
out: "/tmp/some/dir",
},
{
// Test that a subdirectory doesn't change the root.
path: "sub",
root: "/tmp/some/dir",
out: "/tmp/some/dir",
},
{
// Test that a parent directory changes the root.
path: "../common",
root: "/tmp/some/dir",
out: "/tmp/some",
},
{
// Test that a deeply nested parent directory changes the root.
path: "../../../../../../common",
root: "/tmp/some/dir/that/is/very/deeply/nested",
out: "/tmp/some",
},
{
// Test that a parent directory changes the root at the filesystem root boundary.
path: "../common",
root: "/tmp",
out: "/",
},
{
// Test that an invalid parent directory doesn't change the root and returns an empty string.
path: "../common",
root: "/",
out: "",
},
{
// Test that the returned path is cleaned even if the root doesn't change.
path: "sub",
root: "/tmp/some/../dir",
out: "/tmp/dir",
},
{
// Test that a relative root path also works.
path: "../common",
root: "foo/bar",
out: "foo",
},
}
for _, tc := range tcases {
out := s.computeRoot(tc.path, tc.root)
assert.Equal(t, tc.out, filepath.ToSlash(out))
}
}

View File

@ -0,0 +1,198 @@
package mutator_test
import (
"context"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some/dir"), b.SyncRootPath)
// Check that the paths are unchanged.
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_NominalRelative(t *testing.T) {
b := &bundle.Bundle{
RootPath: "./some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
".",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("some/dir"), b.SyncRootPath)
// Check that the paths are unchanged.
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include)
assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_ParentDirectory(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../common",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
assert.Equal(t, []string{filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/bar")}, b.Config.Sync.Include)
assert.Equal(t, []string{filepath.FromSlash("dir/baz"), filepath.FromSlash("dir/qux")}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir/that/is/very/deeply/nested",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../../../../../../common",
},
Include: []string{
"foo",
"bar",
},
Exclude: []string{
"baz",
"qux",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, []string{"common"}, b.Config.Sync.Paths)
assert.Equal(t, []string{
filepath.FromSlash("dir/that/is/very/deeply/nested/foo"),
filepath.FromSlash("dir/that/is/very/deeply/nested/bar"),
}, b.Config.Sync.Include)
assert.Equal(t, []string{
filepath.FromSlash("dir/that/is/very/deeply/nested/baz"),
filepath.FromSlash("dir/that/is/very/deeply/nested/qux"),
}, b.Config.Sync.Exclude)
}
func TestSyncInferRoot_MultiplePaths(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/bundle/root",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"./foo",
"../common",
"./bar",
"../../baz",
},
},
},
}
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
assert.NoError(t, diags.Error())
assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath)
// Check that the paths are updated.
assert.Equal(t, filepath.FromSlash("bundle/root/foo"), b.Config.Sync.Paths[0])
assert.Equal(t, filepath.FromSlash("bundle/common"), b.Config.Sync.Paths[1])
assert.Equal(t, filepath.FromSlash("bundle/root/bar"), b.Config.Sync.Paths[2])
assert.Equal(t, filepath.FromSlash("baz"), b.Config.Sync.Paths[3])
}
func TestSyncInferRoot_Error(t *testing.T) {
b := &bundle.Bundle{
RootPath: "/tmp/some/dir",
Config: config.Root{
Sync: config.Sync{
Paths: []string{
"../../../../error",
"../../../thisworks",
"../../../../../error",
},
},
},
}
bundletest.SetLocation(b, "sync.paths", "databricks.yml")
ctx := context.Background()
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
require.Len(t, diags, 2)
assert.Equal(t, `invalid sync path "../../../../error"`, diags[0].Summary)
assert.Equal(t, "databricks.yml:0:0", diags[0].Locations[0].String())
assert.Equal(t, "sync.paths[0]", diags[0].Paths[0].String())
assert.Equal(t, `invalid sync path "../../../../../error"`, diags[1].Summary)
assert.Equal(t, "databricks.yml:0:0", diags[1].Locations[0].String())
assert.Equal(t, "sync.paths[2]", diags[1].Paths[0].String())
}

View File

@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund
return err
}
internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
internalDirRel, err := filepath.Rel(b.SyncRootPath, internalDir)
if err != nil {
return err
}

View File

@ -56,8 +56,12 @@ func TestGenerateTrampoline(t *testing.T) {
}
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Bundle: config.Bundle{
Target: "development",
},
@ -89,6 +93,6 @@ func TestGenerateTrampoline(t *testing.T) {
require.Equal(t, "Hello from Trampoline", string(bytes))
task := b.Config.Resources.Jobs["test"].Tasks[0]
require.Equal(t, task.NotebookTask.NotebookPath, ".databricks/bundle/development/.internal/notebook_test_to_trampoline")
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_test_to_trampoline", task.NotebookTask.NotebookPath)
require.Nil(t, task.PythonWheelTask)
}

View File

@ -93,14 +93,14 @@ func (t *translateContext) rewritePath(
return nil
}
// Local path must be contained in the bundle root.
// Local path must be contained in the sync root.
// If it isn't, it won't be synchronized into the workspace.
localRelPath, err := filepath.Rel(t.b.RootPath, localPath)
localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath)
if err != nil {
return err
}
if strings.HasPrefix(localRelPath, "..") {
return fmt.Errorf("path %s is not contained in bundle root path", localPath)
return fmt.Errorf("path %s is not contained in sync root path", localPath)
}
// Prefix remote path with its remote root path.
@ -118,7 +118,7 @@ func (t *translateContext) rewritePath(
}
func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("notebook %s not found", literal)
}
@ -134,7 +134,7 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe
}
func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath))
nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath))
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("file %s not found", literal)
}
@ -148,7 +148,7 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat
}
func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
info, err := t.b.BundleRoot.Stat(filepath.ToSlash(localRelPath))
info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
if err != nil {
return "", err
}

View File

@ -41,8 +41,8 @@ func touchEmptyFile(t *testing.T, path string) {
func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -113,8 +113,8 @@ func TestTranslatePaths(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -289,8 +289,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -380,12 +380,12 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
)
}
func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
func TestTranslatePathsOutsideSyncRoot(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -411,15 +411,15 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml"))
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
assert.ErrorContains(t, diags.Error(), "is not contained in bundle root")
assert.ErrorContains(t, diags.Error(), "is not contained in sync root path")
}
func TestJobNotebookDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -449,8 +449,8 @@ func TestJobFileDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -480,8 +480,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
@ -511,8 +511,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
@ -543,8 +543,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -578,8 +578,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -613,8 +613,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "my_file.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -648,8 +648,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_notebook.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle",
@ -684,8 +684,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "env2.py"))
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
@ -724,8 +724,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
func TestTranslatePathWithComplexVariables(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
RootPath: dir,
BundleRoot: vfs.MustNew(dir),
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Variables: map[string]*variable.Variable{
"cluster_libraries": {

View File

@ -1,6 +1,10 @@
package config
type Sync struct {
// Paths contains a list of paths to synchronize relative to the bundle root path.
// If not configured, this defaults to synchronizing everything in the bundle root path (i.e. `.`).
Paths []string `json:"paths,omitempty"`
// Include contains a list of globs evaluated relative to the bundle root path
// to explicitly include files that were excluded by the user's gitignore.
Include []string `json:"include,omitempty"`

View File

@ -28,8 +28,8 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp
}
opts := &sync.SyncOptions{
LocalRoot: rb.BundleRoot(),
Paths: []string{"."},
LocalRoot: rb.SyncRoot(),
Paths: rb.Config().Sync.Paths,
Include: includes,
Exclude: rb.Config().Sync.Exclude,

View File

@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
}
log.Infof(ctx, "Creating new snapshot")
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.BundleRoot), opts)
snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.SyncRoot), opts)
if err != nil {
return diag.FromErr(err)
}

View File

@ -64,6 +64,10 @@ func testStatePull(t *testing.T, opts statePullOpts) {
b := &bundle.Bundle{
RootPath: tmpDir,
BundleRoot: vfs.MustNew(tmpDir),
SyncRootPath: tmpDir,
SyncRoot: vfs.MustNew(tmpDir),
Config: config.Root{
Bundle: config.Bundle{
Target: "default",
@ -81,11 +85,11 @@ func testStatePull(t *testing.T, opts statePullOpts) {
ctx := context.Background()
for _, file := range opts.localFiles {
testutil.Touch(t, b.RootPath, "bar", file)
testutil.Touch(t, b.SyncRootPath, "bar", file)
}
for _, file := range opts.localNotebooks {
testutil.TouchNotebook(t, b.RootPath, "bar", file)
testutil.TouchNotebook(t, b.SyncRootPath, "bar", file)
}
if opts.withExistingSnapshot {

View File

@ -21,7 +21,18 @@ func Initialize() bundle.Mutator {
"initialize",
[]bundle.Mutator{
validate.AllResourcesHaveValues(),
// Update all path fields in the sync block to be relative to the bundle root path.
mutator.RewriteSyncPaths(),
// Configure the default sync path to equal the bundle root if not explicitly configured.
// By default, this means all files in the bundle root directory are synchronized.
mutator.SyncDefaultPath(),
// Figure out if the sync root path is identical or an ancestor of the bundle root path.
// If it is an ancestor, this updates all paths to be relative to the sync root path.
mutator.SyncInferRoot(),
mutator.MergeJobClusters(),
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),

View File

@ -2,7 +2,6 @@ package python
import (
"context"
"path"
"path/filepath"
"testing"
@ -18,11 +17,15 @@ func TestNoTransformByDefault(t *testing.T) {
tmpDir := t.TempDir()
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Bundle: config.Bundle{
Target: "development",
},
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
@ -63,11 +66,15 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
tmpDir := t.TempDir()
b := &bundle.Bundle{
RootPath: tmpDir,
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
SyncRootPath: filepath.Join(tmpDir, "parent"),
Config: config.Root{
Bundle: config.Bundle{
Target: "development",
},
Workspace: config.Workspace{
FilePath: "/Workspace/files",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
@ -102,14 +109,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
task := b.Config.Resources.Jobs["job1"].Tasks[0]
require.Nil(t, task.PythonWheelTask)
require.NotNil(t, task.NotebookTask)
dir, err := b.InternalDir(context.Background())
require.NoError(t, err)
internalDirRel, err := filepath.Rel(b.RootPath, dir)
require.NoError(t, err)
require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath)
require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_job1_key1", task.NotebookTask.NotebookPath)
require.Len(t, task.Libraries, 1)
require.Equal(t, "/Workspace/Users/test@test.com/bundle/dist/test.jar", task.Libraries[0].Jar)

View File

@ -40,6 +40,8 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
diags := bundle.Apply(ctx, b, bundle.Seq(
phases.LoadNamedTarget(env),
mutator.RewriteSyncPaths(),
mutator.SyncDefaultPath(),
mutator.SyncInferRoot(),
mutator.MergeJobClusters(),
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),

View File

@ -0,0 +1,20 @@
bundle:
name: sync_paths
workspace:
host: https://acme.cloud.databricks.com/
sync:
paths:
- src
targets:
development:
sync:
paths:
- development
staging:
sync:
paths:
- staging

View File

@ -0,0 +1,26 @@
bundle:
name: sync_paths
workspace:
host: https://acme.cloud.databricks.com/
targets:
development:
sync:
paths:
- development
staging:
sync:
paths:
- staging
undefined: ~
nil:
sync:
paths: ~
empty:
sync:
paths: []

View File

@ -0,0 +1,10 @@
bundle:
name: shared_code
workspace:
host: https://acme.cloud.databricks.com/
sync:
paths:
- "../common"
- "."

View File

@ -0,0 +1 @@
Placeholder for files to be deployed as part of multiple bundles.

View File

@ -12,14 +12,20 @@ func TestSyncOverride(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/override", "development")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override", "staging")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override", "prod")
assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("src/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
}
@ -28,14 +34,20 @@ func TestSyncOverrideNoRootSync(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/override_no_root", "development")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override_no_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/override_no_root", "prod")
assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude)
}
@ -44,10 +56,14 @@ func TestSyncNil(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/nil", "development")
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Nil(t, b.Config.Sync.Include)
assert.Nil(t, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/nil", "staging")
assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
}
@ -56,10 +72,59 @@ func TestSyncNilRoot(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/nil_root", "development")
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.Nil(t, b.Config.Sync.Include)
assert.Nil(t, b.Config.Sync.Exclude)
b = loadTarget(t, "./sync/nil_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include)
assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude)
}
func TestSyncPaths(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/paths", "development")
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
assert.Equal(t, []string{"src", "development"}, b.Config.Sync.Paths)
b = loadTarget(t, "./sync/paths", "staging")
assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath)
assert.Equal(t, []string{"src", "staging"}, b.Config.Sync.Paths)
}
func TestSyncPathsNoRoot(t *testing.T) {
var b *bundle.Bundle
b = loadTarget(t, "./sync/paths_no_root", "development")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"development"}, b.Config.Sync.Paths)
b = loadTarget(t, "./sync/paths_no_root", "staging")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"staging"}, b.Config.Sync.Paths)
// If not set at all, it defaults to "."
b = loadTarget(t, "./sync/paths_no_root", "undefined")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Equal(t, []string{"."}, b.Config.Sync.Paths)
// If set to nil, it won't sync anything.
b = loadTarget(t, "./sync/paths_no_root", "nil")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Len(t, b.Config.Sync.Paths, 0)
// If set to an empty sequence, it won't sync anything.
b = loadTarget(t, "./sync/paths_no_root", "empty")
assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath)
assert.Len(t, b.Config.Sync.Paths, 0)
}
func TestSyncSharedCode(t *testing.T) {
b := loadTarget(t, "./sync/shared_code/bundle", "default")
assert.Equal(t, filepath.FromSlash("sync/shared_code"), b.SyncRootPath)
assert.ElementsMatch(t, []string{"common", "bundle"}, b.Config.Sync.Paths)
}

View File

@ -17,8 +17,10 @@ import (
func TestSyncOptionsFromBundle(t *testing.T) {
tempDir := t.TempDir()
b := &bundle.Bundle{
RootPath: tempDir,
BundleRoot: vfs.MustNew(tempDir),
RootPath: tempDir,
BundleRoot: vfs.MustNew(tempDir),
SyncRootPath: tempDir,
SyncRoot: vfs.MustNew(tempDir),
Config: config.Root{
Bundle: config.Bundle{
Target: "default",