Make sync command work in bundle context; reorder args (#207)

Invoke with `bricks sync SRC DST`.

In bundle context `SRC` and `DST` arguments are taken from bundle configuration.

This PR adds `bricks bundle sync` to disambiguate between the two.
Once the VS Code extension is bundle aware they can again be consolidated.
Consolidating them today would regress the VS Code experience if a
`bundle.yml` file is present in the file tree.
This commit is contained in:
Pieter Noordhuis 2023-02-20 11:33:30 +01:00 committed by GitHub
parent ded7d8c404
commit 1715a987cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 281 additions and 110 deletions

View File

@ -13,11 +13,21 @@ func Context(ctx context.Context, b *Bundle) context.Context {
return context.WithValue(ctx, &bundleKey, b) return context.WithValue(ctx, &bundleKey, b)
} }
// GetOrNil returns the bundle as configured on the context.
// It returns nil if it isn't configured.
func GetOrNil(ctx context.Context) *Bundle {
bundle, ok := ctx.Value(&bundleKey).(*Bundle)
if !ok {
return nil
}
return bundle
}
// Get returns the bundle as configured on the context. // Get returns the bundle as configured on the context.
// It panics if it isn't configured. // It panics if it isn't configured.
func Get(ctx context.Context) *Bundle { func Get(ctx context.Context) *Bundle {
bundle, ok := ctx.Value(&bundleKey).(*Bundle) bundle := GetOrNil(ctx)
if !ok { if bundle == nil {
panic("context not configured with bundle") panic("context not configured with bundle")
} }
return bundle return bundle

80
cmd/bundle/sync.go Normal file
View File

@ -0,0 +1,80 @@
package bundle
import (
"fmt"
"log"
"time"
"github.com/databricks/bricks/bundle"
"github.com/databricks/bricks/bundle/phases"
"github.com/databricks/bricks/cmd/root"
"github.com/databricks/bricks/libs/sync"
"github.com/spf13/cobra"
)
func syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) {
cacheDir, err := b.CacheDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilePath.Workspace,
Full: full,
PollInterval: interval,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}
return &opts, nil
}
var syncCmd = &cobra.Command{
Use: "sync [flags]",
Short: "Synchronize bundle tree to the workspace",
Args: cobra.NoArgs,
PreRunE: root.MustConfigureBundle,
RunE: func(cmd *cobra.Command, args []string) error {
b := bundle.Get(cmd.Context())
// Run initialize phase to make sure paths are set.
err := bundle.Apply(cmd.Context(), b, []bundle.Mutator{
phases.Initialize(),
})
if err != nil {
return err
}
opts, err := syncOptionsFromBundle(cmd, b)
if err != nil {
return err
}
ctx := cmd.Context()
s, err := sync.New(ctx, *opts)
if err != nil {
return err
}
log.Printf("[INFO] Remote file sync location: %v", opts.RemotePath)
if watch {
return s.RunContinuous(ctx)
}
return s.RunOnce(ctx)
},
}
var interval time.Duration
var full bool
var watch bool
func init() {
AddCommand(syncCmd)
syncCmd.Flags().DurationVar(&interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
syncCmd.Flags().BoolVar(&full, "full", false, "perform full synchronization (default is incremental)")
syncCmd.Flags().BoolVar(&watch, "watch", false, "watch local file system for changes")
}

View File

@ -1,63 +1,103 @@
package sync package sync
import ( import (
"flag"
"fmt" "fmt"
"log" "log"
"path/filepath"
"time" "time"
"github.com/databricks/bricks/bundle"
"github.com/databricks/bricks/cmd/root" "github.com/databricks/bricks/cmd/root"
"github.com/databricks/bricks/libs/git"
"github.com/databricks/bricks/libs/sync" "github.com/databricks/bricks/libs/sync"
"github.com/databricks/bricks/project" "github.com/databricks/databricks-sdk-go"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
// syncCmd represents the sync command func syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
if len(args) > 0 {
return nil, fmt.Errorf("SRC and DST are not configurable in the context of a bundle")
}
cacheDir, err := b.CacheDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilePath.Workspace,
Full: full,
PollInterval: interval,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}
return &opts, nil
}
func syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions, error) {
if len(args) != 2 {
return nil, flag.ErrHelp
}
opts := sync.SyncOptions{
LocalPath: args[0],
RemotePath: args[1],
Full: full,
PollInterval: interval,
// We keep existing behavior for VS Code extension where if there is
// no bundle defined, we store the snapshots in `.databricks`.
// The sync code will automatically create this directory if it doesn't
// exist and add it to the `.gitignore` file in the root.
SnapshotBasePath: filepath.Join(args[0], ".databricks"),
WorkspaceClient: databricks.Must(databricks.NewWorkspaceClient()),
}
return &opts, nil
}
var syncCmd = &cobra.Command{ var syncCmd = &cobra.Command{
Use: "sync", Use: "sync [flags] SRC DST",
Short: "Synchronize a local directory to a workspace directory", Short: "Synchronize a local directory to a workspace directory",
Args: cobra.MaximumNArgs(2),
PreRunE: project.Configure, // PreRunE: root.TryConfigureBundle,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
var opts *sync.SyncOptions
var err error
//
// To be uncommented and used once our VS Code extension is bundle aware.
// Until then, this could interfere with extension usage where a `bundle.yml` file is present.
// See https://github.com/databricks/bricks/pull/207.
//
// b := bundle.GetOrNil(cmd.Context())
// if b != nil {
// // Run initialize phase to make sure paths are set.
// err = bundle.Apply(cmd.Context(), b, []bundle.Mutator{
// phases.Initialize(),
// })
// if err != nil {
// return err
// }
// opts, err = syncOptionsFromBundle(cmd, args, b)
// } else {
opts, err = syncOptionsFromArgs(cmd, args)
// }
if err != nil {
return err
}
ctx := cmd.Context() ctx := cmd.Context()
prj := project.Get(ctx) s, err := sync.New(ctx, *opts)
wsc := prj.WorkspacesClient()
me, err := prj.Me()
if err != nil { if err != nil {
return err return err
} }
if *remotePath == "" { log.Printf("[INFO] Remote file sync location: %v", opts.RemotePath)
repositoryName, err := git.RepositoryName()
if err != nil {
return err
}
*remotePath = fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName)
}
log.Printf("[INFO] Remote file sync location: %v", *remotePath) if watch {
cacheDir, err := prj.CacheDir()
if err != nil {
return err
}
opts := sync.SyncOptions{
LocalPath: prj.Root(),
RemotePath: *remotePath,
Full: *full,
SnapshotBasePath: cacheDir,
PollInterval: *interval,
WorkspaceClient: wsc,
}
s, err := sync.New(ctx, opts)
if err != nil {
return err
}
if *watch {
return s.RunContinuous(ctx) return s.RunContinuous(ctx)
} }
@ -66,17 +106,13 @@ var syncCmd = &cobra.Command{
} }
// project files polling interval // project files polling interval
var interval *time.Duration var interval time.Duration
var full bool
var remotePath *string var watch bool
var full *bool
var watch *bool
func init() { func init() {
root.RootCmd.AddCommand(syncCmd) root.RootCmd.AddCommand(syncCmd)
interval = syncCmd.Flags().Duration("interval", 1*time.Second, "file system polling interval (for --watch)") syncCmd.Flags().DurationVar(&interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
remotePath = syncCmd.Flags().String("remote-path", "", "remote path to store repo in. eg: /Repos/me@example.com/test-repo") syncCmd.Flags().BoolVar(&full, "full", false, "perform full synchronization (default is incremental)")
full = syncCmd.Flags().Bool("full", false, "perform full synchronization (default is incremental)") syncCmd.Flags().BoolVar(&watch, "watch", false, "watch local file system for changes")
watch = syncCmd.Flags().Bool("watch", false, "watch local file system for changes")
} }

55
cmd/sync/sync_test.go Normal file
View File

@ -0,0 +1,55 @@
package sync
import (
"flag"
"path/filepath"
"testing"
"github.com/databricks/bricks/bundle"
"github.com/databricks/bricks/bundle/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSyncOptionsFromBundle(t *testing.T) {
tempDir := t.TempDir()
b := &bundle.Bundle{
Config: config.Root{
Path: tempDir,
Bundle: config.Bundle{
Environment: "default",
},
Workspace: config.Workspace{
FilePath: config.PathLike{
Workspace: "/Users/jane@doe.com/path",
},
},
},
}
opts, err := syncOptionsFromBundle(syncCmd, []string{}, b)
require.NoError(t, err)
assert.Equal(t, tempDir, opts.LocalPath)
assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath)
assert.Equal(t, filepath.Join(tempDir, ".databricks", "bundle", "default"), opts.SnapshotBasePath)
assert.NotNil(t, opts.WorkspaceClient)
}
func TestSyncOptionsFromArgsRequiredTwoArgs(t *testing.T) {
var err error
_, err = syncOptionsFromArgs(syncCmd, []string{})
require.ErrorIs(t, err, flag.ErrHelp)
_, err = syncOptionsFromArgs(syncCmd, []string{"foo"})
require.ErrorIs(t, err, flag.ErrHelp)
_, err = syncOptionsFromArgs(syncCmd, []string{"foo", "bar", "qux"})
require.ErrorIs(t, err, flag.ErrHelp)
}
func TestSyncOptionsFromArgs(t *testing.T) {
opts, err := syncOptionsFromArgs(syncCmd, []string{"/local", "/remote"})
require.NoError(t, err)
assert.Equal(t, "/local", opts.LocalPath)
assert.Equal(t, "/remote", opts.RemotePath)
}

View File

@ -25,12 +25,16 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
var (
repoUrl = "https://github.com/databricks/databricks-empty-ide-project.git"
repoFiles = []string{"README-IDE.md"}
)
// This test needs auth env vars to run. // This test needs auth env vars to run.
// Please run using the deco env test or deco env shell // Please run using the deco env test or deco env shell
func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Context) (localRoot, remoteRoot string) { func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Context) (localRoot, remoteRoot string) {
me, err := wsc.CurrentUser.Me(ctx) me, err := wsc.CurrentUser.Me(ctx)
require.NoError(t, err) require.NoError(t, err)
repoUrl := "https://github.com/shreyas-goenka/empty-repo.git"
repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-")) repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-"))
repoInfo, err := wsc.Repos.Create(ctx, repos.CreateRepo{ repoInfo, err := wsc.Repos.Create(ctx, repos.CreateRepo{
@ -45,15 +49,14 @@ func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Contex
assert.NoError(t, err) assert.NoError(t, err)
}) })
// clone public empty remote repo
tempDir := t.TempDir() tempDir := t.TempDir()
cmd := exec.Command("git", "clone", repoUrl)
cmd.Dir = tempDir
err = cmd.Run()
require.NoError(t, err)
localRoot = filepath.Join(tempDir, "empty-repo") localRoot = filepath.Join(tempDir, "empty-repo")
remoteRoot = repoPath remoteRoot = repoPath
// clone public empty remote repo
cmd := exec.Command("git", "clone", repoUrl, localRoot)
err = cmd.Run()
require.NoError(t, err)
return localRoot, remoteRoot return localRoot, remoteRoot
} }
@ -167,8 +170,7 @@ func TestAccFullFileSync(t *testing.T) {
localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx) localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx)
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--full", "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--full", "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -179,16 +181,14 @@ func TestAccFullFileSync(t *testing.T) {
remoteRoot: remoteRepoPath, remoteRoot: remoteRepoPath,
} }
// .gitkeep comes from cloning during repo setup // .gitignore is created by the sync process to enforce .databricks is not synced
// .gitignore is created by the sync process to enforce .databricks is not assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
// synced
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"})
// New file // New file
localFilePath := filepath.Join(localRepoPath, "foo.txt") localFilePath := filepath.Join(localRepoPath, "foo.txt")
f := testfile.CreateFile(t, localFilePath) f := testfile.CreateFile(t, localFilePath)
defer f.Close(t) defer f.Close(t)
assertSync.remoteDirContent(ctx, "", []string{"foo.txt", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore"))
assertSync.remoteFileContent(ctx, "foo.txt", "") assertSync.remoteFileContent(ctx, "foo.txt", "")
// Write to file // Write to file
@ -201,7 +201,7 @@ func TestAccFullFileSync(t *testing.T) {
// delete // delete
f.Remove(t) f.Remove(t)
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
} }
func TestAccIncrementalFileSync(t *testing.T) { func TestAccIncrementalFileSync(t *testing.T) {
@ -213,8 +213,7 @@ func TestAccIncrementalFileSync(t *testing.T) {
localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx) localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx)
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -225,18 +224,16 @@ func TestAccIncrementalFileSync(t *testing.T) {
remoteRoot: remoteRepoPath, remoteRoot: remoteRepoPath,
} }
// .gitkeep comes from cloning during repo setup // .gitignore is created by the sync process to enforce .databricks is not synced
// .gitignore is created by the sync process to enforce .databricks is not assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
// synced
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"})
// New file // New file
localFilePath := filepath.Join(localRepoPath, "foo.txt") localFilePath := filepath.Join(localRepoPath, "foo.txt")
f := testfile.CreateFile(t, localFilePath) f := testfile.CreateFile(t, localFilePath)
defer f.Close(t) defer f.Close(t)
assertSync.remoteDirContent(ctx, "", []string{"foo.txt", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore"))
assertSync.remoteFileContent(ctx, "foo.txt", "") assertSync.remoteFileContent(ctx, "foo.txt", "")
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo.txt"}) assertSync.snapshotContains(append(repoFiles, "foo.txt", ".gitignore"))
// Write to file // Write to file
f.Overwrite(t, `{"statement": "Mi Gente"}`) f.Overwrite(t, `{"statement": "Mi Gente"}`)
@ -248,8 +245,8 @@ func TestAccIncrementalFileSync(t *testing.T) {
// delete // delete
f.Remove(t) f.Remove(t)
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.snapshotContains([]string{".gitkeep", ".gitignore"}) assertSync.snapshotContains(append(repoFiles, ".gitignore"))
} }
func TestAccNestedFolderSync(t *testing.T) { func TestAccNestedFolderSync(t *testing.T) {
@ -261,8 +258,7 @@ func TestAccNestedFolderSync(t *testing.T) {
localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx) localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx)
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -273,10 +269,8 @@ func TestAccNestedFolderSync(t *testing.T) {
remoteRoot: remoteRepoPath, remoteRoot: remoteRepoPath,
} }
// .gitkeep comes from cloning during repo setup // .gitignore is created by the sync process to enforce .databricks is not synced
// .gitignore is created by the sync process to enforce .databricks is not assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
// synced
assertSync.remoteDirContent(ctx, "/", []string{".gitkeep", ".gitignore"})
// New file // New file
localFilePath := filepath.Join(localRepoPath, "dir1/dir2/dir3/foo.txt") localFilePath := filepath.Join(localRepoPath, "dir1/dir2/dir3/foo.txt")
@ -284,17 +278,17 @@ func TestAccNestedFolderSync(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
f := testfile.CreateFile(t, localFilePath) f := testfile.CreateFile(t, localFilePath)
defer f.Close(t) defer f.Close(t)
assertSync.remoteDirContent(ctx, "", []string{"dir1", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "dir1"))
assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"}) assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"})
assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"}) assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"})
assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"}) assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"})
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", filepath.FromSlash("dir1/dir2/dir3/foo.txt")}) assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("dir1/dir2/dir3/foo.txt")))
// delete // delete
f.Remove(t) f.Remove(t)
// directories are not cleaned up right now. This is not ideal // directories are not cleaned up right now. This is not ideal
assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{}) assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{})
assertSync.snapshotContains([]string{".gitkeep", ".gitignore"}) assertSync.snapshotContains(append(repoFiles, ".gitignore"))
} }
// sync does not clean up empty directories from the workspace file system. // sync does not clean up empty directories from the workspace file system.
@ -315,8 +309,7 @@ func TestAccIncrementalFileOverwritesFolder(t *testing.T) {
localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx) localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx)
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -333,22 +326,22 @@ func TestAccIncrementalFileOverwritesFolder(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
f := testfile.CreateFile(t, localFilePath) f := testfile.CreateFile(t, localFilePath)
defer f.Close(t) defer f.Close(t)
assertSync.remoteDirContent(ctx, "", []string{"foo", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"}) assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"})
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", filepath.FromSlash("foo/bar.txt")}) assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("foo/bar.txt")))
// delete foo/bar.txt // delete foo/bar.txt
f.Remove(t) f.Remove(t)
os.Remove(filepath.Join(localRepoPath, "foo")) os.Remove(filepath.Join(localRepoPath, "foo"))
assertSync.remoteDirContent(ctx, "foo", []string{}) assertSync.remoteDirContent(ctx, "foo", []string{})
assertSync.objectType(ctx, "foo", "DIRECTORY") assertSync.objectType(ctx, "foo", "DIRECTORY")
assertSync.snapshotContains([]string{".gitkeep", ".gitignore"}) assertSync.snapshotContains(append(repoFiles, ".gitignore"))
f2 := testfile.CreateFile(t, filepath.Join(localRepoPath, "foo")) f2 := testfile.CreateFile(t, filepath.Join(localRepoPath, "foo"))
defer f2.Close(t) defer f2.Close(t)
assertSync.remoteDirContent(ctx, "", []string{"foo", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.objectType(ctx, "foo", "FILE") assertSync.objectType(ctx, "foo", "FILE")
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo"}) assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo"))
} }
func TestAccIncrementalSyncPythonNotebookToFile(t *testing.T) { func TestAccIncrementalSyncPythonNotebookToFile(t *testing.T) {
@ -366,8 +359,7 @@ func TestAccIncrementalSyncPythonNotebookToFile(t *testing.T) {
f.Overwrite(t, "# Databricks notebook source") f.Overwrite(t, "# Databricks notebook source")
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -379,21 +371,21 @@ func TestAccIncrementalSyncPythonNotebookToFile(t *testing.T) {
} }
// notebook was uploaded properly // notebook was uploaded properly
assertSync.remoteDirContent(ctx, "", []string{"foo", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON") assertSync.language(ctx, "foo", "PYTHON")
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo.py"}) assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
// convert to vanilla python file // convert to vanilla python file
f.Overwrite(t, "# No longer a python notebook") f.Overwrite(t, "# No longer a python notebook")
assertSync.objectType(ctx, "foo.py", "FILE") assertSync.objectType(ctx, "foo.py", "FILE")
assertSync.remoteDirContent(ctx, "", []string{"foo.py", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py"))
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo.py"}) assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
// delete the vanilla python file // delete the vanilla python file
f.Remove(t) f.Remove(t)
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.snapshotContains([]string{".gitkeep", ".gitignore"}) assertSync.snapshotContains(append(repoFiles, ".gitignore"))
} }
func TestAccIncrementalSyncFileToPythonNotebook(t *testing.T) { func TestAccIncrementalSyncFileToPythonNotebook(t *testing.T) {
@ -405,8 +397,7 @@ func TestAccIncrementalSyncFileToPythonNotebook(t *testing.T) {
localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx) localRepoPath, remoteRepoPath := setupRepo(t, wsc, ctx)
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -423,16 +414,16 @@ func TestAccIncrementalSyncFileToPythonNotebook(t *testing.T) {
defer f.Close(t) defer f.Close(t)
// assert file upload // assert file upload
assertSync.remoteDirContent(ctx, "", []string{"foo.py", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py"))
assertSync.objectType(ctx, "foo.py", "FILE") assertSync.objectType(ctx, "foo.py", "FILE")
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo.py"}) assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
// convert to notebook // convert to notebook
f.Overwrite(t, "# Databricks notebook source") f.Overwrite(t, "# Databricks notebook source")
assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON") assertSync.language(ctx, "foo", "PYTHON")
assertSync.remoteDirContent(ctx, "", []string{"foo", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.snapshotContains([]string{".gitkeep", ".gitignore", "foo.py"}) assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
} }
func TestAccIncrementalSyncPythonNotebookDelete(t *testing.T) { func TestAccIncrementalSyncPythonNotebookDelete(t *testing.T) {
@ -450,8 +441,7 @@ func TestAccIncrementalSyncPythonNotebookDelete(t *testing.T) {
f.Overwrite(t, "# Databricks notebook source") f.Overwrite(t, "# Databricks notebook source")
// Run `bricks sync` in the background. // Run `bricks sync` in the background.
t.Setenv("BRICKS_ROOT", localRepoPath) c := NewCobraTestRunner(t, "sync", localRepoPath, remoteRepoPath, "--watch")
c := NewCobraTestRunner(t, "sync", "--remote-path", remoteRepoPath, "--watch")
c.RunBackground() c.RunBackground()
assertSync := assertSync{ assertSync := assertSync{
@ -463,11 +453,11 @@ func TestAccIncrementalSyncPythonNotebookDelete(t *testing.T) {
} }
// notebook was uploaded properly // notebook was uploaded properly
assertSync.remoteDirContent(ctx, "", []string{"foo", ".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.objectType(ctx, "foo", "NOTEBOOK") assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON") assertSync.language(ctx, "foo", "PYTHON")
// Delete notebook // Delete notebook
f.Remove(t) f.Remove(t)
assertSync.remoteDirContent(ctx, "", []string{".gitkeep", ".gitignore"}) assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
} }

View File

@ -79,7 +79,7 @@ func GetFileName(host, remotePath string) string {
func SnapshotPath(opts *SyncOptions) (string, error) { func SnapshotPath(opts *SyncOptions) (string, error) {
snapshotDir := filepath.Join(opts.SnapshotBasePath, syncSnapshotDirName) snapshotDir := filepath.Join(opts.SnapshotBasePath, syncSnapshotDirName)
if _, err := os.Stat(snapshotDir); os.IsNotExist(err) { if _, err := os.Stat(snapshotDir); os.IsNotExist(err) {
err = os.Mkdir(snapshotDir, os.ModeDir|os.ModePerm) err = os.MkdirAll(snapshotDir, 0755)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create config directory: %s", err) return "", fmt.Errorf("failed to create config directory: %s", err)
} }