From fc46d21f8b3bdfbc56d3bf3d1ce2d4a320ddc5c7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 23 Jan 2023 13:52:39 +0100 Subject: [PATCH] Move sync logic from cmd/sync to libs/sync (#173) Mechanical change. Ported global variables the logic relied on to a new `sync.Sync` struct. --- cmd/sync/sync.go | 15 ++++++----- cmd/sync/sync_test.go | 12 --------- internal/sync_test.go | 3 ++- {cmd => libs}/sync/repofiles/repofiles.go | 0 .../sync/repofiles/repofiles_test.go | 0 {cmd => libs}/sync/snapshot.go | 0 {cmd => libs}/sync/snapshot_test.go | 0 libs/sync/sync.go | 26 +++++++++++++++++++ {cmd => libs}/sync/watchdog.go | 14 ++++++---- 9 files changed, 46 insertions(+), 24 deletions(-) delete mode 100644 cmd/sync/sync_test.go rename {cmd => libs}/sync/repofiles/repofiles.go (100%) rename {cmd => libs}/sync/repofiles/repofiles_test.go (100%) rename {cmd => libs}/sync/snapshot.go (100%) rename {cmd => libs}/sync/snapshot_test.go (100%) create mode 100644 libs/sync/sync.go rename {cmd => libs}/sync/watchdog.go (93%) diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 8e97b5bc..798a5681 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -8,8 +8,8 @@ import ( "time" "github.com/databricks/bricks/cmd/root" - "github.com/databricks/bricks/cmd/sync/repofiles" "github.com/databricks/bricks/git" + "github.com/databricks/bricks/libs/sync" "github.com/databricks/bricks/project" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -120,11 +120,14 @@ var syncCmd = &cobra.Command{ return err } - root := prj.Root() - repoFiles := repofiles.Create(*remotePath, root, wsc) - syncCallback := syncCallback(ctx, repoFiles) - err = spawnWatchdog(ctx, *interval, syncCallback, *remotePath) - return err + s := sync.Sync{ + LocalPath: prj.Root(), + RemotePath: *remotePath, + PersistSnapshot: *persistSnapshot, + PollInterval: *interval, + } + + return s.RunWatchdog(ctx, wsc) }, } diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go deleted file mode 100644 index 17dc78a4..00000000 --- a/cmd/sync/sync_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package sync - -import ( - "testing" -) - -func TestItSyncs(t *testing.T) { - // ctx := context.Background() - // root.RootCmd.SetArgs([]string{"sync"}) - // err := root.RootCmd.ExecuteContext(ctx) - // assert.NoError(t, err) -} diff --git a/internal/sync_test.go b/internal/sync_test.go index 3ceec2d3..1c5ac6f8 100644 --- a/internal/sync_test.go +++ b/internal/sync_test.go @@ -14,7 +14,8 @@ import ( "testing" "time" - "github.com/databricks/bricks/cmd/sync" + _ "github.com/databricks/bricks/cmd/sync" + "github.com/databricks/bricks/libs/sync" "github.com/databricks/bricks/libs/testfile" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" diff --git a/cmd/sync/repofiles/repofiles.go b/libs/sync/repofiles/repofiles.go similarity index 100% rename from cmd/sync/repofiles/repofiles.go rename to libs/sync/repofiles/repofiles.go diff --git a/cmd/sync/repofiles/repofiles_test.go b/libs/sync/repofiles/repofiles_test.go similarity index 100% rename from cmd/sync/repofiles/repofiles_test.go rename to libs/sync/repofiles/repofiles_test.go diff --git a/cmd/sync/snapshot.go b/libs/sync/snapshot.go similarity index 100% rename from cmd/sync/snapshot.go rename to libs/sync/snapshot.go diff --git a/cmd/sync/snapshot_test.go b/libs/sync/snapshot_test.go similarity index 100% rename from cmd/sync/snapshot_test.go rename to libs/sync/snapshot_test.go diff --git a/libs/sync/sync.go b/libs/sync/sync.go new file mode 100644 index 00000000..9a247254 --- /dev/null +++ b/libs/sync/sync.go @@ -0,0 +1,26 @@ +package sync + +import ( + "context" + "time" + + "github.com/databricks/bricks/libs/sync/repofiles" + "github.com/databricks/databricks-sdk-go" +) + +type Sync struct { + LocalPath string + RemotePath string + + PersistSnapshot bool + + PollInterval time.Duration +} + +// RunWatchdog kicks off a polling loop to monitor local changes and synchronize +// them to the remote workspace path. +func (s *Sync) RunWatchdog(ctx context.Context, wsc *databricks.WorkspaceClient) error { + repoFiles := repofiles.Create(s.RemotePath, s.LocalPath, wsc) + syncCallback := syncCallback(ctx, repoFiles) + return spawnWatchdog(ctx, s.PollInterval, syncCallback, s.RemotePath, s.PersistSnapshot) +} diff --git a/cmd/sync/watchdog.go b/libs/sync/watchdog.go similarity index 93% rename from cmd/sync/watchdog.go rename to libs/sync/watchdog.go index 616fddc3..8e45b0a6 100644 --- a/cmd/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/databricks/bricks/cmd/sync/repofiles" + "github.com/databricks/bricks/libs/sync/repofiles" "github.com/databricks/bricks/project" "golang.org/x/sync/errgroup" ) @@ -16,6 +16,8 @@ type watchdog struct { ticker *time.Ticker wg sync.WaitGroup failure error // data race? make channel? + + persistSnapshot bool } // See https://docs.databricks.com/resources/limits.html#limits-api-rate-limits for per api @@ -70,9 +72,11 @@ func syncCallback(ctx context.Context, repoFiles *repofiles.RepoFiles) func(loca func spawnWatchdog(ctx context.Context, interval time.Duration, applyDiff func(diff) error, - remotePath string) error { + remotePath string, + persistSnapshot bool) error { w := &watchdog{ - ticker: time.NewTicker(interval), + ticker: time.NewTicker(interval), + persistSnapshot: persistSnapshot, } w.wg.Add(1) go w.main(ctx, applyDiff, remotePath) @@ -90,7 +94,7 @@ func (w *watchdog) main(ctx context.Context, applyDiff func(diff) error, remoteP w.failure = err return } - if *persistSnapshot { + if w.persistSnapshot { err := snapshot.loadSnapshot(ctx) if err != nil { log.Printf("[ERROR] cannot load snapshot: %s", err) @@ -128,7 +132,7 @@ func (w *watchdog) main(ctx context.Context, applyDiff func(diff) error, remoteP w.failure = err return } - if *persistSnapshot { + if w.persistSnapshot { err = snapshot.storeSnapshot(ctx) if err != nil { log.Printf("[ERROR] cannot store snapshot: %s", err)