From 2e3c5ca9ea080fce34d05b40c9b53002daac444d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 16 Aug 2023 15:50:00 +0200 Subject: [PATCH] removed sync include code --- bundle/bundle.go | 17 --------- bundle/config/root.go | 3 -- bundle/config/sync.go | 6 ---- bundle/deploy/files/sync.go | 8 ----- bundle/python/transform.go | 16 +++++++-- cmd/bundle/sync.go | 8 ----- cmd/sync/sync.go | 8 ----- libs/sync/sync.go | 72 ++++++------------------------------- 8 files changed, 24 insertions(+), 114 deletions(-) delete mode 100644 bundle/config/sync.go diff --git a/bundle/bundle.go b/bundle/bundle.go index 1281b4f4..0147883c 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -22,8 +22,6 @@ import ( "github.com/hashicorp/terraform-exec/tfexec" ) -const InternalFolder = ".internal" - type Bundle struct { Config config.Root @@ -153,21 +151,6 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) { return dir, nil } -func (b *Bundle) InternalDir() (string, error) { - cacheDir, err := b.CacheDir() - if err != nil { - return "", err - } - - dir := filepath.Join(cacheDir, InternalFolder) - err = os.MkdirAll(dir, 0700) - if err != nil { - return dir, err - } - - return dir, nil -} - func (b *Bundle) GitRepository() (*git.Repository, error) { rootPath, err := folders.FindDirWithLeaf(b.Config.Path, ".git") if err != nil { diff --git a/bundle/config/root.go b/bundle/config/root.go index 7db6771d..52f88737 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -74,9 +74,6 @@ type Root struct { // If not specified, the code below initializes this field with a // single default-initialized environment called "default". Environments map[string]*Environment `json:"environments,omitempty"` - - // Sync section specifies options for files syncronisation - Sync Sync `json:"sync,omitempty"` } func Load(path string) (*Root, error) { diff --git a/bundle/config/sync.go b/bundle/config/sync.go deleted file mode 100644 index ce420f3b..00000000 --- a/bundle/config/sync.go +++ /dev/null @@ -1,6 +0,0 @@ -package config - -type Sync struct { - Include []string `json:"include,omitempty"` - Exclude []string `json:"exclude,omitempty"` -} diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index 90ea2252..d0987d63 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -14,21 +14,13 @@ func getSync(ctx context.Context, b *bundle.Bundle) (*sync.Sync, error) { return nil, fmt.Errorf("cannot get bundle cache directory: %w", err) } - internalDir, err := b.InternalDir() - if err != nil { - return nil, fmt.Errorf("cannot get bundle internal directory: %w", err) - } - opts := sync.SyncOptions{ LocalPath: b.Config.Path, RemotePath: b.Config.Workspace.FilesPath, - Include: b.Config.Sync.Include, - Exclude: b.Config.Sync.Exclude, Full: false, CurrentUser: b.Config.Workspace.CurrentUser.User, - InternalDir: internalDir, SnapshotBasePath: cacheDir, WorkspaceClient: b.WorkspaceClient(), } diff --git a/bundle/python/transform.go b/bundle/python/transform.go index 659a892e..42a5cf6e 100644 --- a/bundle/python/transform.go +++ b/bundle/python/transform.go @@ -64,7 +64,7 @@ func (m *transform) Apply(ctx context.Context, b *bundle.Bundle) error { return err } - internalDir, err := b.InternalDir() + internalDir, err := getInternalDir(b) if err != nil { return err } @@ -85,11 +85,21 @@ func (m *transform) Apply(ctx context.Context, b *bundle.Bundle) error { return nil } -func generateNotebookWrapper(b *bundle.Bundle, task *jobs.PythonWheelTask, libraries []compute.Library) (string, error) { - internalDir, err := b.InternalDir() +func getInternalDir(b *bundle.Bundle) (string, error) { + cacheDir, err := b.CacheDir() if err != nil { return "", err } + internalDir := filepath.Join(cacheDir, ".internal") + return internalDir, nil +} + +func generateNotebookWrapper(b *bundle.Bundle, task *jobs.PythonWheelTask, libraries []compute.Library) (string, error) { + internalDir, err := getInternalDir(b) + if err != nil { + return "", err + } + notebookName := fmt.Sprintf("notebook_%s_%s", task.PackageName, task.EntryPoint) path := filepath.Join(internalDir, notebookName+".py") diff --git a/cmd/bundle/sync.go b/cmd/bundle/sync.go index 3c503c6b..2fff7baf 100644 --- a/cmd/bundle/sync.go +++ b/cmd/bundle/sync.go @@ -23,20 +23,12 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) return nil, fmt.Errorf("cannot get bundle cache directory: %w", err) } - internalDir, err := b.InternalDir() - if err != nil { - return nil, fmt.Errorf("cannot get bundle internal directory: %w", err) - } - opts := sync.SyncOptions{ LocalPath: b.Config.Path, RemotePath: b.Config.Workspace.FilesPath, - Include: b.Config.Sync.Include, - Exclude: b.Config.Sync.Exclude, Full: f.full, PollInterval: f.interval, - InternalDir: internalDir, SnapshotBasePath: cacheDir, WorkspaceClient: b.WorkspaceClient(), } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 8a08b30d..a9380b75 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -35,21 +35,13 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b * return nil, fmt.Errorf("cannot get bundle cache directory: %w", err) } - internalDir, err := b.InternalDir() - if err != nil { - return nil, fmt.Errorf("cannot get bundle internal directory: %w", err) - } - opts := sync.SyncOptions{ LocalPath: b.Config.Path, RemotePath: b.Config.Workspace.FilesPath, - Include: b.Config.Sync.Include, - Exclude: b.Config.Sync.Exclude, Full: f.full, PollInterval: f.interval, - InternalDir: internalDir, SnapshotBasePath: cacheDir, WorkspaceClient: b.WorkspaceClient(), } diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 3a00a192..4dc03c57 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -3,27 +3,21 @@ package sync import ( "context" "fmt" - "path/filepath" "time" "github.com/databricks/cli/libs/filer" - "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" - "golang.org/x/exp/slices" ) type SyncOptions struct { LocalPath string RemotePath string - Include []string - Exclude []string Full bool - InternalDir string SnapshotBasePath string PollInterval time.Duration @@ -38,9 +32,7 @@ type SyncOptions struct { type Sync struct { *SyncOptions - fileSet *git.FileSet - includeFileSet *fileset.GlobSet - excludeFileSet *fileset.GlobSet + fileSet *git.FileSet snapshot *Snapshot filer filer.Filer @@ -61,17 +53,6 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } - includes := []string{} - if opts.InternalDir != "" { - includes = append(includes, filepath.Join(opts.InternalDir, "*.*")) - } - if opts.Include != nil { - includes = append(includes, opts.Include...) - } - - includeFileSet := fileset.NewGlobSet(opts.LocalPath, includes) - excludeFileSet := fileset.NewGlobSet(opts.LocalPath, opts.Exclude) - // Verify that the remote path we're about to synchronize to is valid and allowed. err = EnsureRemotePathIsUsable(ctx, opts.WorkspaceClient, opts.RemotePath, opts.CurrentUser) if err != nil { @@ -108,13 +89,11 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return &Sync{ SyncOptions: &opts, - fileSet: fileSet, - includeFileSet: includeFileSet, - excludeFileSet: excludeFileSet, - snapshot: snapshot, - filer: filer, - notifier: &NopNotifier{}, - seq: 0, + fileSet: fileSet, + snapshot: snapshot, + filer: filer, + notifier: &NopNotifier{}, + seq: 0, }, nil } @@ -154,44 +133,15 @@ func (s *Sync) notifyComplete(ctx context.Context, d diff) { } func (s *Sync) RunOnce(ctx context.Context) error { - all := make([]fileset.File, 0) - if s.SyncOptions.Include == nil { - // tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement - // https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418 - gitFiles, err := s.fileSet.All() - if err != nil { - log.Errorf(ctx, "cannot list files: %s", err) - return err - } - all = append(all, gitFiles...) - } - - include, err := s.includeFileSet.All() + // tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement + // https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418 + all, err := s.fileSet.All() if err != nil { - log.Errorf(ctx, "cannot list include files: %s", err) + log.Errorf(ctx, "cannot list files: %s", err) return err } - all = append(all, include...) - - exclude, err := s.excludeFileSet.All() - if err != nil { - log.Errorf(ctx, "cannot list exclude files: %s", err) - return err - } - - files := make([]fileset.File, 0) - for _, f := range all { - if slices.ContainsFunc(exclude, func(a fileset.File) bool { - return a.Absolute == f.Absolute - }) { - continue - } - - files = append(files, f) - } - - change, err := s.snapshot.diff(ctx, files) + change, err := s.snapshot.diff(ctx, all) if err != nil { return err }