steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

removed sync include code

This commit is contained in:
Andrew Nester 2023-08-16 15:50:00 +02:00
parent 2be69af604
commit 2e3c5ca9ea
No known key found for this signature in database
GPG Key ID: 12BC628A44B7DA57
8 changed files with 24 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
bundle/bundle.go
View File

@ -22,8 +22,6 @@ import (
"github.com/hashicorp/terraform-exec/tfexec"
)
const InternalFolder = ".internal"
type Bundle struct {
Config config.Root
@ -153,21 +151,6 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
return dir, nil
}
func (b *Bundle) InternalDir() (string, error) {
cacheDir, err := b.CacheDir()
if err != nil {
return "", err
}
dir := filepath.Join(cacheDir, InternalFolder)
err = os.MkdirAll(dir, 0700)
if err != nil {
return dir, err
}
return dir, nil
}
func (b *Bundle) GitRepository() (*git.Repository, error) {
rootPath, err := folders.FindDirWithLeaf(b.Config.Path, ".git")
if err != nil {

3
bundle/config/root.go
View File

@ -74,9 +74,6 @@ type Root struct {
// If not specified, the code below initializes this field with a
// single default-initialized environment called "default".
Environments map[string]*Environment `json:"environments,omitempty"`
// Sync section specifies options for files syncronisation
Sync Sync `json:"sync,omitempty"`
}
func Load(path string) (*Root, error) {

6
bundle/config/sync.go
View File

@ -1,6 +0,0 @@
package config
type Sync struct {
Include []string `json:"include,omitempty"`
Exclude []string `json:"exclude,omitempty"`
}

8
bundle/deploy/files/sync.go
View File

@ -14,21 +14,13 @@ func getSync(ctx context.Context, b *bundle.Bundle) (*sync.Sync, error) {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
internalDir, err := b.InternalDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle internal directory: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: b.Config.Sync.Include,
Exclude: b.Config.Sync.Exclude,
Full: false,
CurrentUser: b.Config.Workspace.CurrentUser.User,
InternalDir: internalDir,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}

16
bundle/python/transform.go
View File

@ -64,7 +64,7 @@ func (m *transform) Apply(ctx context.Context, b *bundle.Bundle) error {
return err
}
internalDir, err := b.InternalDir()
internalDir, err := getInternalDir(b)
if err != nil {
return err
}
@ -85,11 +85,21 @@ func (m *transform) Apply(ctx context.Context, b *bundle.Bundle) error {
return nil
}
func generateNotebookWrapper(b *bundle.Bundle, task *jobs.PythonWheelTask, libraries []compute.Library) (string, error) {
internalDir, err := b.InternalDir()
func getInternalDir(b *bundle.Bundle) (string, error) {
cacheDir, err := b.CacheDir()
if err != nil {
return "", err
}
internalDir := filepath.Join(cacheDir, ".internal")
return internalDir, nil
}
func generateNotebookWrapper(b *bundle.Bundle, task *jobs.PythonWheelTask, libraries []compute.Library) (string, error) {
internalDir, err := getInternalDir(b)
if err != nil {
return "", err
}
notebookName := fmt.Sprintf("notebook_%s_%s", task.PackageName, task.EntryPoint)
path := filepath.Join(internalDir, notebookName+".py")

8
cmd/bundle/sync.go
View File

@ -23,20 +23,12 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle)
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
internalDir, err := b.InternalDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle internal directory: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: b.Config.Sync.Include,
Exclude: b.Config.Sync.Exclude,
Full: f.full,
PollInterval: f.interval,
InternalDir: internalDir,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}

8
cmd/sync/sync.go
View File

@ -35,21 +35,13 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
internalDir, err := b.InternalDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle internal directory: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: b.Config.Sync.Include,
Exclude: b.Config.Sync.Exclude,
Full: f.full,
PollInterval: f.interval,
InternalDir: internalDir,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}

72
libs/sync/sync.go
View File

@ -3,27 +3,21 @@ package sync
import (
"context"
"fmt"
"path/filepath"
"time"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/fileset"
"github.com/databricks/cli/libs/git"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/iam"
"golang.org/x/exp/slices"
)
type SyncOptions struct {
LocalPath string
RemotePath string
Include []string
Exclude []string
Full bool
InternalDir string
SnapshotBasePath string
PollInterval time.Duration
@ -38,9 +32,7 @@ type SyncOptions struct {
type Sync struct {
*SyncOptions
fileSet *git.FileSet
includeFileSet *fileset.GlobSet
excludeFileSet *fileset.GlobSet
fileSet *git.FileSet
snapshot *Snapshot
filer filer.Filer
@ -61,17 +53,6 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) {
return nil, err
}
includes := []string{}
if opts.InternalDir != "" {
includes = append(includes, filepath.Join(opts.InternalDir, "*.*"))
}
if opts.Include != nil {
includes = append(includes, opts.Include...)
}
includeFileSet := fileset.NewGlobSet(opts.LocalPath, includes)
excludeFileSet := fileset.NewGlobSet(opts.LocalPath, opts.Exclude)
// Verify that the remote path we're about to synchronize to is valid and allowed.
err = EnsureRemotePathIsUsable(ctx, opts.WorkspaceClient, opts.RemotePath, opts.CurrentUser)
if err != nil {
@ -108,13 +89,11 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) {
return &Sync{
SyncOptions: &opts,
fileSet: fileSet,
includeFileSet: includeFileSet,
excludeFileSet: excludeFileSet,
snapshot: snapshot,
filer: filer,
notifier: &NopNotifier{},
seq: 0,
fileSet: fileSet,
snapshot: snapshot,
filer: filer,
notifier: &NopNotifier{},
seq: 0,
}, nil
}
@ -154,44 +133,15 @@ func (s *Sync) notifyComplete(ctx context.Context, d diff) {
}
func (s *Sync) RunOnce(ctx context.Context) error {
all := make([]fileset.File, 0)
if s.SyncOptions.Include == nil {
// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
gitFiles, err := s.fileSet.All()
if err != nil {
log.Errorf(ctx, "cannot list files: %s", err)
return err
}
all = append(all, gitFiles...)
}
include, err := s.includeFileSet.All()
// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
all, err := s.fileSet.All()
if err != nil {
log.Errorf(ctx, "cannot list include files: %s", err)
log.Errorf(ctx, "cannot list files: %s", err)
return err
}
all = append(all, include...)
exclude, err := s.excludeFileSet.All()
if err != nil {
log.Errorf(ctx, "cannot list exclude files: %s", err)
return err
}
files := make([]fileset.File, 0)
for _, f := range all {
if slices.ContainsFunc(exclude, func(a fileset.File) bool {
return a.Absolute == f.Absolute
}) {
continue
}
files = append(files, f)
}
change, err := s.snapshot.diff(ctx, files)
change, err := s.snapshot.diff(ctx, all)
if err != nil {
return err
}