From 32ae59c1bced6381bafc62a1a27e6b1be2b9cdfb Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Thu, 7 Jul 2022 20:56:59 +0200 Subject: [PATCH] Experimental sync command --- .gitignore | 2 + cmd/sync/github.go | 107 ++++++++++++++++++++++++++++++++++++++++ cmd/sync/github_test.go | 14 ++++++ cmd/sync/snapshot.go | 65 ++++++++++++++++++++++++ cmd/sync/sync.go | 76 ++++++++++++++++++++++++++++ cmd/sync/sync_test.go | 16 ++++++ cmd/sync/watchdog.go | 60 ++++++++++++++++++++++ folders/folders.go | 31 ++++++++++++ git/fileset.go | 107 ++++++++++++++++++++++++++++++++++++++++ git/git.go | 73 +++++++++++++++++++++++++++ git/git_test.go | 22 +++++++++ go.mod | 3 ++ go.sum | 11 +++++ main.go | 3 +- project/config.go | 68 +------------------------ project/config_test.go | 6 --- retries/retries.go | 75 ++++++++++++++++++++++++++++ 17 files changed, 666 insertions(+), 73 deletions(-) create mode 100644 cmd/sync/github.go create mode 100644 cmd/sync/github_test.go create mode 100644 cmd/sync/snapshot.go create mode 100644 cmd/sync/sync.go create mode 100644 cmd/sync/sync_test.go create mode 100644 cmd/sync/watchdog.go create mode 100644 folders/folders.go create mode 100644 git/fileset.go create mode 100644 git/git.go create mode 100644 git/git_test.go create mode 100644 retries/retries.go diff --git a/.gitignore b/.gitignore index 176f02c0..1e35c543 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_Store + # Binaries for programs and plugins *.exe *.exe~ diff --git a/cmd/sync/github.go b/cmd/sync/github.go new file mode 100644 index 00000000..94d18489 --- /dev/null +++ b/cmd/sync/github.go @@ -0,0 +1,107 @@ +package sync + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/atotto/clipboard" + "github.com/databricks/bricks/retries" + "github.com/pkg/browser" +) + +// Bricks CLI GitHub OAuth App Client ID +const githubOauthClientID = "b91230382436c4592741" + +func githubGetPAT(ctx context.Context) (string, error) { + deviceRequest := url.Values{} + deviceRequest.Set("client_id", githubOauthClientID) + // TODO: scope + response, err := http.PostForm("https://github.com/login/device/code", deviceRequest) + if err != nil { + return "", err + } + raw, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", err + } + deviceResponse, err := url.ParseQuery(string(raw)) + if err != nil { + return "", err + } + // TODO: give instructions to user and wait for the prompt + userCode := deviceResponse.Get("user_code") + err = clipboard.WriteAll(userCode) + if err != nil { + return "", fmt.Errorf("cannot copy to clipboard: %w", err) + } + verificationURL := deviceResponse.Get("verification_uri") + fmt.Printf("\nEnter the following code on %s: \n\n%s\n\n(it should be in your clipboard)", verificationURL, userCode) + err = browser.OpenURL(verificationURL) + if err != nil { + return "", fmt.Errorf("cannot open browser: %w", err) + } + var bearer string + err = retries.Wait(ctx, 15*time.Minute, func() *retries.Err { + form := url.Values{} + form.Set("client_id", githubOauthClientID) + form.Set("device_code", deviceResponse.Get("device_code")) + form.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code") + response, err := http.PostForm("https://github.com/login/oauth/access_token", form) + if err != nil { + return retries.Halt(err) + } + raw, err := ioutil.ReadAll(response.Body) + if err != nil { + return retries.Continuef("failed to read body: %w", err) + } + result, err := url.ParseQuery(string(raw)) + if err != nil { + return retries.Continuef("failed to parse body: %w", err) + } + bearer = result.Get("access_token") + if bearer != "" { + return nil + } + if result.Get("error") == "slow_down" { + t, _ := strconv.Atoi(result.Get("interval")) + time.Sleep(time.Duration(t)*time.Second) + log.Printf("[WARN] Rate limited, sleeping for %d seconds", t) + } + reason := result.Get("error_description") + if reason == "" { + reason = "access token is not ready" + } + return retries.Continues(reason) + }) + if err != nil { + return "", fmt.Errorf("failed to acquire access token: %w", err) + } + raw, err = json.Marshal(struct { + note string + scopes []string + }{"test token", []string{}}) + if err != nil { + return "", err + } + request, err := http.NewRequest("POST", "https://api.github.com/api/v3/authorizations", bytes.NewReader(raw)) + if err != nil { + return "", err + } + request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearer)) + response, err = http.DefaultClient.Do(request) + if err != nil { + return "", err + } + raw, _ = ioutil.ReadAll(response.Body) + log.Printf("[INFO] %s", raw) + // TODO: convert to PAT + return bearer, nil +} diff --git a/cmd/sync/github_test.go b/cmd/sync/github_test.go new file mode 100644 index 00000000..29876456 --- /dev/null +++ b/cmd/sync/github_test.go @@ -0,0 +1,14 @@ +package sync + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGithubGetPAT(t *testing.T) { + pat, err := githubGetPAT(context.Background()) + assert.NoError(t, err) + assert.NotEqual(t, "..", pat) +} \ No newline at end of file diff --git a/cmd/sync/snapshot.go b/cmd/sync/snapshot.go new file mode 100644 index 00000000..47828c7c --- /dev/null +++ b/cmd/sync/snapshot.go @@ -0,0 +1,65 @@ +package sync + +import ( + "fmt" + "strings" + "time" + + "github.com/databricks/bricks/git" +) + +type snapshot map[string]time.Time + +type diff struct { + put []string + delete []string +} + +func (d diff) IsEmpty() bool { + return len(d.put) == 0 && len(d.delete) == 0 +} + + +func (d diff) String() string { + if d.IsEmpty() { + return "no changes" + } + var changes []string + if len(d.put) > 0 { + changes = append(changes, fmt.Sprintf("PUT: %s", strings.Join(d.put, ", "))) + } + if len(d.delete) > 0 { + changes = append(changes, fmt.Sprintf("DELETE: %s", strings.Join(d.delete, ", "))) + } + return strings.Join(changes, ", ") +} + +func (s snapshot) diff(all []git.File) (change diff) { + currentFilenames := map[string]bool{} + for _, f := range all { + // create set of current files to figure out if removals are needed + currentFilenames[f.Relative] = true + // get current modified timestamp + modified := f.Modified() + lastSeenModified, seen := s[f.Relative] + if !(!seen || modified.After(lastSeenModified)) { + continue + } + change.put = append(change.put, f.Relative) + s[f.Relative] = modified + } + // figure out files in the snapshot, but not on local filesystem + for relative := range s { + _, exists := currentFilenames[relative] + if exists { + continue + } + // and add them to a delete batch + change.delete = append(change.delete, relative) + } + // and remove them from the snapshot + for _, v := range change.delete { + delete(s, v) + } + return +} diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go new file mode 100644 index 00000000..66752628 --- /dev/null +++ b/cmd/sync/sync.go @@ -0,0 +1,76 @@ +package sync + +import ( + "context" + "fmt" + "io" + "log" + "path" + "strings" + "time" + + "github.com/databricks/bricks/cmd/root" + "github.com/databricks/bricks/git" + "github.com/databricks/bricks/project" + "github.com/databrickslabs/terraform-provider-databricks/repos" + "github.com/databrickslabs/terraform-provider-databricks/workspace" + "github.com/spf13/cobra" +) + +// syncCmd represents the sync command +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "run syncs for the project", + RunE: func(cmd *cobra.Command, args []string) error { + origin, err := git.HttpsOrigin() + if err != nil { + return err + } + log.Printf("[INFO] %s", origin) + ctx := cmd.Context() + client := project.Current.Client() + reposAPI := repos.NewReposAPI(ctx, client) + + + checkouts, err := reposAPI.List("/") + if err != nil { + return err + } + for _, v := range checkouts { + log.Printf("[INFO] %s", v.Url) + } + me := project.Current.Me() + repositoryName, err := git.RepositoryName() + if err != nil { + return err + } + base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName) + return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error { + wsAPI := workspace.NewNotebooksAPI(ctx, client) + for _, v := range d.delete { + err := wsAPI.Delete(path.Join(base, v), true) + if err != nil { + return err + } + } + return nil + }) + }, +} + +func ImportFile(ctx context.Context, path string, content io.Reader) error { + client := project.Current.Client() + apiPath := fmt.Sprintf( + "/workspace-files/import-file/%s?overwrite=true", + strings.TrimLeft(path, "/")) + // TODO: change upstream client to support io.Reader as body + return client.Post(ctx, apiPath, content, nil) +} + +// project files polling interval +var interval *time.Duration + +func init() { + root.RootCmd.AddCommand(syncCmd) + interval = syncCmd.Flags().Duration("interval", 1*time.Second, "project files polling interval") +} diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go new file mode 100644 index 00000000..22b2c688 --- /dev/null +++ b/cmd/sync/sync_test.go @@ -0,0 +1,16 @@ +package sync + +import ( + "context" + "testing" + + "github.com/databricks/bricks/cmd/root" + "github.com/stretchr/testify/assert" +) + +func TestItSyncs(t *testing.T) { + ctx := context.Background() + root.RootCmd.SetArgs([]string{"sync"}) + err := root.RootCmd.ExecuteContext(ctx) + assert.NoError(t, err) +} \ No newline at end of file diff --git a/cmd/sync/watchdog.go b/cmd/sync/watchdog.go new file mode 100644 index 00000000..e186d9c1 --- /dev/null +++ b/cmd/sync/watchdog.go @@ -0,0 +1,60 @@ +package sync + +import ( + "context" + "log" + "sync" + "time" + + "github.com/databricks/bricks/git" +) + +type watchdog struct { + files git.FileSet + ticker *time.Ticker + wg sync.WaitGroup + failure error // data race? make channel? +} + +func watchForChanges(ctx context.Context, files git.FileSet, + interval time.Duration, cb func(diff) error) error { + w := &watchdog{ + files: files, + ticker: time.NewTicker(interval), + } + w.wg.Add(1) + go w.main(ctx, cb) + w.wg.Wait() + return w.failure +} + +// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement +// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418 +func (w *watchdog) main(ctx context.Context, cb func(diff) error) { + defer w.wg.Done() + // load from json or sync it every time there's an action + state := snapshot{} + for { + select { + case <-ctx.Done(): + return + case <-w.ticker.C: + all, err := w.files.All() + if err != nil { + log.Printf("[ERROR] cannot list files: %s", err) + w.failure = err + return + } + change := state.diff(all) + if change.IsEmpty() { + continue + } + log.Printf("[INFO] Action: %v", change) + err = cb(change) + if err != nil { + w.failure = err + return + } + } + } +} diff --git a/folders/folders.go b/folders/folders.go new file mode 100644 index 00000000..dda792e0 --- /dev/null +++ b/folders/folders.go @@ -0,0 +1,31 @@ +package folders + +import ( + "errors" + "fmt" + "os" + "path" +) + +func FindDirWithLeaf(leaf string) (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("cannot find $PWD: %s", err) + } + for { + _, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf)) + if errors.Is(err, os.ErrNotExist) { + // TODO: test on windows + next := path.Dir(dir) + if dir == next { // or stop at $HOME?.. + return "", fmt.Errorf("cannot find %s anywhere", leaf) + } + dir = next + continue + } + if err != nil { + return "", err + } + return dir, nil + } +} diff --git a/git/fileset.go b/git/fileset.go new file mode 100644 index 00000000..f9fa0072 --- /dev/null +++ b/git/fileset.go @@ -0,0 +1,107 @@ +package git + +import ( + "fmt" + "io/fs" + "io/ioutil" + "os" + "path" + "strings" + "time" + + ignore "github.com/sabhiram/go-gitignore" +) + +type File struct { + fs.DirEntry + Absolute, Relative string +} + +func (f File) Modified() (ts time.Time) { + info, err := f.Info() + if err != nil { + // return default time, beginning of epoch + return ts + } + return info.ModTime() +} + +// FileSet facilitates fast recursive file listing with +// respect to patterns defined in `.gitignore` file +type FileSet struct { + root string + ignore *ignore.GitIgnore +} + +// MustGetFileSet retrieves FileSet from Git repository checkout root +// or panics if no root is detected. +func MustGetFileSet() FileSet { + root, err := Root() + if err != nil { + panic(err) + } + return New(root) +} + +func New(root string) FileSet { + lines := []string{".git"} + rawIgnore, err := ioutil.ReadFile(fmt.Sprintf("%s/.gitignore", root)) + if err == nil { + // add entries from .gitignore if the file exists (did read correctly) + for _, line := range strings.Split(string(rawIgnore), "\n") { + // underlying library doesn't behave well with Rule 5 of .gitignore, + // hence this workaround + lines = append(lines, strings.Trim(line, "/")) + } + } + return FileSet{ + root: root, + ignore: ignore.CompileIgnoreLines(lines...), + } +} + +func (w *FileSet) All() ([]File, error) { + return w.RecursiveChildren(w.root) +} + +func (w *FileSet) RecursiveChildren(dir string) (found []File, err error) { + queue, err := readDir(dir, w.root) + if err != nil { + return nil, err + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + if w.ignore.MatchesPath(current.Relative) { + continue + } + if !current.IsDir() { + found = append(found, current) + continue + } + children, err := readDir(current.Absolute, w.root) + if err != nil { + return nil, err + } + queue = append(queue, children...) + } + return found, nil +} + +func readDir(dir, root string) (queue []File, err error) { + f, err := os.Open(dir) + if err != nil { + return + } + defer f.Close() + dirs, err := f.ReadDir(-1) + if err != nil { + return + } + for _, v := range dirs { + absolute := path.Join(dir, v.Name()) + relative := strings.TrimLeft(strings.Replace(absolute, root, "", 1), "/") + queue = append(queue, File{v, absolute, relative}) + } + return +} diff --git a/git/git.go b/git/git.go new file mode 100644 index 00000000..583344e5 --- /dev/null +++ b/git/git.go @@ -0,0 +1,73 @@ +package git + +import ( + "fmt" + "net/url" + "path" + "strings" + + "github.com/databricks/bricks/folders" + giturls "github.com/whilp/git-urls" + "gopkg.in/ini.v1" +) + +func Root() (string, error) { + return folders.FindDirWithLeaf(".git") +} + +// Origin finds the git repository the project is cloned from, so that +// we could automatically verify if this project is checked out in repos +// home folder of the user according to recommended best practices. Can +// also be used to determine a good enough default project name. +func Origin() (*url.URL, error) { + root, err := Root() + if err != nil { + return nil, err + } + file := fmt.Sprintf("%s/.git/config", root) + gitConfig, err := ini.Load(file) + if err != nil { + return nil, err + } + section := gitConfig.Section(`remote "origin"`) + if section == nil { + return nil, fmt.Errorf("remote `origin` is not defined in %s", file) + } + url := section.Key("url") + if url == nil { + return nil, fmt.Errorf("git origin url is not defined") + } + return giturls.Parse(url.Value()) +} + +// HttpsOrigin returns URL in the format expected by Databricks Repos +// platform functionality. Gradually expand implementation to work with +// other formats of git URLs. +func HttpsOrigin() (string, error) { + origin, err := Origin() + if err != nil { + return "", err + } + // if current repo is checked out with a SSH key + if origin.Scheme != "https" { + origin.Scheme = "https" + } + // `git@` is not required for HTTPS, as Databricks Repos are checked + // out using an API token instead of username. But does it hold true + // for all of the git implementations? + if origin.User != nil { + origin.User = nil + } + return origin.String(), nil +} + +// RepositoryName returns repository name as last path entry from detected +// git repository up the tree or returns error if it fails to do so. +func RepositoryName() (string, error) { + origin, err := Origin() + if err != nil { + return "", err + } + base := path.Base(origin.Path) + return strings.TrimSuffix(base, ".git"), nil +} diff --git a/git/git_test.go b/git/git_test.go new file mode 100644 index 00000000..13721742 --- /dev/null +++ b/git/git_test.go @@ -0,0 +1,22 @@ +package git + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetGitOrigin(t *testing.T) { + this, err := RepositoryName() + assert.NoError(t, err) + assert.Equal(t, "bricks", this) +} + +func TestHttpsOrigin(t *testing.T) { + url, err := HttpsOrigin() + assert.NoError(t, err) + // must pass on the upcoming forks + assert.True(t, strings.HasPrefix(url, "https://github.com"), url) + assert.True(t, strings.HasSuffix(url, "bricks.git"), url) +} \ No newline at end of file diff --git a/go.mod b/go.mod index ea11404c..45f087aa 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/databricks/bricks go 1.16 require ( + github.com/atotto/clipboard v0.1.4 github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0 github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/hashicorp/go-version v1.5.0 // MPL 2.0 @@ -11,6 +12,8 @@ require ( github.com/hashicorp/terraform-json v0.13.0 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license github.com/mitchellh/go-homedir v1.1.0 // MIT + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT github.com/spf13/cobra v1.4.0 // Apache 2.0 github.com/stretchr/testify v1.7.1 // MIT github.com/whilp/git-urls v1.0.0 // MIT diff --git a/go.sum b/go.sum index 89e8dfc2..5a6277e4 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJE github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -142,6 +144,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -313,6 +317,8 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -354,6 +360,8 @@ github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -362,6 +370,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -592,6 +602,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go index f4dedf0a..10a3ce7b 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,11 @@ package main import ( - "github.com/databricks/bricks/cmd/root" _ "github.com/databricks/bricks/cmd/fs" _ "github.com/databricks/bricks/cmd/init" _ "github.com/databricks/bricks/cmd/launch" + "github.com/databricks/bricks/cmd/root" + _ "github.com/databricks/bricks/cmd/sync" _ "github.com/databricks/bricks/cmd/test" ) diff --git a/project/config.go b/project/config.go index 1256f136..ed1c5ef9 100644 --- a/project/config.go +++ b/project/config.go @@ -1,19 +1,14 @@ package project import ( - "errors" "fmt" "io/ioutil" - "net/url" "os" - "path" "reflect" - "strings" + "github.com/databricks/bricks/folders" "github.com/databrickslabs/terraform-provider-databricks/clusters" "github.com/ghodss/yaml" - gitUrls "github.com/whilp/git-urls" - "gopkg.in/ini.v1" ) type Isolation string @@ -108,64 +103,5 @@ func validateAndApplyProjectDefaults(prj Project) (Project, error) { } func findProjectRoot() (string, error) { - return findDirWithLeaf(ConfigFile) -} - -// finds the original git repository the project is cloned from, so that -// we could automatically verify if this project is checked out in repos -// home folder of the user according to recommended best practices. Can -// also be used to determine a good enough default project name. -func getGitOrigin() (*url.URL, error) { - root, err := findDirWithLeaf(".git") - if err != nil { - return nil, err - } - file := fmt.Sprintf("%s/.git/config", root) - gitConfig, err := ini.Load(file) - if err != nil { - return nil, err - } - section := gitConfig.Section(`remote "origin"`) - if section == nil { - return nil, fmt.Errorf("remote `origin` is not defined in %s", file) - } - url := section.Key("url") - if url == nil { - return nil, fmt.Errorf("git origin url is not defined") - } - return gitUrls.Parse(url.Value()) -} - -// GitRepositoryName returns repository name as last path entry from detected -// git repository up the tree or returns error if it fails to do so. -func GitRepositoryName() (string, error) { - origin, err := getGitOrigin() - if err != nil { - return "", err - } - base := path.Base(origin.Path) - return strings.ReplaceAll(base, ".git", ""), nil -} - -func findDirWithLeaf(leaf string) (string, error) { - dir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("cannot find $PWD: %s", err) - } - for { - _, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf)) - if errors.Is(err, os.ErrNotExist) { - // TODO: test on windows - next := path.Dir(dir) - if dir == next { // or stop at $HOME?.. - return "", fmt.Errorf("cannot find %s anywhere", leaf) - } - dir = next - continue - } - if err != nil { - return "", err - } - return dir, nil - } + return folders.FindDirWithLeaf(ConfigFile) } diff --git a/project/config_test.go b/project/config_test.go index 4dbd2333..36fbdf3d 100644 --- a/project/config_test.go +++ b/project/config_test.go @@ -28,12 +28,6 @@ func TestFindProjectRootInRoot(t *testing.T) { assert.EqualError(t, err, "cannot find databricks.yml anywhere") } -func TestGetGitOrigin(t *testing.T) { - this, err := GitRepositoryName() - assert.NoError(t, err) - assert.Equal(t, "bricks", this) -} - func TestLoadProjectConf(t *testing.T) { wd, _ := os.Getwd() defer os.Chdir(wd) diff --git a/retries/retries.go b/retries/retries.go new file mode 100644 index 00000000..8a0e3179 --- /dev/null +++ b/retries/retries.go @@ -0,0 +1,75 @@ +package retries + +import ( + "context" + "fmt" + "log" + "math/rand" + "strings" + "time" +) + +type Err struct { + Err error + Halt bool +} + +func Halt(err error) *Err { + return &Err{err, true} +} + +func Continue(err error) *Err { + return &Err{err, false} +} + +func Continues(msg string) *Err { + return Continue(fmt.Errorf(msg)) +} + +func Continuef(format string, err error, args ...interface{}) *Err { + wrapped := fmt.Errorf(format, append([]interface{}{err}, args...)) + return Continue(wrapped) +} + +type WaitFn func() *Err + +var maxWait = 10 * time.Second +var minJitter = 50 * time.Millisecond +var maxJitter = 750 * time.Millisecond + +func Wait(pctx context.Context, timeout time.Duration, fn WaitFn) error { + ctx, cancel := context.WithTimeout(pctx, timeout) + defer cancel() + var attempt int + var lastErr error + for { + attempt++ + res := fn() + if res == nil { + return nil + } + if res.Halt { + return res.Err + } + lastErr = res.Err + wait := time.Duration(attempt) * time.Second + if wait > maxWait { + wait = maxWait + } + // add some random jitter + rand.Seed(time.Now().UnixNano()) + jitter := rand.Intn(int(maxJitter)-int(minJitter)+1) + int(minJitter) + wait += time.Duration(jitter) + timer := time.NewTimer(wait) + log.Printf("[TRACE] %s. Sleeping %s", + strings.TrimSuffix(res.Err.Error(), "."), + wait.Round(time.Millisecond)) + select { + // stop when either this or parent context times out + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("timed out: %w", lastErr) + case <-timer.C: + } + } +}