Experimental sync command

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+
 # Binaries for programs and plugins
 *.exe
 *.exe~

cmd/sync/github.go

@@ -0,0 +1,107 @@
+package sync
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/atotto/clipboard"
+	"github.com/databricks/bricks/retries"
+	"github.com/pkg/browser"
+)
+
+// Bricks CLI GitHub OAuth App Client ID
+const githubOauthClientID = "b91230382436c4592741"
+
+func githubGetPAT(ctx context.Context) (string, error) {
+	deviceRequest := url.Values{}
+	deviceRequest.Set("client_id", githubOauthClientID)
+	// TODO: scope
+	response, err := http.PostForm("https://github.com/login/device/code", deviceRequest)
+	if err != nil {
+		return "", err
+	}
+	raw, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+	deviceResponse, err := url.ParseQuery(string(raw))
+	if err != nil {
+		return "", err
+	}
+	// TODO: give instructions to user and wait for the prompt
+	userCode := deviceResponse.Get("user_code")
+	err = clipboard.WriteAll(userCode)
+	if err != nil {
+		return "", fmt.Errorf("cannot copy to clipboard: %w", err)
+	}
+	verificationURL := deviceResponse.Get("verification_uri")
+	fmt.Printf("\nEnter the following code on %s: \n\n%s\n\n(it should be in your clipboard)", verificationURL, userCode)
+	err = browser.OpenURL(verificationURL)
+	if err != nil {
+		return "", fmt.Errorf("cannot open browser: %w", err)
+	}
+	var bearer string
+	err = retries.Wait(ctx, 15*time.Minute, func() *retries.Err {
+		form := url.Values{}
+		form.Set("client_id", githubOauthClientID)
+		form.Set("device_code", deviceResponse.Get("device_code"))
+		form.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")
+		response, err := http.PostForm("https://github.com/login/oauth/access_token", form)
+		if err != nil {
+			return retries.Halt(err)
+		}
+		raw, err := ioutil.ReadAll(response.Body)
+		if err != nil {
+			return retries.Continuef("failed to read body: %w", err)
+		}
+		result, err := url.ParseQuery(string(raw))
+		if err != nil {
+			return retries.Continuef("failed to parse body: %w", err)
+		}
+		bearer = result.Get("access_token")
+		if bearer != "" {
+			return nil
+		}
+		if result.Get("error") == "slow_down" {
+			t, _ := strconv.Atoi(result.Get("interval"))
+			time.Sleep(time.Duration(t)*time.Second)
+			log.Printf("[WARN] Rate limited, sleeping for %d seconds", t)
+		}
+		reason := result.Get("error_description")
+		if reason == "" {
+			reason = "access token is not ready"
+		}
+		return retries.Continues(reason)
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to acquire access token: %w", err)
+	}
+	raw, err = json.Marshal(struct {
+		note   string
+		scopes []string
+	}{"test token", []string{}})
+	if err != nil {
+		return "", err
+	}
+	request, err := http.NewRequest("POST", "https://api.github.com/api/v3/authorizations", bytes.NewReader(raw))
+	if err != nil {
+		return "", err
+	}
+	request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearer))
+	response, err = http.DefaultClient.Do(request)
+	if err != nil {
+		return "", err
+	}
+	raw, _ = ioutil.ReadAll(response.Body)
+	log.Printf("[INFO] %s", raw)
+	// TODO: convert to PAT
+	return bearer, nil
+}

cmd/sync/github_test.go

@@ -0,0 +1,14 @@
+package sync
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGithubGetPAT(t *testing.T) {
+	pat, err := githubGetPAT(context.Background())
+	assert.NoError(t, err)
+	assert.NotEqual(t, "..", pat)
+}

cmd/sync/snapshot.go

@@ -0,0 +1,65 @@
+package sync
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/databricks/bricks/git"
+)
+
+type snapshot map[string]time.Time
+
+type diff struct {
+	put    []string
+	delete []string
+}
+
+func (d diff) IsEmpty() bool {
+	return len(d.put) == 0 && len(d.delete) == 0
+}
+
+
+func (d diff) String() string {
+	if d.IsEmpty() {
+		return "no changes"
+	}
+	var changes []string
+	if len(d.put) > 0 {
+		changes = append(changes, fmt.Sprintf("PUT: %s", strings.Join(d.put, ", ")))
+	}
+	if len(d.delete) > 0 {
+		changes = append(changes, fmt.Sprintf("DELETE: %s", strings.Join(d.delete, ", ")))
+	}
+	return strings.Join(changes, ", ")
+}
+
+func (s snapshot) diff(all []git.File) (change diff) {
+	currentFilenames := map[string]bool{}
+	for _, f := range all {
+		// create set of current files to figure out if removals are needed
+		currentFilenames[f.Relative] = true
+		// get current modified timestamp
+		modified := f.Modified()
+		lastSeenModified, seen := s[f.Relative]
+		if !(!seen || modified.After(lastSeenModified)) {
+			continue
+		}
+		change.put = append(change.put, f.Relative)
+		s[f.Relative] = modified
+	}
+	// figure out files in the snapshot, but not on local filesystem
+	for relative := range s {
+		_, exists := currentFilenames[relative]
+		if exists {
+			continue
+		}
+		// and add them to a delete batch
+		change.delete = append(change.delete, relative)
+	}
+	// and remove them from the snapshot
+	for _, v := range change.delete {
+		delete(s, v)
+	}
+	return
+}

cmd/sync/sync.go

@@ -0,0 +1,76 @@
+package sync
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/databricks/bricks/cmd/root"
+	"github.com/databricks/bricks/git"
+	"github.com/databricks/bricks/project"
+	"github.com/databrickslabs/terraform-provider-databricks/repos"
+	"github.com/databrickslabs/terraform-provider-databricks/workspace"
+	"github.com/spf13/cobra"
+)
+
+// syncCmd represents the sync command
+var syncCmd = &cobra.Command{
+	Use:   "sync",
+	Short: "run syncs for the project",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		origin, err := git.HttpsOrigin()
+		if err != nil {
+			return err
+		}
+		log.Printf("[INFO] %s", origin)
+		ctx := cmd.Context()
+		client := project.Current.Client()
+		reposAPI := repos.NewReposAPI(ctx, client)
+
+
+		checkouts, err := reposAPI.List("/")
+		if err != nil {
+			return err
+		}
+		for _, v := range checkouts {
+			log.Printf("[INFO] %s", v.Url)
+		}
+		me := project.Current.Me()
+		repositoryName, err := git.RepositoryName()
+		if err != nil {
+			return err
+		}
+		base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName)
+		return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error {
+			wsAPI := workspace.NewNotebooksAPI(ctx, client)
+			for _, v := range d.delete {
+				err := wsAPI.Delete(path.Join(base, v), true)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	},
+}
+
+func ImportFile(ctx context.Context, path string, content io.Reader) error {
+	client := project.Current.Client()
+	apiPath := fmt.Sprintf(
+		"/workspace-files/import-file/%s?overwrite=true",
+		strings.TrimLeft(path, "/"))
+	// TODO: change upstream client to support io.Reader as body
+	return client.Post(ctx, apiPath, content, nil)
+}
+
+// project files polling interval
+var interval *time.Duration
+
+func init() {
+	root.RootCmd.AddCommand(syncCmd)
+	interval = syncCmd.Flags().Duration("interval", 1*time.Second, "project files polling interval")
+}

cmd/sync/sync_test.go

@@ -0,0 +1,16 @@
+package sync
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/bricks/cmd/root"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestItSyncs(t *testing.T) {
+	ctx := context.Background()
+	root.RootCmd.SetArgs([]string{"sync"})
+	err := root.RootCmd.ExecuteContext(ctx)
+	assert.NoError(t, err)
+}

cmd/sync/watchdog.go

@@ -0,0 +1,60 @@
+package sync
+
+import (
+	"context"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/databricks/bricks/git"
+)
+
+type watchdog struct {
+	files   git.FileSet
+	ticker  *time.Ticker
+	wg      sync.WaitGroup
+	failure error // data race? make channel?
+}
+
+func watchForChanges(ctx context.Context, files git.FileSet,
+	interval time.Duration, cb func(diff) error) error {
+	w := &watchdog{
+		files:  files,
+		ticker: time.NewTicker(interval),
+	}
+	w.wg.Add(1)
+	go w.main(ctx, cb)
+	w.wg.Wait()
+	return w.failure
+}
+
+// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
+// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
+func (w *watchdog) main(ctx context.Context, cb func(diff) error) {
+	defer w.wg.Done()
+	// load from json or sync it every time there's an action
+	state := snapshot{}
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-w.ticker.C:
+			all, err := w.files.All()
+			if err != nil {
+				log.Printf("[ERROR] cannot list files: %s", err)
+				w.failure = err
+				return
+			}
+			change := state.diff(all)
+			if change.IsEmpty() {
+				continue
+			}
+			log.Printf("[INFO] Action: %v", change)
+			err = cb(change)
+			if err != nil {
+				w.failure = err
+				return
+			}
+		}
+	}
+}

folders/folders.go

@@ -0,0 +1,31 @@
+package folders
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+)
+
+func FindDirWithLeaf(leaf string) (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("cannot find $PWD: %s", err)
+	}
+	for {
+		_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
+		if errors.Is(err, os.ErrNotExist) {
+			// TODO: test on windows
+			next := path.Dir(dir)
+			if dir == next { // or stop at $HOME?..
+				return "", fmt.Errorf("cannot find %s anywhere", leaf)
+			}
+			dir = next
+			continue
+		}
+		if err != nil {
+			return "", err
+		}
+		return dir, nil
+	}
+}

git/fileset.go

@@ -0,0 +1,107 @@
+package git
+
+import (
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	ignore "github.com/sabhiram/go-gitignore"
+)
+
+type File struct {
+	fs.DirEntry
+	Absolute, Relative string
+}
+
+func (f File) Modified() (ts time.Time) {
+	info, err := f.Info()
+	if err != nil {
+		// return default time, beginning of epoch
+		return ts
+	}
+	return info.ModTime()
+}
+
+// FileSet facilitates fast recursive file listing with
+// respect to patterns defined in `.gitignore` file
+type FileSet struct {
+	root   string
+	ignore *ignore.GitIgnore
+}
+
+// MustGetFileSet retrieves FileSet from Git repository checkout root
+// or panics if no root is detected.
+func MustGetFileSet() FileSet {
+	root, err := Root()
+	if err != nil {
+		panic(err)
+	}
+	return New(root)
+}
+
+func New(root string) FileSet {
+	lines := []string{".git"}
+	rawIgnore, err := ioutil.ReadFile(fmt.Sprintf("%s/.gitignore", root))
+	if err == nil {
+		// add entries from .gitignore if the file exists (did read correctly)
+		for _, line := range strings.Split(string(rawIgnore), "\n") {
+			// underlying library doesn't behave well with Rule 5 of .gitignore,
+			// hence this workaround
+			lines = append(lines, strings.Trim(line, "/"))
+		}
+	}
+	return FileSet{
+		root:   root,
+		ignore: ignore.CompileIgnoreLines(lines...),
+	}
+}
+
+func (w *FileSet) All() ([]File, error) {
+	return w.RecursiveChildren(w.root)
+}
+
+func (w *FileSet) RecursiveChildren(dir string) (found []File, err error) {
+	queue, err := readDir(dir, w.root)
+	if err != nil {
+		return nil, err
+	}
+	for len(queue) > 0 {
+		current := queue[0]
+		queue = queue[1:]
+		if w.ignore.MatchesPath(current.Relative) {
+			continue
+		}
+		if !current.IsDir() {
+			found = append(found, current)
+			continue
+		}
+		children, err := readDir(current.Absolute, w.root)
+		if err != nil {
+			return nil, err
+		}
+		queue = append(queue, children...)
+	}
+	return found, nil
+}
+
+func readDir(dir, root string) (queue []File, err error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	dirs, err := f.ReadDir(-1)
+	if err != nil {
+		return
+	}
+	for _, v := range dirs {
+		absolute := path.Join(dir, v.Name())
+		relative := strings.TrimLeft(strings.Replace(absolute, root, "", 1), "/")
+		queue = append(queue, File{v, absolute, relative})
+	}
+	return
+}

git/git.go

@@ -0,0 +1,73 @@
+package git
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/databricks/bricks/folders"
+	giturls "github.com/whilp/git-urls"
+	"gopkg.in/ini.v1"
+)
+
+func Root() (string, error) {
+	return folders.FindDirWithLeaf(".git")
+}
+
+// Origin finds the git repository the project is cloned from, so that
+// we could automatically verify if this project is checked out in repos
+// home folder of the user according to recommended best practices. Can
+// also be used to determine a good enough default project name.
+func Origin() (*url.URL, error) {
+	root, err := Root()
+	if err != nil {
+		return nil, err
+	}
+	file := fmt.Sprintf("%s/.git/config", root)
+	gitConfig, err := ini.Load(file)
+	if err != nil {
+		return nil, err
+	}
+	section := gitConfig.Section(`remote "origin"`)
+	if section == nil {
+		return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
+	}
+	url := section.Key("url")
+	if url == nil {
+		return nil, fmt.Errorf("git origin url is not defined")
+	}
+	return giturls.Parse(url.Value())
+}
+
+// HttpsOrigin returns URL in the format expected by Databricks Repos
+// platform functionality. Gradually expand implementation to work with
+// other formats of git URLs.
+func HttpsOrigin() (string, error) {
+	origin, err := Origin()
+	if err != nil {
+	  return "", err
+	}
+	// if current repo is checked out with a SSH key
+	if origin.Scheme != "https" {
+		origin.Scheme = "https"
+	}
+	// `git@` is not required for HTTPS, as Databricks Repos are checked 
+	// out using an API token instead of username. But does it hold true
+	// for all of the git implementations?
+	if origin.User != nil {
+		origin.User = nil
+	}
+	return origin.String(), nil
+}
+
+// RepositoryName returns repository name as last path entry from detected
+// git repository up the tree or returns error if it fails to do so.
+func RepositoryName() (string, error) {
+	origin, err := Origin()
+	if err != nil {
+		return "", err
+	}
+	base := path.Base(origin.Path)
+	return strings.TrimSuffix(base, ".git"), nil
+}

git/git_test.go

@@ -0,0 +1,22 @@
+package git
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetGitOrigin(t *testing.T) {
+	this, err := RepositoryName()
+	assert.NoError(t, err)
+	assert.Equal(t, "bricks", this)
+}
+
+func TestHttpsOrigin(t *testing.T) {
+	url, err := HttpsOrigin()
+	assert.NoError(t, err)
+	// must pass on the upcoming forks
+	assert.True(t, strings.HasPrefix(url, "https://github.com"), url)
+	assert.True(t, strings.HasSuffix(url, "bricks.git"), url)
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/databricks/bricks
 go 1.16
 
 require (
+	github.com/atotto/clipboard v0.1.4
 	github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/hashicorp/go-version v1.5.0 // MPL 2.0
@@ -11,6 +12,8 @@ require (
 	github.com/hashicorp/terraform-json v0.13.0 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
 	github.com/mitchellh/go-homedir v1.1.0 // MIT
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
 	github.com/spf13/cobra v1.4.0 // Apache 2.0
 	github.com/stretchr/testify v1.7.1 // MIT
 	github.com/whilp/git-urls v1.0.0 // MIT

go.sum

@@ -99,6 +99,8 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJE
 github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
+github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -142,6 +144,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
@@ -313,6 +317,8 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
+github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -354,6 +360,8 @@ github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k
 github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
 github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -362,6 +370,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -592,6 +602,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

main.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/databricks/bricks/cmd/root"
 	_ "github.com/databricks/bricks/cmd/fs"
 	_ "github.com/databricks/bricks/cmd/init"
 	_ "github.com/databricks/bricks/cmd/launch"
+	"github.com/databricks/bricks/cmd/root"
+	_ "github.com/databricks/bricks/cmd/sync"
 	_ "github.com/databricks/bricks/cmd/test"
 )
 

project/config.go

@@ -1,19 +1,14 @@
 package project
 
 import (
-	"errors"
 	"fmt"
 	"io/ioutil"
-	"net/url"
 	"os"
-	"path"
 	"reflect"
-	"strings"
 
+	"github.com/databricks/bricks/folders"
 	"github.com/databrickslabs/terraform-provider-databricks/clusters"
 	"github.com/ghodss/yaml"
-	gitUrls "github.com/whilp/git-urls"
-	"gopkg.in/ini.v1"
 )
 
 type Isolation string
@@ -108,64 +103,5 @@ func validateAndApplyProjectDefaults(prj Project) (Project, error) {
 }
 
 func findProjectRoot() (string, error) {
-	return findDirWithLeaf(ConfigFile)
+	return folders.FindDirWithLeaf(ConfigFile)
-}
-
-// finds the original git repository the project is cloned from, so that
-// we could automatically verify if this project is checked out in repos
-// home folder of the user according to recommended best practices. Can
-// also be used to determine a good enough default project name.
-func getGitOrigin() (*url.URL, error) {
-	root, err := findDirWithLeaf(".git")
-	if err != nil {
-		return nil, err
-	}
-	file := fmt.Sprintf("%s/.git/config", root)
-	gitConfig, err := ini.Load(file)
-	if err != nil {
-		return nil, err
-	}
-	section := gitConfig.Section(`remote "origin"`)
-	if section == nil {
-		return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
-	}
-	url := section.Key("url")
-	if url == nil {
-		return nil, fmt.Errorf("git origin url is not defined")
-	}
-	return gitUrls.Parse(url.Value())
-}
-
-// GitRepositoryName returns repository name as last path entry from detected
-// git repository up the tree or returns error if it fails to do so.
-func GitRepositoryName() (string, error) {
-	origin, err := getGitOrigin()
-	if err != nil {
-		return "", err
-	}
-	base := path.Base(origin.Path)
-	return strings.ReplaceAll(base, ".git", ""), nil
-}
-
-func findDirWithLeaf(leaf string) (string, error) {
-	dir, err := os.Getwd()
-	if err != nil {
-		return "", fmt.Errorf("cannot find $PWD: %s", err)
-	}
-	for {
-		_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
-		if errors.Is(err, os.ErrNotExist) {
-			// TODO: test on windows
-			next := path.Dir(dir)
-			if dir == next { // or stop at $HOME?..
-				return "", fmt.Errorf("cannot find %s anywhere", leaf)
-			}
-			dir = next
-			continue
-		}
-		if err != nil {
-			return "", err
-		}
-		return dir, nil
-	}
 }

project/config_test.go

@@ -28,12 +28,6 @@ func TestFindProjectRootInRoot(t *testing.T) {
 	assert.EqualError(t, err, "cannot find databricks.yml anywhere")
 }
 
-func TestGetGitOrigin(t *testing.T) {
-	this, err := GitRepositoryName()
-	assert.NoError(t, err)
-	assert.Equal(t, "bricks", this)
-}
-
 func TestLoadProjectConf(t *testing.T) {
 	wd, _ := os.Getwd()
 	defer os.Chdir(wd)

retries/retries.go

@@ -0,0 +1,75 @@
+package retries
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math/rand"
+	"strings"
+	"time"
+)
+
+type Err struct {
+	Err  error
+	Halt bool
+}
+
+func Halt(err error) *Err {
+	return &Err{err, true}
+}
+
+func Continue(err error) *Err {
+	return &Err{err, false}
+}
+
+func Continues(msg string) *Err {
+	return Continue(fmt.Errorf(msg))
+}
+
+func Continuef(format string, err error, args ...interface{}) *Err {
+	wrapped := fmt.Errorf(format, append([]interface{}{err}, args...))
+	return Continue(wrapped)
+}
+
+type WaitFn func() *Err
+
+var maxWait = 10 * time.Second
+var minJitter = 50 * time.Millisecond
+var maxJitter = 750 * time.Millisecond
+
+func Wait(pctx context.Context, timeout time.Duration, fn WaitFn) error {
+	ctx, cancel := context.WithTimeout(pctx, timeout)
+	defer cancel()
+	var attempt int
+	var lastErr error
+	for {
+		attempt++
+		res := fn()
+		if res == nil {
+			return nil
+		}
+		if res.Halt {
+			return res.Err
+		}
+		lastErr = res.Err
+		wait := time.Duration(attempt) * time.Second
+		if wait > maxWait {
+			wait = maxWait
+		}
+		// add some random jitter
+		rand.Seed(time.Now().UnixNano())
+		jitter := rand.Intn(int(maxJitter)-int(minJitter)+1) + int(minJitter)
+		wait += time.Duration(jitter)
+		timer := time.NewTimer(wait)
+		log.Printf("[TRACE] %s. Sleeping %s",
+			strings.TrimSuffix(res.Err.Error(), "."),
+			wait.Round(time.Millisecond))
+		select {
+		// stop when either this or parent context times out
+		case <-ctx.Done():
+			timer.Stop()
+			return fmt.Errorf("timed out: %w", lastErr)
+		case <-timer.C:
+		}
+	}
+}