mirror of https://github.com/databricks/cli.git
Experimental sync command
This commit is contained in:
Serge Smertin
parent
95a68937fe
commit
32ae59c1bc
|
|
@ -1,3 +1,5 @@
|
|||
.DS_Store
|
||||
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/atotto/clipboard"
|
||||
"github.com/databricks/bricks/retries"
|
||||
"github.com/pkg/browser"
|
||||
)
|
||||
|
||||
// Bricks CLI GitHub OAuth App Client ID
|
||||
const githubOauthClientID = "b91230382436c4592741"
|
||||
|
||||
func githubGetPAT(ctx context.Context) (string, error) {
|
||||
deviceRequest := url.Values{}
|
||||
deviceRequest.Set("client_id", githubOauthClientID)
|
||||
// TODO: scope
|
||||
response, err := http.PostForm("https://github.com/login/device/code", deviceRequest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
deviceResponse, err := url.ParseQuery(string(raw))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// TODO: give instructions to user and wait for the prompt
|
||||
userCode := deviceResponse.Get("user_code")
|
||||
err = clipboard.WriteAll(userCode)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot copy to clipboard: %w", err)
|
||||
}
|
||||
verificationURL := deviceResponse.Get("verification_uri")
|
||||
fmt.Printf("\nEnter the following code on %s: \n\n%s\n\n(it should be in your clipboard)", verificationURL, userCode)
|
||||
err = browser.OpenURL(verificationURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot open browser: %w", err)
|
||||
}
|
||||
var bearer string
|
||||
err = retries.Wait(ctx, 15*time.Minute, func() *retries.Err {
|
||||
form := url.Values{}
|
||||
form.Set("client_id", githubOauthClientID)
|
||||
form.Set("device_code", deviceResponse.Get("device_code"))
|
||||
form.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")
|
||||
response, err := http.PostForm("https://github.com/login/oauth/access_token", form)
|
||||
if err != nil {
|
||||
return retries.Halt(err)
|
||||
}
|
||||
raw, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return retries.Continuef("failed to read body: %w", err)
|
||||
}
|
||||
result, err := url.ParseQuery(string(raw))
|
||||
if err != nil {
|
||||
return retries.Continuef("failed to parse body: %w", err)
|
||||
}
|
||||
bearer = result.Get("access_token")
|
||||
if bearer != "" {
|
||||
return nil
|
||||
}
|
||||
if result.Get("error") == "slow_down" {
|
||||
t, _ := strconv.Atoi(result.Get("interval"))
|
||||
time.Sleep(time.Duration(t)*time.Second)
|
||||
log.Printf("[WARN] Rate limited, sleeping for %d seconds", t)
|
||||
}
|
||||
reason := result.Get("error_description")
|
||||
if reason == "" {
|
||||
reason = "access token is not ready"
|
||||
}
|
||||
return retries.Continues(reason)
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire access token: %w", err)
|
||||
}
|
||||
raw, err = json.Marshal(struct {
|
||||
note string
|
||||
scopes []string
|
||||
}{"test token", []string{}})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request, err := http.NewRequest("POST", "https://api.github.com/api/v3/authorizations", bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearer))
|
||||
response, err = http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, _ = ioutil.ReadAll(response.Body)
|
||||
log.Printf("[INFO] %s", raw)
|
||||
// TODO: convert to PAT
|
||||
return bearer, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGithubGetPAT(t *testing.T) {
|
||||
pat, err := githubGetPAT(context.Background())
|
||||
assert.NoError(t, err)
|
||||
assert.NotEqual(t, "..", pat)
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/databricks/bricks/git"
|
||||
)
|
||||
|
||||
type snapshot map[string]time.Time
|
||||
|
||||
type diff struct {
|
||||
put []string
|
||||
delete []string
|
||||
}
|
||||
|
||||
func (d diff) IsEmpty() bool {
|
||||
return len(d.put) == 0 && len(d.delete) == 0
|
||||
}
|
||||
|
||||
|
||||
func (d diff) String() string {
|
||||
if d.IsEmpty() {
|
||||
return "no changes"
|
||||
}
|
||||
var changes []string
|
||||
if len(d.put) > 0 {
|
||||
changes = append(changes, fmt.Sprintf("PUT: %s", strings.Join(d.put, ", ")))
|
||||
}
|
||||
if len(d.delete) > 0 {
|
||||
changes = append(changes, fmt.Sprintf("DELETE: %s", strings.Join(d.delete, ", ")))
|
||||
}
|
||||
return strings.Join(changes, ", ")
|
||||
}
|
||||
|
||||
func (s snapshot) diff(all []git.File) (change diff) {
|
||||
currentFilenames := map[string]bool{}
|
||||
for _, f := range all {
|
||||
// create set of current files to figure out if removals are needed
|
||||
currentFilenames[f.Relative] = true
|
||||
// get current modified timestamp
|
||||
modified := f.Modified()
|
||||
lastSeenModified, seen := s[f.Relative]
|
||||
if !(!seen || modified.After(lastSeenModified)) {
|
||||
continue
|
||||
}
|
||||
change.put = append(change.put, f.Relative)
|
||||
s[f.Relative] = modified
|
||||
}
|
||||
// figure out files in the snapshot, but not on local filesystem
|
||||
for relative := range s {
|
||||
_, exists := currentFilenames[relative]
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
// and add them to a delete batch
|
||||
change.delete = append(change.delete, relative)
|
||||
}
|
||||
// and remove them from the snapshot
|
||||
for _, v := range change.delete {
|
||||
delete(s, v)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/databricks/bricks/git"
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/repos"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/workspace"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// syncCmd represents the sync command
|
||||
var syncCmd = &cobra.Command{
|
||||
Use: "sync",
|
||||
Short: "run syncs for the project",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
origin, err := git.HttpsOrigin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[INFO] %s", origin)
|
||||
ctx := cmd.Context()
|
||||
client := project.Current.Client()
|
||||
reposAPI := repos.NewReposAPI(ctx, client)
|
||||
|
||||
|
||||
checkouts, err := reposAPI.List("/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range checkouts {
|
||||
log.Printf("[INFO] %s", v.Url)
|
||||
}
|
||||
me := project.Current.Me()
|
||||
repositoryName, err := git.RepositoryName()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName)
|
||||
return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error {
|
||||
wsAPI := workspace.NewNotebooksAPI(ctx, client)
|
||||
for _, v := range d.delete {
|
||||
err := wsAPI.Delete(path.Join(base, v), true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
func ImportFile(ctx context.Context, path string, content io.Reader) error {
|
||||
client := project.Current.Client()
|
||||
apiPath := fmt.Sprintf(
|
||||
"/workspace-files/import-file/%s?overwrite=true",
|
||||
strings.TrimLeft(path, "/"))
|
||||
// TODO: change upstream client to support io.Reader as body
|
||||
return client.Post(ctx, apiPath, content, nil)
|
||||
}
|
||||
|
||||
// project files polling interval
|
||||
var interval *time.Duration
|
||||
|
||||
func init() {
|
||||
root.RootCmd.AddCommand(syncCmd)
|
||||
interval = syncCmd.Flags().Duration("interval", 1*time.Second, "project files polling interval")
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestItSyncs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root.RootCmd.SetArgs([]string{"sync"})
|
||||
err := root.RootCmd.ExecuteContext(ctx)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/databricks/bricks/git"
|
||||
)
|
||||
|
||||
type watchdog struct {
|
||||
files git.FileSet
|
||||
ticker *time.Ticker
|
||||
wg sync.WaitGroup
|
||||
failure error // data race? make channel?
|
||||
}
|
||||
|
||||
func watchForChanges(ctx context.Context, files git.FileSet,
|
||||
interval time.Duration, cb func(diff) error) error {
|
||||
w := &watchdog{
|
||||
files: files,
|
||||
ticker: time.NewTicker(interval),
|
||||
}
|
||||
w.wg.Add(1)
|
||||
go w.main(ctx, cb)
|
||||
w.wg.Wait()
|
||||
return w.failure
|
||||
}
|
||||
|
||||
// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
|
||||
// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
|
||||
func (w *watchdog) main(ctx context.Context, cb func(diff) error) {
|
||||
defer w.wg.Done()
|
||||
// load from json or sync it every time there's an action
|
||||
state := snapshot{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-w.ticker.C:
|
||||
all, err := w.files.All()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] cannot list files: %s", err)
|
||||
w.failure = err
|
||||
return
|
||||
}
|
||||
change := state.diff(all)
|
||||
if change.IsEmpty() {
|
||||
continue
|
||||
}
|
||||
log.Printf("[INFO] Action: %v", change)
|
||||
err = cb(change)
|
||||
if err != nil {
|
||||
w.failure = err
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package folders
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
func FindDirWithLeaf(leaf string) (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot find $PWD: %s", err)
|
||||
}
|
||||
for {
|
||||
_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// TODO: test on windows
|
||||
next := path.Dir(dir)
|
||||
if dir == next { // or stop at $HOME?..
|
||||
return "", fmt.Errorf("cannot find %s anywhere", leaf)
|
||||
}
|
||||
dir = next
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
package git
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
fs.DirEntry
|
||||
Absolute, Relative string
|
||||
}
|
||||
|
||||
func (f File) Modified() (ts time.Time) {
|
||||
info, err := f.Info()
|
||||
if err != nil {
|
||||
// return default time, beginning of epoch
|
||||
return ts
|
||||
}
|
||||
return info.ModTime()
|
||||
}
|
||||
|
||||
// FileSet facilitates fast recursive file listing with
|
||||
// respect to patterns defined in `.gitignore` file
|
||||
type FileSet struct {
|
||||
root string
|
||||
ignore *ignore.GitIgnore
|
||||
}
|
||||
|
||||
// MustGetFileSet retrieves FileSet from Git repository checkout root
|
||||
// or panics if no root is detected.
|
||||
func MustGetFileSet() FileSet {
|
||||
root, err := Root()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return New(root)
|
||||
}
|
||||
|
||||
func New(root string) FileSet {
|
||||
lines := []string{".git"}
|
||||
rawIgnore, err := ioutil.ReadFile(fmt.Sprintf("%s/.gitignore", root))
|
||||
if err == nil {
|
||||
// add entries from .gitignore if the file exists (did read correctly)
|
||||
for _, line := range strings.Split(string(rawIgnore), "\n") {
|
||||
// underlying library doesn't behave well with Rule 5 of .gitignore,
|
||||
// hence this workaround
|
||||
lines = append(lines, strings.Trim(line, "/"))
|
||||
}
|
||||
}
|
||||
return FileSet{
|
||||
root: root,
|
||||
ignore: ignore.CompileIgnoreLines(lines...),
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FileSet) All() ([]File, error) {
|
||||
return w.RecursiveChildren(w.root)
|
||||
}
|
||||
|
||||
func (w *FileSet) RecursiveChildren(dir string) (found []File, err error) {
|
||||
queue, err := readDir(dir, w.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for len(queue) > 0 {
|
||||
current := queue[0]
|
||||
queue = queue[1:]
|
||||
if w.ignore.MatchesPath(current.Relative) {
|
||||
continue
|
||||
}
|
||||
if !current.IsDir() {
|
||||
found = append(found, current)
|
||||
continue
|
||||
}
|
||||
children, err := readDir(current.Absolute, w.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queue = append(queue, children...)
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func readDir(dir, root string) (queue []File, err error) {
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
dirs, err := f.ReadDir(-1)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, v := range dirs {
|
||||
absolute := path.Join(dir, v.Name())
|
||||
relative := strings.TrimLeft(strings.Replace(absolute, root, "", 1), "/")
|
||||
queue = append(queue, File{v, absolute, relative})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package git
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/bricks/folders"
|
||||
giturls "github.com/whilp/git-urls"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
func Root() (string, error) {
|
||||
return folders.FindDirWithLeaf(".git")
|
||||
}
|
||||
|
||||
// Origin finds the git repository the project is cloned from, so that
|
||||
// we could automatically verify if this project is checked out in repos
|
||||
// home folder of the user according to recommended best practices. Can
|
||||
// also be used to determine a good enough default project name.
|
||||
func Origin() (*url.URL, error) {
|
||||
root, err := Root()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file := fmt.Sprintf("%s/.git/config", root)
|
||||
gitConfig, err := ini.Load(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
section := gitConfig.Section(`remote "origin"`)
|
||||
if section == nil {
|
||||
return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
|
||||
}
|
||||
url := section.Key("url")
|
||||
if url == nil {
|
||||
return nil, fmt.Errorf("git origin url is not defined")
|
||||
}
|
||||
return giturls.Parse(url.Value())
|
||||
}
|
||||
|
||||
// HttpsOrigin returns URL in the format expected by Databricks Repos
|
||||
// platform functionality. Gradually expand implementation to work with
|
||||
// other formats of git URLs.
|
||||
func HttpsOrigin() (string, error) {
|
||||
origin, err := Origin()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// if current repo is checked out with a SSH key
|
||||
if origin.Scheme != "https" {
|
||||
origin.Scheme = "https"
|
||||
}
|
||||
// `git@` is not required for HTTPS, as Databricks Repos are checked
|
||||
// out using an API token instead of username. But does it hold true
|
||||
// for all of the git implementations?
|
||||
if origin.User != nil {
|
||||
origin.User = nil
|
||||
}
|
||||
return origin.String(), nil
|
||||
}
|
||||
|
||||
// RepositoryName returns repository name as last path entry from detected
|
||||
// git repository up the tree or returns error if it fails to do so.
|
||||
func RepositoryName() (string, error) {
|
||||
origin, err := Origin()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base := path.Base(origin.Path)
|
||||
return strings.TrimSuffix(base, ".git"), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package git
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetGitOrigin(t *testing.T) {
|
||||
this, err := RepositoryName()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "bricks", this)
|
||||
}
|
||||
|
||||
func TestHttpsOrigin(t *testing.T) {
|
||||
url, err := HttpsOrigin()
|
||||
assert.NoError(t, err)
|
||||
// must pass on the upcoming forks
|
||||
assert.True(t, strings.HasPrefix(url, "https://github.com"), url)
|
||||
assert.True(t, strings.HasSuffix(url, "bricks.git"), url)
|
||||
}
|
||||
3
go.mod
3
go.mod
|
|
@ -3,6 +3,7 @@ module github.com/databricks/bricks
|
|||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0
|
||||
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
|
||||
github.com/hashicorp/go-version v1.5.0 // MPL 2.0
|
||||
|
|
@ -11,6 +12,8 @@ require (
|
|||
github.com/hashicorp/terraform-json v0.13.0 // MPL 2.0
|
||||
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
|
||||
github.com/mitchellh/go-homedir v1.1.0 // MIT
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
|
||||
github.com/spf13/cobra v1.4.0 // Apache 2.0
|
||||
github.com/stretchr/testify v1.7.1 // MIT
|
||||
github.com/whilp/git-urls v1.0.0 // MIT
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -99,6 +99,8 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJE
|
|||
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
|
|
@ -142,6 +144,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
|
|||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
|
||||
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
|
|
@ -313,6 +317,8 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ
|
|||
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
|
||||
github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
|
|
@ -354,6 +360,8 @@ github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k
|
|||
github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
|
||||
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
|
@ -362,6 +370,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
|
|||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
|
|
@ -592,6 +602,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
|||
3
main.go
3
main.go
|
|
@ -1,10 +1,11 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
_ "github.com/databricks/bricks/cmd/fs"
|
||||
_ "github.com/databricks/bricks/cmd/init"
|
||||
_ "github.com/databricks/bricks/cmd/launch"
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
_ "github.com/databricks/bricks/cmd/sync"
|
||||
_ "github.com/databricks/bricks/cmd/test"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
package project
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/bricks/folders"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/clusters"
|
||||
"github.com/ghodss/yaml"
|
||||
gitUrls "github.com/whilp/git-urls"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
type Isolation string
|
||||
|
|
@ -108,64 +103,5 @@ func validateAndApplyProjectDefaults(prj Project) (Project, error) {
|
|||
}
|
||||
|
||||
func findProjectRoot() (string, error) {
|
||||
return findDirWithLeaf(ConfigFile)
|
||||
}
|
||||
|
||||
// finds the original git repository the project is cloned from, so that
|
||||
// we could automatically verify if this project is checked out in repos
|
||||
// home folder of the user according to recommended best practices. Can
|
||||
// also be used to determine a good enough default project name.
|
||||
func getGitOrigin() (*url.URL, error) {
|
||||
root, err := findDirWithLeaf(".git")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file := fmt.Sprintf("%s/.git/config", root)
|
||||
gitConfig, err := ini.Load(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
section := gitConfig.Section(`remote "origin"`)
|
||||
if section == nil {
|
||||
return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
|
||||
}
|
||||
url := section.Key("url")
|
||||
if url == nil {
|
||||
return nil, fmt.Errorf("git origin url is not defined")
|
||||
}
|
||||
return gitUrls.Parse(url.Value())
|
||||
}
|
||||
|
||||
// GitRepositoryName returns repository name as last path entry from detected
|
||||
// git repository up the tree or returns error if it fails to do so.
|
||||
func GitRepositoryName() (string, error) {
|
||||
origin, err := getGitOrigin()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base := path.Base(origin.Path)
|
||||
return strings.ReplaceAll(base, ".git", ""), nil
|
||||
}
|
||||
|
||||
func findDirWithLeaf(leaf string) (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot find $PWD: %s", err)
|
||||
}
|
||||
for {
|
||||
_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// TODO: test on windows
|
||||
next := path.Dir(dir)
|
||||
if dir == next { // or stop at $HOME?..
|
||||
return "", fmt.Errorf("cannot find %s anywhere", leaf)
|
||||
}
|
||||
dir = next
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
return folders.FindDirWithLeaf(ConfigFile)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,12 +28,6 @@ func TestFindProjectRootInRoot(t *testing.T) {
|
|||
assert.EqualError(t, err, "cannot find databricks.yml anywhere")
|
||||
}
|
||||
|
||||
func TestGetGitOrigin(t *testing.T) {
|
||||
this, err := GitRepositoryName()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "bricks", this)
|
||||
}
|
||||
|
||||
func TestLoadProjectConf(t *testing.T) {
|
||||
wd, _ := os.Getwd()
|
||||
defer os.Chdir(wd)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
package retries
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Err struct {
|
||||
Err error
|
||||
Halt bool
|
||||
}
|
||||
|
||||
func Halt(err error) *Err {
|
||||
return &Err{err, true}
|
||||
}
|
||||
|
||||
func Continue(err error) *Err {
|
||||
return &Err{err, false}
|
||||
}
|
||||
|
||||
func Continues(msg string) *Err {
|
||||
return Continue(fmt.Errorf(msg))
|
||||
}
|
||||
|
||||
func Continuef(format string, err error, args ...interface{}) *Err {
|
||||
wrapped := fmt.Errorf(format, append([]interface{}{err}, args...))
|
||||
return Continue(wrapped)
|
||||
}
|
||||
|
||||
type WaitFn func() *Err
|
||||
|
||||
var maxWait = 10 * time.Second
|
||||
var minJitter = 50 * time.Millisecond
|
||||
var maxJitter = 750 * time.Millisecond
|
||||
|
||||
func Wait(pctx context.Context, timeout time.Duration, fn WaitFn) error {
|
||||
ctx, cancel := context.WithTimeout(pctx, timeout)
|
||||
defer cancel()
|
||||
var attempt int
|
||||
var lastErr error
|
||||
for {
|
||||
attempt++
|
||||
res := fn()
|
||||
if res == nil {
|
||||
return nil
|
||||
}
|
||||
if res.Halt {
|
||||
return res.Err
|
||||
}
|
||||
lastErr = res.Err
|
||||
wait := time.Duration(attempt) * time.Second
|
||||
if wait > maxWait {
|
||||
wait = maxWait
|
||||
}
|
||||
// add some random jitter
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
jitter := rand.Intn(int(maxJitter)-int(minJitter)+1) + int(minJitter)
|
||||
wait += time.Duration(jitter)
|
||||
timer := time.NewTimer(wait)
|
||||
log.Printf("[TRACE] %s. Sleeping %s",
|
||||
strings.TrimSuffix(res.Err.Error(), "."),
|
||||
wait.Round(time.Millisecond))
|
||||
select {
|
||||
// stop when either this or parent context times out
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return fmt.Errorf("timed out: %w", lastErr)
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue