steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Experimental sync command

This commit is contained in:
Serge Smertin 2022-07-07 20:56:59 +02:00
parent 95a68937fe
commit 32ae59c1bc
17 changed files with 666 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
.DS_Store
# Binaries for programs and plugins
*.exe
*.exe~

107
cmd/sync/github.go Normal file
View File

@ -0,0 +1,107 @@
package sync
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"strconv"
"time"
"github.com/atotto/clipboard"
"github.com/databricks/bricks/retries"
"github.com/pkg/browser"
)
// Bricks CLI GitHub OAuth App Client ID
const githubOauthClientID = "b91230382436c4592741"
func githubGetPAT(ctx context.Context) (string, error) {
deviceRequest := url.Values{}
deviceRequest.Set("client_id", githubOauthClientID)
// TODO: scope
response, err := http.PostForm("https://github.com/login/device/code", deviceRequest)
if err != nil {
return "", err
}
raw, err := ioutil.ReadAll(response.Body)
if err != nil {
return "", err
}
deviceResponse, err := url.ParseQuery(string(raw))
if err != nil {
return "", err
}
// TODO: give instructions to user and wait for the prompt
userCode := deviceResponse.Get("user_code")
err = clipboard.WriteAll(userCode)
if err != nil {
return "", fmt.Errorf("cannot copy to clipboard: %w", err)
}
verificationURL := deviceResponse.Get("verification_uri")
fmt.Printf("\nEnter the following code on %s: \n\n%s\n\n(it should be in your clipboard)", verificationURL, userCode)
err = browser.OpenURL(verificationURL)
if err != nil {
return "", fmt.Errorf("cannot open browser: %w", err)
}
var bearer string
err = retries.Wait(ctx, 15*time.Minute, func() *retries.Err {
form := url.Values{}
form.Set("client_id", githubOauthClientID)
form.Set("device_code", deviceResponse.Get("device_code"))
form.Set("grant_type", "urn:ietf:params:oauth:grant-type:device_code")
response, err := http.PostForm("https://github.com/login/oauth/access_token", form)
if err != nil {
return retries.Halt(err)
}
raw, err := ioutil.ReadAll(response.Body)
if err != nil {
return retries.Continuef("failed to read body: %w", err)
}
result, err := url.ParseQuery(string(raw))
if err != nil {
return retries.Continuef("failed to parse body: %w", err)
}
bearer = result.Get("access_token")
if bearer != "" {
return nil
}
if result.Get("error") == "slow_down" {
t, _ := strconv.Atoi(result.Get("interval"))
time.Sleep(time.Duration(t)*time.Second)
log.Printf("[WARN] Rate limited, sleeping for %d seconds", t)
}
reason := result.Get("error_description")
if reason == "" {
reason = "access token is not ready"
}
return retries.Continues(reason)
})
if err != nil {
return "", fmt.Errorf("failed to acquire access token: %w", err)
}
raw, err = json.Marshal(struct {
note string
scopes []string
}{"test token", []string{}})
if err != nil {
return "", err
}
request, err := http.NewRequest("POST", "https://api.github.com/api/v3/authorizations", bytes.NewReader(raw))
if err != nil {
return "", err
}
request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearer))
response, err = http.DefaultClient.Do(request)
if err != nil {
return "", err
}
raw, _ = ioutil.ReadAll(response.Body)
log.Printf("[INFO] %s", raw)
// TODO: convert to PAT
return bearer, nil
}

14
cmd/sync/github_test.go Normal file
View File

@ -0,0 +1,14 @@
package sync
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGithubGetPAT(t *testing.T) {
pat, err := githubGetPAT(context.Background())
assert.NoError(t, err)
assert.NotEqual(t, "..", pat)
}

65
cmd/sync/snapshot.go Normal file
View File

@ -0,0 +1,65 @@
package sync
import (
"fmt"
"strings"
"time"
"github.com/databricks/bricks/git"
)
type snapshot map[string]time.Time
type diff struct {
put []string
delete []string
}
func (d diff) IsEmpty() bool {
return len(d.put) == 0 && len(d.delete) == 0
}
func (d diff) String() string {
if d.IsEmpty() {
return "no changes"
}
var changes []string
if len(d.put) > 0 {
changes = append(changes, fmt.Sprintf("PUT: %s", strings.Join(d.put, ", ")))
}
if len(d.delete) > 0 {
changes = append(changes, fmt.Sprintf("DELETE: %s", strings.Join(d.delete, ", ")))
}
return strings.Join(changes, ", ")
}
func (s snapshot) diff(all []git.File) (change diff) {
currentFilenames := map[string]bool{}
for _, f := range all {
// create set of current files to figure out if removals are needed
currentFilenames[f.Relative] = true
// get current modified timestamp
modified := f.Modified()
lastSeenModified, seen := s[f.Relative]
if !(!seen || modified.After(lastSeenModified)) {
continue
}
change.put = append(change.put, f.Relative)
s[f.Relative] = modified
}
// figure out files in the snapshot, but not on local filesystem
for relative := range s {
_, exists := currentFilenames[relative]
if exists {
continue
}
// and add them to a delete batch
change.delete = append(change.delete, relative)
}
// and remove them from the snapshot
for _, v := range change.delete {
delete(s, v)
}
return
}

76
cmd/sync/sync.go Normal file
View File

@ -0,0 +1,76 @@
package sync
import (
"context"
"fmt"
"io"
"log"
"path"
"strings"
"time"
"github.com/databricks/bricks/cmd/root"
"github.com/databricks/bricks/git"
"github.com/databricks/bricks/project"
"github.com/databrickslabs/terraform-provider-databricks/repos"
"github.com/databrickslabs/terraform-provider-databricks/workspace"
"github.com/spf13/cobra"
)
// syncCmd represents the sync command
var syncCmd = &cobra.Command{
Use: "sync",
Short: "run syncs for the project",
RunE: func(cmd *cobra.Command, args []string) error {
origin, err := git.HttpsOrigin()
if err != nil {
return err
}
log.Printf("[INFO] %s", origin)
ctx := cmd.Context()
client := project.Current.Client()
reposAPI := repos.NewReposAPI(ctx, client)
checkouts, err := reposAPI.List("/")
if err != nil {
return err
}
for _, v := range checkouts {
log.Printf("[INFO] %s", v.Url)
}
me := project.Current.Me()
repositoryName, err := git.RepositoryName()
if err != nil {
return err
}
base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName)
return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error {
wsAPI := workspace.NewNotebooksAPI(ctx, client)
for _, v := range d.delete {
err := wsAPI.Delete(path.Join(base, v), true)
if err != nil {
return err
}
}
return nil
})
},
}
func ImportFile(ctx context.Context, path string, content io.Reader) error {
client := project.Current.Client()
apiPath := fmt.Sprintf(
"/workspace-files/import-file/%s?overwrite=true",
strings.TrimLeft(path, "/"))
// TODO: change upstream client to support io.Reader as body
return client.Post(ctx, apiPath, content, nil)
}
// project files polling interval
var interval *time.Duration
func init() {
root.RootCmd.AddCommand(syncCmd)
interval = syncCmd.Flags().Duration("interval", 1*time.Second, "project files polling interval")
}

16
cmd/sync/sync_test.go Normal file
View File

@ -0,0 +1,16 @@
package sync
import (
"context"
"testing"
"github.com/databricks/bricks/cmd/root"
"github.com/stretchr/testify/assert"
)
func TestItSyncs(t *testing.T) {
ctx := context.Background()
root.RootCmd.SetArgs([]string{"sync"})
err := root.RootCmd.ExecuteContext(ctx)
assert.NoError(t, err)
}

60
cmd/sync/watchdog.go Normal file
View File

@ -0,0 +1,60 @@
package sync
import (
"context"
"log"
"sync"
"time"
"github.com/databricks/bricks/git"
)
type watchdog struct {
files git.FileSet
ticker *time.Ticker
wg sync.WaitGroup
failure error // data race? make channel?
}
func watchForChanges(ctx context.Context, files git.FileSet,
interval time.Duration, cb func(diff) error) error {
w := &watchdog{
files: files,
ticker: time.NewTicker(interval),
}
w.wg.Add(1)
go w.main(ctx, cb)
w.wg.Wait()
return w.failure
}
// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
func (w *watchdog) main(ctx context.Context, cb func(diff) error) {
defer w.wg.Done()
// load from json or sync it every time there's an action
state := snapshot{}
for {
select {
case <-ctx.Done():
return
case <-w.ticker.C:
all, err := w.files.All()
if err != nil {
log.Printf("[ERROR] cannot list files: %s", err)
w.failure = err
return
}
change := state.diff(all)
if change.IsEmpty() {
continue
}
log.Printf("[INFO] Action: %v", change)
err = cb(change)
if err != nil {
w.failure = err
return
}
}
}
}

31
folders/folders.go Normal file
View File

@ -0,0 +1,31 @@
package folders
import (
"errors"
"fmt"
"os"
"path"
)
func FindDirWithLeaf(leaf string) (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("cannot find $PWD: %s", err)
}
for {
_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
if errors.Is(err, os.ErrNotExist) {
// TODO: test on windows
next := path.Dir(dir)
if dir == next { // or stop at $HOME?..
return "", fmt.Errorf("cannot find %s anywhere", leaf)
}
dir = next
continue
}
if err != nil {
return "", err
}
return dir, nil
}
}

107
git/fileset.go Normal file
View File

@ -0,0 +1,107 @@
package git
import (
"fmt"
"io/fs"
"io/ioutil"
"os"
"path"
"strings"
"time"
ignore "github.com/sabhiram/go-gitignore"
)
type File struct {
fs.DirEntry
Absolute, Relative string
}
func (f File) Modified() (ts time.Time) {
info, err := f.Info()
if err != nil {
// return default time, beginning of epoch
return ts
}
return info.ModTime()
}
// FileSet facilitates fast recursive file listing with
// respect to patterns defined in `.gitignore` file
type FileSet struct {
root string
ignore *ignore.GitIgnore
}
// MustGetFileSet retrieves FileSet from Git repository checkout root
// or panics if no root is detected.
func MustGetFileSet() FileSet {
root, err := Root()
if err != nil {
panic(err)
}
return New(root)
}
func New(root string) FileSet {
lines := []string{".git"}
rawIgnore, err := ioutil.ReadFile(fmt.Sprintf("%s/.gitignore", root))
if err == nil {
// add entries from .gitignore if the file exists (did read correctly)
for _, line := range strings.Split(string(rawIgnore), "\n") {
// underlying library doesn't behave well with Rule 5 of .gitignore,
// hence this workaround
lines = append(lines, strings.Trim(line, "/"))
}
}
return FileSet{
root: root,
ignore: ignore.CompileIgnoreLines(lines...),
}
}
func (w *FileSet) All() ([]File, error) {
return w.RecursiveChildren(w.root)
}
func (w *FileSet) RecursiveChildren(dir string) (found []File, err error) {
queue, err := readDir(dir, w.root)
if err != nil {
return nil, err
}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
if w.ignore.MatchesPath(current.Relative) {
continue
}
if !current.IsDir() {
found = append(found, current)
continue
}
children, err := readDir(current.Absolute, w.root)
if err != nil {
return nil, err
}
queue = append(queue, children...)
}
return found, nil
}
func readDir(dir, root string) (queue []File, err error) {
f, err := os.Open(dir)
if err != nil {
return
}
defer f.Close()
dirs, err := f.ReadDir(-1)
if err != nil {
return
}
for _, v := range dirs {
absolute := path.Join(dir, v.Name())
relative := strings.TrimLeft(strings.Replace(absolute, root, "", 1), "/")
queue = append(queue, File{v, absolute, relative})
}
return
}

73
git/git.go Normal file
View File

@ -0,0 +1,73 @@
package git
import (
"fmt"
"net/url"
"path"
"strings"
"github.com/databricks/bricks/folders"
giturls "github.com/whilp/git-urls"
"gopkg.in/ini.v1"
)
func Root() (string, error) {
return folders.FindDirWithLeaf(".git")
}
// Origin finds the git repository the project is cloned from, so that
// we could automatically verify if this project is checked out in repos
// home folder of the user according to recommended best practices. Can
// also be used to determine a good enough default project name.
func Origin() (*url.URL, error) {
root, err := Root()
if err != nil {
return nil, err
}
file := fmt.Sprintf("%s/.git/config", root)
gitConfig, err := ini.Load(file)
if err != nil {
return nil, err
}
section := gitConfig.Section(`remote "origin"`)
if section == nil {
return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
}
url := section.Key("url")
if url == nil {
return nil, fmt.Errorf("git origin url is not defined")
}
return giturls.Parse(url.Value())
}
// HttpsOrigin returns URL in the format expected by Databricks Repos
// platform functionality. Gradually expand implementation to work with
// other formats of git URLs.
func HttpsOrigin() (string, error) {
origin, err := Origin()
if err != nil {
return "", err
}
// if current repo is checked out with a SSH key
if origin.Scheme != "https" {
origin.Scheme = "https"
}
// `git@` is not required for HTTPS, as Databricks Repos are checked
// out using an API token instead of username. But does it hold true
// for all of the git implementations?
if origin.User != nil {
origin.User = nil
}
return origin.String(), nil
}
// RepositoryName returns repository name as last path entry from detected
// git repository up the tree or returns error if it fails to do so.
func RepositoryName() (string, error) {
origin, err := Origin()
if err != nil {
return "", err
}
base := path.Base(origin.Path)
return strings.TrimSuffix(base, ".git"), nil
}

22
git/git_test.go Normal file
View File

@ -0,0 +1,22 @@
package git
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetGitOrigin(t *testing.T) {
this, err := RepositoryName()
assert.NoError(t, err)
assert.Equal(t, "bricks", this)
}
func TestHttpsOrigin(t *testing.T) {
url, err := HttpsOrigin()
assert.NoError(t, err)
// must pass on the upcoming forks
assert.True(t, strings.HasPrefix(url, "https://github.com"), url)
assert.True(t, strings.HasSuffix(url, "bricks.git"), url)
}

3
go.mod
View File

@ -3,6 +3,7 @@ module github.com/databricks/bricks
go 1.16
require (
github.com/atotto/clipboard v0.1.4
github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/hashicorp/go-version v1.5.0 // MPL 2.0
@ -11,6 +12,8 @@ require (
github.com/hashicorp/terraform-json v0.13.0 // MPL 2.0
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
github.com/mitchellh/go-homedir v1.1.0 // MIT
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
github.com/spf13/cobra v1.4.0 // Apache 2.0
github.com/stretchr/testify v1.7.1 // MIT
github.com/whilp/git-urls v1.0.0 // MIT

11
go.sum
View File

@ -99,6 +99,8 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJE
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@ -142,6 +144,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
@ -313,6 +317,8 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@ -354,6 +360,8 @@ github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k
github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -362,6 +370,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@ -592,6 +602,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

3
main.go
View File

@ -1,10 +1,11 @@
package main
import (
"github.com/databricks/bricks/cmd/root"
_ "github.com/databricks/bricks/cmd/fs"
_ "github.com/databricks/bricks/cmd/init"
_ "github.com/databricks/bricks/cmd/launch"
"github.com/databricks/bricks/cmd/root"
_ "github.com/databricks/bricks/cmd/sync"
_ "github.com/databricks/bricks/cmd/test"
)

68
project/config.go
View File

@ -1,19 +1,14 @@
package project
import (
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"path"
"reflect"
"strings"
"github.com/databricks/bricks/folders"
"github.com/databrickslabs/terraform-provider-databricks/clusters"
"github.com/ghodss/yaml"
gitUrls "github.com/whilp/git-urls"
"gopkg.in/ini.v1"
)
type Isolation string
@ -108,64 +103,5 @@ func validateAndApplyProjectDefaults(prj Project) (Project, error) {
}
func findProjectRoot() (string, error) {
return findDirWithLeaf(ConfigFile)
}
// finds the original git repository the project is cloned from, so that
// we could automatically verify if this project is checked out in repos
// home folder of the user according to recommended best practices. Can
// also be used to determine a good enough default project name.
func getGitOrigin() (*url.URL, error) {
root, err := findDirWithLeaf(".git")
if err != nil {
return nil, err
}
file := fmt.Sprintf("%s/.git/config", root)
gitConfig, err := ini.Load(file)
if err != nil {
return nil, err
}
section := gitConfig.Section(`remote "origin"`)
if section == nil {
return nil, fmt.Errorf("remote `origin` is not defined in %s", file)
}
url := section.Key("url")
if url == nil {
return nil, fmt.Errorf("git origin url is not defined")
}
return gitUrls.Parse(url.Value())
}
// GitRepositoryName returns repository name as last path entry from detected
// git repository up the tree or returns error if it fails to do so.
func GitRepositoryName() (string, error) {
origin, err := getGitOrigin()
if err != nil {
return "", err
}
base := path.Base(origin.Path)
return strings.ReplaceAll(base, ".git", ""), nil
}
func findDirWithLeaf(leaf string) (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("cannot find $PWD: %s", err)
}
for {
_, err = os.Stat(fmt.Sprintf("%s/%s", dir, leaf))
if errors.Is(err, os.ErrNotExist) {
// TODO: test on windows
next := path.Dir(dir)
if dir == next { // or stop at $HOME?..
return "", fmt.Errorf("cannot find %s anywhere", leaf)
}
dir = next
continue
}
if err != nil {
return "", err
}
return dir, nil
}
return folders.FindDirWithLeaf(ConfigFile)
}

6
project/config_test.go
View File

@ -28,12 +28,6 @@ func TestFindProjectRootInRoot(t *testing.T) {
assert.EqualError(t, err, "cannot find databricks.yml anywhere")
}
func TestGetGitOrigin(t *testing.T) {
this, err := GitRepositoryName()
assert.NoError(t, err)
assert.Equal(t, "bricks", this)
}
func TestLoadProjectConf(t *testing.T) {
wd, _ := os.Getwd()
defer os.Chdir(wd)

75
retries/retries.go Normal file
View File

@ -0,0 +1,75 @@
package retries
import (
"context"
"fmt"
"log"
"math/rand"
"strings"
"time"
)
type Err struct {
Err error
Halt bool
}
func Halt(err error) *Err {
return &Err{err, true}
}
func Continue(err error) *Err {
return &Err{err, false}
}
func Continues(msg string) *Err {
return Continue(fmt.Errorf(msg))
}
func Continuef(format string, err error, args ...interface{}) *Err {
wrapped := fmt.Errorf(format, append([]interface{}{err}, args...))
return Continue(wrapped)
}
type WaitFn func() *Err
var maxWait = 10 * time.Second
var minJitter = 50 * time.Millisecond
var maxJitter = 750 * time.Millisecond
func Wait(pctx context.Context, timeout time.Duration, fn WaitFn) error {
ctx, cancel := context.WithTimeout(pctx, timeout)
defer cancel()
var attempt int
var lastErr error
for {
attempt++
res := fn()
if res == nil {
return nil
}
if res.Halt {
return res.Err
}
lastErr = res.Err
wait := time.Duration(attempt) * time.Second
if wait > maxWait {
wait = maxWait
}
// add some random jitter
rand.Seed(time.Now().UnixNano())
jitter := rand.Intn(int(maxJitter)-int(minJitter)+1) + int(minJitter)
wait += time.Duration(jitter)
timer := time.NewTimer(wait)
log.Printf("[TRACE] %s. Sleeping %s",
strings.TrimSuffix(res.Err.Error(), "."),
wait.Round(time.Millisecond))
select {
// stop when either this or parent context times out
case <-ctx.Done():
timer.Stop()
return fmt.Errorf("timed out: %w", lastErr)
case <-timer.C:
}
}
}