diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3b487507..5272a5d4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -10,8 +10,11 @@ jobs: tests: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout repository and submodules uses: actions/checkout@v2 + with: + token: ${{ secrets.DECO_GITHUB_TOKEN }} + submodules: recursive - name: Unshallow run: git fetch --prune --unshallow @@ -19,7 +22,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v1 with: - go-version: 1.16.x + go-version: 1.18.x - name: Cache packages uses: actions/cache@v2 @@ -38,7 +41,7 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH go get gotest.tools/gotestsum - go install honnef.co/go/tools/cmd/staticcheck@v0.2.2 + go install honnef.co/go/tools/cmd/staticcheck@latest - name: Pull external libraries run: make vendor diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9921b7b3..7580d257 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.18 - name: Import GPG key id: import_gpg uses: hashicorp/ghaction-import-gpg@v2.1.0 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..316e505d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/databricks-sdk-go"] + path = ext/databricks-sdk-go + url = git@github.com:databricks/databricks-sdk-go.git diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index bdbc94d8..5e4845dd 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -5,8 +5,6 @@ import ( "github.com/databricks/bricks/project" "github.com/spf13/cobra" - - "github.com/databrickslabs/terraform-provider-databricks/storage" ) // lsCmd represents the ls command @@ -16,11 +14,12 @@ var lsCmd = &cobra.Command{ Long: `Lists files`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - api := storage.NewDbfsAPI(cmd.Context(), project.Current.Client()) - files, err := api.List(args[0], false) + wsc := project.Current.WorkspacesClient() + listStatusResponse, err := wsc.Dbfs.ListByPath(cmd.Context(), args[0]) if err != nil { panic(err) } + files := listStatusResponse.Files // TODO: output formatting: JSON, CSV, tables and default for _, v := range files { fmt.Printf("[-] %s (%d, %v)\n", v.Path, v.FileSize, v.IsDir) diff --git a/cmd/launch/launch.go b/cmd/launch/launch.go index b584ac42..402a6963 100644 --- a/cmd/launch/launch.go +++ b/cmd/launch/launch.go @@ -30,7 +30,7 @@ var launchCmd = &cobra.Command{ } func init() { - // TODO: detect if we can skip registering + // TODO: detect if we can skip registering // launch command for DBSQL projects root.RootCmd.AddCommand(launchCmd) } diff --git a/cmd/sync/github.go b/cmd/sync/github.go index 94d18489..ba6774b3 100644 --- a/cmd/sync/github.go +++ b/cmd/sync/github.go @@ -72,7 +72,7 @@ func githubGetPAT(ctx context.Context) (string, error) { } if result.Get("error") == "slow_down" { t, _ := strconv.Atoi(result.Get("interval")) - time.Sleep(time.Duration(t)*time.Second) + time.Sleep(time.Duration(t) * time.Second) log.Printf("[WARN] Rate limited, sleeping for %d seconds", t) } reason := result.Get("error_description") diff --git a/cmd/sync/github_test.go b/cmd/sync/github_test.go index 29876456..13830de4 100644 --- a/cmd/sync/github_test.go +++ b/cmd/sync/github_test.go @@ -11,4 +11,4 @@ func TestGithubGetPAT(t *testing.T) { pat, err := githubGetPAT(context.Background()) assert.NoError(t, err) assert.NotEqual(t, "..", pat) -} \ No newline at end of file +} diff --git a/cmd/sync/snapshot.go b/cmd/sync/snapshot.go index 47828c7c..3dd81bc9 100644 --- a/cmd/sync/snapshot.go +++ b/cmd/sync/snapshot.go @@ -19,7 +19,6 @@ func (d diff) IsEmpty() bool { return len(d.put) == 0 && len(d.delete) == 0 } - func (d diff) String() string { if d.IsEmpty() { return "no changes" diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 66752628..5853a4d8 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -12,8 +12,8 @@ import ( "github.com/databricks/bricks/cmd/root" "github.com/databricks/bricks/git" "github.com/databricks/bricks/project" - "github.com/databrickslabs/terraform-provider-databricks/repos" - "github.com/databrickslabs/terraform-provider-databricks/workspace" + "github.com/databricks/bricks/utilities" + "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" ) @@ -28,11 +28,9 @@ var syncCmd = &cobra.Command{ } log.Printf("[INFO] %s", origin) ctx := cmd.Context() - client := project.Current.Client() - reposAPI := repos.NewReposAPI(ctx, client) - - checkouts, err := reposAPI.List("/") + wsc := project.Current.WorkspacesClient() + checkouts, err := utilities.GetAllRepos(ctx, wsc, "/") if err != nil { return err } @@ -46,9 +44,13 @@ var syncCmd = &cobra.Command{ } base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName) return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error { - wsAPI := workspace.NewNotebooksAPI(ctx, client) for _, v := range d.delete { - err := wsAPI.Delete(path.Join(base, v), true) + err := wsc.Workspace.Delete(ctx, + workspace.DeleteRequest{ + Path: path.Join(base, v), + Recursive: true, + }, + ) if err != nil { return err } diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 22b2c688..a8002b5e 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -13,4 +13,4 @@ func TestItSyncs(t *testing.T) { root.RootCmd.SetArgs([]string{"sync"}) err := root.RootCmd.ExecuteContext(ctx) assert.NoError(t, err) -} \ No newline at end of file +} diff --git a/ext/databricks-sdk-go b/ext/databricks-sdk-go new file mode 160000 index 00000000..9e9cc8f8 --- /dev/null +++ b/ext/databricks-sdk-go @@ -0,0 +1 @@ +Subproject commit 9e9cc8f8115b8d3cb9e778806066c7cffe5b9b76 diff --git a/git/git.go b/git/git.go index 583344e5..848fd164 100644 --- a/git/git.go +++ b/git/git.go @@ -46,13 +46,13 @@ func Origin() (*url.URL, error) { func HttpsOrigin() (string, error) { origin, err := Origin() if err != nil { - return "", err + return "", err } // if current repo is checked out with a SSH key if origin.Scheme != "https" { origin.Scheme = "https" } - // `git@` is not required for HTTPS, as Databricks Repos are checked + // `git@` is not required for HTTPS, as Databricks Repos are checked // out using an API token instead of username. But does it hold true // for all of the git implementations? if origin.User != nil { diff --git a/git/git_test.go b/git/git_test.go index 13721742..332cbdd0 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -19,4 +19,4 @@ func TestHttpsOrigin(t *testing.T) { // must pass on the upcoming forks assert.True(t, strings.HasPrefix(url, "https://github.com"), url) assert.True(t, strings.HasSuffix(url, "bricks.git"), url) -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 22c51dd8..79ed2a1c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/atotto/clipboard v0.1.4 + github.com/databricks/databricks-sdk-go v0.0.0 github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0 github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/hashicorp/go-version v1.5.0 // MPL 2.0 @@ -17,7 +18,7 @@ require ( github.com/spf13/cobra v1.4.0 // Apache 2.0 github.com/stretchr/testify v1.8.0 // MIT github.com/whilp/git-urls v1.0.0 // MIT - golang.org/x/mod v0.5.1 // BSD-3-Clause + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // BSD-3-Clause gopkg.in/ini.v1 v1.67.0 // Apache 2.0 ) @@ -43,12 +44,10 @@ require ( github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-checkpoint v0.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-plugin v1.4.3 // indirect github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl/v2 v2.12.0 // indirect @@ -56,9 +55,6 @@ require ( github.com/hashicorp/terraform-plugin-go v0.9.0 // indirect github.com/hashicorp/terraform-plugin-log v0.4.0 // indirect github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0 // indirect - github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896 // indirect - github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 // indirect - github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.10 // indirect @@ -67,7 +63,6 @@ require ( github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/oklog/run v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect @@ -76,16 +71,18 @@ require ( github.com/zclconf/go-cty v1.10.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect + golang.org/x/net v0.0.0-20220526153639-5463443f8c37 // indirect + golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - google.golang.org/api v0.79.0 // indirect + google.golang.org/api v0.82.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect - google.golang.org/grpc v1.46.0 // indirect + google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 // indirect + google.golang.org/grpc v1.46.2 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/databricks/databricks-sdk-go v0.0.0 => ./ext/databricks-sdk-go diff --git a/go.sum b/go.sum index 4201fdd0..01390964 100644 --- a/go.sum +++ b/go.sum @@ -240,12 +240,12 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -259,7 +259,6 @@ github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXc github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= @@ -293,12 +292,9 @@ github.com/hashicorp/terraform-plugin-log v0.4.0 h1:F3eVnm8r2EfQCe2k9blPIiF/r2TT github.com/hashicorp/terraform-plugin-log v0.4.0/go.mod h1:9KclxdunFownr4pIm1jdmwKRmE4d6HVG2c9XDq47rpg= github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0 h1:9fjPgCenJqnbjo95SDcbJ+YdLyEC1N35cwKWcRWhJTQ= github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0/go.mod h1:hLa0sTiySU/AWEgV2GxJh0/pQIqcCmm30IPja9N9lTg= -github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896 h1:1FGtlkJw87UsTMg5s8jrekrHmUPUJaMcu6ELiVhQrNw= github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896/go.mod h1:bzBPnUIkI0RxauU8Dqo+2KrZZ28Cf48s8V6IHt3p4co= -github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -309,7 +305,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -352,7 +347,6 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k+Mg7cowZ8yv4Trqw9UsJby758= github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs= -github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= @@ -470,8 +464,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -516,8 +511,9 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= +golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -536,8 +532,10 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 h1:uBgVQYJLi/m8M0wzp+aGwBWt90gMRoOVf+aWTW10QHI= +golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -549,6 +547,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -612,8 +611,10 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -688,6 +689,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -724,8 +726,10 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.79.0 h1:vaOcm0WdXvhGkci9a0+CcQVZqSRjN8ksSBlWv99f8Pg= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.79.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.82.0 h1:h6EGeZuzhoKSS7BUznzkW+2wHZ+4Ubd6rsVvvh3dRkw= +google.golang.org/api v0.82.0/go.mod h1:Ld58BeTlL9DIYr2M2ajvoSqmGLei0BMn+kVBmkam1os= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -807,8 +811,10 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 h1:q1kiSVscqoDeqTF27eQ2NnLLDmqF0I373qQNXYMy0fo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 h1:a221mAAEAzq4Lz6ZWRkcS8ptb2mxoxYSt4N68aRyQHM= +google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -838,8 +844,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/project/config.go b/project/config.go index ed1c5ef9..db502b9a 100644 --- a/project/config.go +++ b/project/config.go @@ -7,7 +7,8 @@ import ( "reflect" "github.com/databricks/bricks/folders" - "github.com/databrickslabs/terraform-provider-databricks/clusters" + "github.com/databricks/databricks-sdk-go/service/clusters" + "github.com/ghodss/yaml" ) @@ -33,7 +34,7 @@ type Project struct { Isolation Isolation `json:"isolation,omitempty"` // development-time vs deployment-time resources - DevCluster *clusters.Cluster `json:"dev_cluster,omitempty"` + DevCluster *clusters.ClusterInfo `json:"dev_cluster,omitempty"` // Assertions defines a list of configurations expected to be applied // to the workspace by a higher-privileged user (or service principal) @@ -61,7 +62,7 @@ func (p *Project) IsDevClusterJustReference() bool { if p.DevCluster.ClusterName == "" { return false } - return reflect.DeepEqual(p.DevCluster, &clusters.Cluster{ + return reflect.DeepEqual(p.DevCluster, &clusters.ClusterInfo{ ClusterName: p.DevCluster.ClusterName, }) } @@ -94,7 +95,7 @@ func loadProjectConf() (prj Project, err error) { } func validateAndApplyProjectDefaults(prj Project) (Project, error) { - // defaultCluster := clusters.Cluster{ + // defaultCluster := clusters.ClusterInfo{ // NodeTypeID: "smallest", // SparkVersion: "latest", // AutoterminationMinutes: 30, diff --git a/project/flavor.go b/project/flavor.go index cee95153..a04bec73 100644 --- a/project/flavor.go +++ b/project/flavor.go @@ -61,6 +61,7 @@ func toTheRootAndBack() func() { } var _ Flavor = PythonNotebooks{} + type PythonNotebooks struct{} func (n PythonNotebooks) Name() (string, string) { @@ -71,7 +72,7 @@ func (n PythonNotebooks) Name() (string, string) { func (n PythonNotebooks) Detected() bool { // TODO: Steps: // - get all filenames - // - read first X bytes from random 10 files and check + // - read first X bytes from random 10 files and check // if they're "Databricks Notebook Source" return false } diff --git a/project/project.go b/project/project.go index 4ea22ea4..f0566caf 100644 --- a/project/project.go +++ b/project/project.go @@ -5,8 +5,9 @@ import ( "fmt" "sync" - "github.com/databrickslabs/terraform-provider-databricks/clusters" - "github.com/databrickslabs/terraform-provider-databricks/commands" + "github.com/databricks/databricks-sdk-go/service/clusters" + "github.com/databricks/databricks-sdk-go/service/commands" + "github.com/databricks/databricks-sdk-go/workspaces" "github.com/databrickslabs/terraform-provider-databricks/common" "github.com/databrickslabs/terraform-provider-databricks/scim" ) @@ -19,6 +20,7 @@ type inner struct { once sync.Once project *Project + wsc *workspaces.WorkspacesClient client *common.DatabricksClient me *scim.User } @@ -28,11 +30,8 @@ func (i *inner) init() { defer i.mu.Unlock() i.once.Do(func() { client := &common.DatabricksClient{} - client.WithCommandExecutor(func( - ctx context.Context, c *common.DatabricksClient) common.CommandExecutor { - return commands.NewCommandsAPI(ctx, c) - }) i.client = client + i.wsc = workspaces.New() prj, err := loadProjectConf() if err != nil { panic(err) @@ -42,7 +41,6 @@ func (i *inner) init() { if err != nil { panic(err) } - i.project = &prj }) } @@ -57,6 +55,13 @@ func (i *inner) Project() *Project { return i.project } +// Make sure to initialize the workspaces client on project init +func (i *inner) WorkspacesClient() *workspaces.WorkspacesClient { + i.init() + return i.wsc +} + +// We can replace this with go sdk once https://github.com/databricks/databricks-sdk-go/issues/56 is fixed func (i *inner) Me() *scim.User { i.mu.Lock() defer i.mu.Unlock() @@ -82,6 +87,28 @@ func (i *inner) DeploymentIsolationPrefix() string { panic(fmt.Errorf("unknow project isolation: %s", i.project.Isolation)) } +func getClusterIdFromClusterName(ctx context.Context, + wsc *workspaces.WorkspacesClient, + clusterName string, +) (clusterId string, err error) { + clusterId = "" + clustersList, err := wsc.Clusters.List(ctx, clusters.ListRequest{}) + if err != nil { + return + } + for _, cluster := range clustersList.Clusters { + if cluster.ClusterName == clusterName { + clusterId = cluster.ClusterId + return + } + } + err = fmt.Errorf("could not find cluster with name: %s", clusterName) + return +} + +// Old version of getting development cluster details with isolation implemented. +// Kept just for reference. Remove once isolation is implemented properly +/* func (i *inner) DevelopmentCluster(ctx context.Context) (cluster clusters.ClusterInfo, err error) { api := clusters.NewClustersAPI(ctx, i.Client()) // TODO: rewrite with normal SDK if i.project.DevCluster == nil { @@ -117,3 +144,37 @@ func runCommandOnDev(ctx context.Context, language, command string) common.Comma func RunPythonOnDev(ctx context.Context, command string) common.CommandResults { return runCommandOnDev(ctx, "python", command) } +*/ + +// TODO: Add safe access to i.project and i.project.DevCluster that throws errors if +// the fields are not defined properly +func (i *inner) GetDevelopmentClusterId(ctx context.Context) (clusterId string, err error) { + i.init() + clusterId = i.project.DevCluster.ClusterId + clusterName := i.project.DevCluster.ClusterName + if clusterId != "" { + return + } else if clusterName != "" { + // Add workspaces client on init + return getClusterIdFromClusterName(ctx, i.wsc, clusterName) + } else { + // TODO: Add the project config file location used to error message + err = fmt.Errorf("please define either development cluster's cluster_id or cluster_name in your project config") + return + } +} + +func runCommandOnDev(ctx context.Context, language, command string) commands.CommandResults { + clusterId, err := Current.GetDevelopmentClusterId(ctx) + if err != nil { + return commands.CommandResults{ + ResultType: "error", + Summary: err.Error(), + } + } + return Current.wsc.Commands.Execute(ctx, clusterId, language, command) +} + +func RunPythonOnDev(ctx context.Context, command string) commands.CommandResults { + return runCommandOnDev(ctx, "python", command) +} diff --git a/python/wheel.go b/python/wheel.go index 94cb6d72..508355fc 100644 --- a/python/wheel.go +++ b/python/wheel.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/databricks/bricks/project" - "github.com/databrickslabs/terraform-provider-databricks/storage" + "github.com/databricks/bricks/utilities" ) func BuildWheel(ctx context.Context, dir string) (string, error) { @@ -19,7 +19,6 @@ func BuildWheel(ctx context.Context, dir string) (string, error) { os.RemoveAll("dist") // remove all other irrelevant traces silentlyCleanupWheelFolder(".") - // call simple wheel builder. we may need to pip install wheel as well out, err := Py(ctx, "setup.py", "bdist_wheel") if err != nil { @@ -49,37 +48,41 @@ const DBFSWheelLocation = "dbfs:/FileStore/wheels/simple" func UploadWheelToDBFSWithPEP503(ctx context.Context, dir string) (string, error) { wheel, err := BuildWheel(ctx, dir) if err != nil { - return "", err + return "", err } defer chdirAndBack(dir)() dist, err := ReadDistribution(ctx) if err != nil { - return "", err + return "", err } - // TODO: figure out wheel naming criteria for Soft project isolation to allow multiple + // TODO: figure out wheel naming criteria for Soft project isolation to allow multiple // people workin on the same project to upload wheels and let them be deployed as independent jobs. // we should also consider multiple PEP503 index stacking: per enterprise, per project, per developer. // PEP503 indexes can be rolled out to clusters via checksummed global init script, that creates // a driver/worker `/etc/pip.conf` with FUSE-mounted file:///dbfs/FileStore/wheels/simple/.. // extra index URLs. See more pointers at https://stackoverflow.com/q/30889494/277035 dbfsLoc := fmt.Sprintf("%s/%s/%s", DBFSWheelLocation, dist.NormalizedName(), path.Base(wheel)) - dbfs := storage.NewDbfsAPI(ctx, project.Current.Client()) // circular dep?.. + + wsc := project.Current.WorkspacesClient() wf, err := os.Open(wheel) if err != nil { - return "", err + return "", err } defer wf.Close() raw, err := io.ReadAll(wf) if err != nil { - return "", err - } - err = dbfs.Create(dbfsLoc, raw, true) - if err != nil { - return "", err + return "", err } + // err = dbfs.Create(dbfsLoc, raw, true) + err = utilities.CreateDbfsFile(ctx, + wsc, + dbfsLoc, + raw, + true, + ) // TODO: maintain PEP503 compliance and update meta-files: // ${DBFSWheelLocation}/index.html and ${DBFSWheelLocation}/${NormalizedName}/index.html - return dbfsLoc, nil + return dbfsLoc, err } func silentlyCleanupWheelFolder(dir string) { diff --git a/python/wheel_test.go b/python/wheel_test.go index 2ad2cf4b..cc74eaf2 100644 --- a/python/wheel_test.go +++ b/python/wheel_test.go @@ -21,4 +21,4 @@ func TestWheel(t *testing.T) { func noFile(t *testing.T, name string) { _, err := os.Stat(name) assert.Error(t, err, "file %s should exist", name) -} \ No newline at end of file +} diff --git a/terraform/runner.go b/terraform/runner.go index 168c2480..bbe2d510 100644 --- a/terraform/runner.go +++ b/terraform/runner.go @@ -48,7 +48,7 @@ import ( "os" "github.com/databricks/bricks/project" - "github.com/databrickslabs/terraform-provider-databricks/storage" + "github.com/databricks/bricks/utilities" "github.com/hashicorp/go-version" "github.com/hashicorp/hc-install/product" "github.com/hashicorp/hc-install/releases" @@ -87,8 +87,10 @@ func (d *TerraformDeployer) remoteTfstateLoc() string { // returns structured representation of terraform state on DBFS. func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) { - dbfs := storage.NewDbfsAPI(ctx, project.Current.Client()) - raw, err := dbfs.Read(d.remoteTfstateLoc()) + raw, err := utilities.ReadDbfsFile(ctx, + project.Current.WorkspacesClient(), + d.remoteTfstateLoc(), + ) if err != nil { return nil, 0, err } @@ -102,8 +104,8 @@ func (d *TerraformDeployer) openLocalState() (*os.File, error) { return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir)) } -// returns structured representation of terraform state on local machine. as part of -// the optimistic concurrency control, please make sure to always compare the serial +// returns structured representation of terraform state on local machine. as part of +// the optimistic concurrency control, please make sure to always compare the serial // number of local and remote states before proceeding with deployment. func (d *TerraformDeployer) localState() (*tfjson.State, int, error) { local, err := d.openLocalState() @@ -114,7 +116,7 @@ func (d *TerraformDeployer) localState() (*tfjson.State, int, error) { return d.tfstateFromReader(local) } -// converts input stream into structured representation of terraform state and deployment +// converts input stream into structured representation of terraform state and deployment // serial number, that helps controlling versioning and synchronisation via optimistic locking. func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) { var state tfjson.State @@ -142,7 +144,6 @@ func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, // uploads terraform state from local directory to designated DBFS location. func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error { - dbfs := storage.NewDbfsAPI(ctx, project.Current.Client()) local, err := d.openLocalState() if err != nil { return err @@ -153,7 +154,12 @@ func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error { return err } // TODO: make sure that deployment locks are implemented - return dbfs.Create(d.remoteTfstateLoc(), raw, true) + return utilities.CreateDbfsFile(ctx, + project.Current.WorkspacesClient(), + d.remoteTfstateLoc(), + raw, + true, + ) } // downloads terraform state from DBFS to local working directory. diff --git a/utilities/dbfs.go b/utilities/dbfs.go new file mode 100644 index 00000000..78d12d89 --- /dev/null +++ b/utilities/dbfs.go @@ -0,0 +1,93 @@ +package utilities + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/dbfs" + "github.com/databricks/databricks-sdk-go/workspaces" +) + +// move to go sdk / replace with utility function once +// https://github.com/databricks/databricks-sdk-go/issues/57 is Done +// Tracked in https://github.com/databricks/bricks/issues/25 +func CreateDbfsFile(ctx context.Context, + wsc *workspaces.WorkspacesClient, + path string, + contents []byte, + overwrite bool, +) error { + // see https://docs.databricks.com/dev-tools/api/latest/dbfs.html#add-block + const WRITE_BYTE_CHUNK_SIZE = 1e6 + createResponse, err := wsc.Dbfs.Create(ctx, + dbfs.CreateRequest{ + Overwrite: overwrite, + Path: path, + }, + ) + if err != nil { + return err + } + handle := createResponse.Handle + buffer := bytes.NewBuffer(contents) + for { + byteChunk := buffer.Next(WRITE_BYTE_CHUNK_SIZE) + if len(byteChunk) == 0 { + break + } + b64Data := base64.StdEncoding.EncodeToString(byteChunk) + err := wsc.Dbfs.AddBlock(ctx, + dbfs.AddBlockRequest{ + Data: b64Data, + Handle: handle, + }, + ) + if err != nil { + return fmt.Errorf("cannot add block: %w", err) + } + } + err = wsc.Dbfs.Close(ctx, + dbfs.CloseRequest{ + Handle: handle, + }, + ) + if err != nil { + return fmt.Errorf("cannot close handle: %w", err) + } + return nil +} + +func ReadDbfsFile(ctx context.Context, + wsc *workspaces.WorkspacesClient, + path string, +) (content []byte, err error) { + // see https://docs.databricks.com/dev-tools/api/latest/dbfs.html#read + const READ_BYTE_CHUNK_SIZE = 1e6 + fetchLoop := true + offSet := 0 + length := int(READ_BYTE_CHUNK_SIZE) + for fetchLoop { + dbfsReadReponse, err := wsc.Dbfs.Read(ctx, + dbfs.ReadRequest{ + Path: path, + Offset: offSet, + Length: length, + }, + ) + if err != nil { + return content, fmt.Errorf("cannot read %s: %w", path, err) + } + if dbfsReadReponse.BytesRead == 0 || dbfsReadReponse.BytesRead < int64(length) { + fetchLoop = false + } + decodedBytes, err := base64.StdEncoding.DecodeString(dbfsReadReponse.Data) + if err != nil { + return content, err + } + content = append(content, decodedBytes...) + offSet += length + } + return content, err +} diff --git a/utilities/repos.go b/utilities/repos.go new file mode 100644 index 00000000..14807b54 --- /dev/null +++ b/utilities/repos.go @@ -0,0 +1,29 @@ +package utilities + +import ( + "context" + + "github.com/databricks/databricks-sdk-go/service/repos" + "github.com/databricks/databricks-sdk-go/workspaces" +) + +// Remove once this function is in go sdk +// https://github.com/databricks/databricks-sdk-go/issues/58 +// Tracked in : https://github.com/databricks/bricks/issues/26 +func GetAllRepos(ctx context.Context, wsc *workspaces.WorkspacesClient, pathPrefix string) (resultRepos []repos.RepoInfo, err error) { + nextPageToken := "" + for { + listReposResponse, err := wsc.Repos.List(ctx, repos.ListRequest{ + PathPrefix: pathPrefix, + NextPageToken: nextPageToken, + }) + if err != nil { + break + } + resultRepos = append(resultRepos, listReposResponse.Repos...) + if nextPageToken == "" { + break + } + } + return +}