mirror of https://github.com/databricks/cli.git
Replace terraform dependency with go sdk (#19)
Issue: https://github.com/databricks/bricks/issues/17 `./bricks fs ls ...` command works `./bricks launch ...` command works Did not test other changes as the readme claims other commands don't work anyways :) cc: @nfx TODO left for this PR: 2. Replace terraform scim.Me once its there in go SDK (https://github.com/databricks/databricks-sdk-go/issues/56)
This commit is contained in:
shreyas-goenka
GitHub
parent
f070c24399
commit
96efd0e2e4
|
|
@ -10,8 +10,11 @@ jobs:
|
|||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
- name: Checkout repository and submodules
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
token: ${{ secrets.DECO_GITHUB_TOKEN }}
|
||||
submodules: recursive
|
||||
|
||||
- name: Unshallow
|
||||
run: git fetch --prune --unshallow
|
||||
|
|
@ -19,7 +22,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.16.x
|
||||
go-version: 1.18.x
|
||||
|
||||
- name: Cache packages
|
||||
uses: actions/cache@v2
|
||||
|
|
@ -38,7 +41,7 @@ jobs:
|
|||
echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
|
||||
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
|
||||
go get gotest.tools/gotestsum
|
||||
go install honnef.co/go/tools/cmd/staticcheck@v0.2.2
|
||||
go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
|
||||
- name: Pull external libraries
|
||||
run: make vendor
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ jobs:
|
|||
- name: Set up Go
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
go-version: 1.18
|
||||
- name: Import GPG key
|
||||
id: import_gpg
|
||||
uses: hashicorp/ghaction-import-gpg@v2.1.0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "ext/databricks-sdk-go"]
|
||||
path = ext/databricks-sdk-go
|
||||
url = git@github.com:databricks/databricks-sdk-go.git
|
||||
|
|
@ -5,8 +5,6 @@ import (
|
|||
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/databrickslabs/terraform-provider-databricks/storage"
|
||||
)
|
||||
|
||||
// lsCmd represents the ls command
|
||||
|
|
@ -16,11 +14,12 @@ var lsCmd = &cobra.Command{
|
|||
Long: `Lists files`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
api := storage.NewDbfsAPI(cmd.Context(), project.Current.Client())
|
||||
files, err := api.List(args[0], false)
|
||||
wsc := project.Current.WorkspacesClient()
|
||||
listStatusResponse, err := wsc.Dbfs.ListByPath(cmd.Context(), args[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
files := listStatusResponse.Files
|
||||
// TODO: output formatting: JSON, CSV, tables and default
|
||||
for _, v := range files {
|
||||
fmt.Printf("[-] %s (%d, %v)\n", v.Path, v.FileSize, v.IsDir)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ var launchCmd = &cobra.Command{
|
|||
}
|
||||
|
||||
func init() {
|
||||
// TODO: detect if we can skip registering
|
||||
// TODO: detect if we can skip registering
|
||||
// launch command for DBSQL projects
|
||||
root.RootCmd.AddCommand(launchCmd)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ func githubGetPAT(ctx context.Context) (string, error) {
|
|||
}
|
||||
if result.Get("error") == "slow_down" {
|
||||
t, _ := strconv.Atoi(result.Get("interval"))
|
||||
time.Sleep(time.Duration(t)*time.Second)
|
||||
time.Sleep(time.Duration(t) * time.Second)
|
||||
log.Printf("[WARN] Rate limited, sleeping for %d seconds", t)
|
||||
}
|
||||
reason := result.Get("error_description")
|
||||
|
|
|
|||
|
|
@ -11,4 +11,4 @@ func TestGithubGetPAT(t *testing.T) {
|
|||
pat, err := githubGetPAT(context.Background())
|
||||
assert.NoError(t, err)
|
||||
assert.NotEqual(t, "..", pat)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ func (d diff) IsEmpty() bool {
|
|||
return len(d.put) == 0 && len(d.delete) == 0
|
||||
}
|
||||
|
||||
|
||||
func (d diff) String() string {
|
||||
if d.IsEmpty() {
|
||||
return "no changes"
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import (
|
|||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/databricks/bricks/git"
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/repos"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/workspace"
|
||||
"github.com/databricks/bricks/utilities"
|
||||
"github.com/databricks/databricks-sdk-go/service/workspace"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
@ -28,11 +28,9 @@ var syncCmd = &cobra.Command{
|
|||
}
|
||||
log.Printf("[INFO] %s", origin)
|
||||
ctx := cmd.Context()
|
||||
client := project.Current.Client()
|
||||
reposAPI := repos.NewReposAPI(ctx, client)
|
||||
|
||||
|
||||
checkouts, err := reposAPI.List("/")
|
||||
wsc := project.Current.WorkspacesClient()
|
||||
checkouts, err := utilities.GetAllRepos(ctx, wsc, "/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -46,9 +44,13 @@ var syncCmd = &cobra.Command{
|
|||
}
|
||||
base := fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName)
|
||||
return watchForChanges(ctx, git.MustGetFileSet(), *interval, func(d diff) error {
|
||||
wsAPI := workspace.NewNotebooksAPI(ctx, client)
|
||||
for _, v := range d.delete {
|
||||
err := wsAPI.Delete(path.Join(base, v), true)
|
||||
err := wsc.Workspace.Delete(ctx,
|
||||
workspace.DeleteRequest{
|
||||
Path: path.Join(base, v),
|
||||
Recursive: true,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,4 +13,4 @@ func TestItSyncs(t *testing.T) {
|
|||
root.RootCmd.SetArgs([]string{"sync"})
|
||||
err := root.RootCmd.ExecuteContext(ctx)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 9e9cc8f8115b8d3cb9e778806066c7cffe5b9b76
|
||||
|
|
@ -46,13 +46,13 @@ func Origin() (*url.URL, error) {
|
|||
func HttpsOrigin() (string, error) {
|
||||
origin, err := Origin()
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", err
|
||||
}
|
||||
// if current repo is checked out with a SSH key
|
||||
if origin.Scheme != "https" {
|
||||
origin.Scheme = "https"
|
||||
}
|
||||
// `git@` is not required for HTTPS, as Databricks Repos are checked
|
||||
// `git@` is not required for HTTPS, as Databricks Repos are checked
|
||||
// out using an API token instead of username. But does it hold true
|
||||
// for all of the git implementations?
|
||||
if origin.User != nil {
|
||||
|
|
|
|||
|
|
@ -19,4 +19,4 @@ func TestHttpsOrigin(t *testing.T) {
|
|||
// must pass on the upcoming forks
|
||||
assert.True(t, strings.HasPrefix(url, "https://github.com"), url)
|
||||
assert.True(t, strings.HasSuffix(url, "bricks.git"), url)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
23
go.mod
23
go.mod
|
|
@ -4,6 +4,7 @@ go 1.18
|
|||
|
||||
require (
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/databricks/databricks-sdk-go v0.0.0
|
||||
github.com/databrickslabs/terraform-provider-databricks v0.5.8 // Apache 2.0
|
||||
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
|
||||
github.com/hashicorp/go-version v1.5.0 // MPL 2.0
|
||||
|
|
@ -17,7 +18,7 @@ require (
|
|||
github.com/spf13/cobra v1.4.0 // Apache 2.0
|
||||
github.com/stretchr/testify v1.8.0 // MIT
|
||||
github.com/whilp/git-urls v1.0.0 // MIT
|
||||
golang.org/x/mod v0.5.1 // BSD-3-Clause
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // BSD-3-Clause
|
||||
gopkg.in/ini.v1 v1.67.0 // Apache 2.0
|
||||
)
|
||||
|
||||
|
|
@ -43,12 +44,10 @@ require (
|
|||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-checkpoint v0.5.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-plugin v1.4.3 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl/v2 v2.12.0 // indirect
|
||||
|
|
@ -56,9 +55,6 @@ require (
|
|||
github.com/hashicorp/terraform-plugin-go v0.9.0 // indirect
|
||||
github.com/hashicorp/terraform-plugin-log v0.4.0 // indirect
|
||||
github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0 // indirect
|
||||
github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896 // indirect
|
||||
github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 // indirect
|
||||
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.10 // indirect
|
||||
|
|
@ -67,7 +63,6 @@ require (
|
|||
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/oklog/run v1.0.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
|
||||
|
|
@ -76,16 +71,18 @@ require (
|
|||
github.com/zclconf/go-cty v1.10.0 // indirect
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
|
||||
golang.org/x/net v0.0.0-20220526153639-5463443f8c37 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 // indirect
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
|
||||
google.golang.org/api v0.79.0 // indirect
|
||||
google.golang.org/api v0.82.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 // indirect
|
||||
google.golang.org/grpc v1.46.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 // indirect
|
||||
google.golang.org/grpc v1.46.2 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
replace github.com/databricks/databricks-sdk-go v0.0.0 => ./ext/databricks-sdk-go
|
||||
|
|
|
|||
37
go.sum
37
go.sum
|
|
@ -240,12 +240,12 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
|
|||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
|
||||
github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
|
||||
github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI=
|
||||
github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
|
||||
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=
|
||||
github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
|
|
@ -259,7 +259,6 @@ github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXc
|
|||
github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM=
|
||||
github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
|
||||
|
|
@ -293,12 +292,9 @@ github.com/hashicorp/terraform-plugin-log v0.4.0 h1:F3eVnm8r2EfQCe2k9blPIiF/r2TT
|
|||
github.com/hashicorp/terraform-plugin-log v0.4.0/go.mod h1:9KclxdunFownr4pIm1jdmwKRmE4d6HVG2c9XDq47rpg=
|
||||
github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0 h1:9fjPgCenJqnbjo95SDcbJ+YdLyEC1N35cwKWcRWhJTQ=
|
||||
github.com/hashicorp/terraform-plugin-sdk/v2 v2.16.0/go.mod h1:hLa0sTiySU/AWEgV2GxJh0/pQIqcCmm30IPja9N9lTg=
|
||||
github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896 h1:1FGtlkJw87UsTMg5s8jrekrHmUPUJaMcu6ELiVhQrNw=
|
||||
github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896/go.mod h1:bzBPnUIkI0RxauU8Dqo+2KrZZ28Cf48s8V6IHt3p4co=
|
||||
github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
|
||||
github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
|
||||
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
|
||||
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
|
||||
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
|
|
@ -309,7 +305,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
|
|||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
|
||||
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
|
||||
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
|
|
@ -352,7 +347,6 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
|
|||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce h1:RPclfga2SEJmgMmz2k+Mg7cowZ8yv4Trqw9UsJby758=
|
||||
github.com/nsf/jsondiff v0.0.0-20200515183724-f29ed568f4ce/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
|
||||
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
|
|
@ -470,8 +464,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
|
||||
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
|
||||
golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -516,8 +511,9 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
|||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8=
|
||||
golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -536,8 +532,10 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ
|
|||
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 h1:uBgVQYJLi/m8M0wzp+aGwBWt90gMRoOVf+aWTW10QHI=
|
||||
golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -549,6 +547,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -612,8 +611,10 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
|
||||
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -688,6 +689,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
|
|||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
|
@ -724,8 +726,10 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
|
|||
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
|
||||
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
|
||||
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
|
||||
google.golang.org/api v0.79.0 h1:vaOcm0WdXvhGkci9a0+CcQVZqSRjN8ksSBlWv99f8Pg=
|
||||
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
|
||||
google.golang.org/api v0.79.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
|
||||
google.golang.org/api v0.82.0 h1:h6EGeZuzhoKSS7BUznzkW+2wHZ+4Ubd6rsVvvh3dRkw=
|
||||
google.golang.org/api v0.82.0/go.mod h1:Ld58BeTlL9DIYr2M2ajvoSqmGLei0BMn+kVBmkam1os=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -807,8 +811,10 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX
|
|||
google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3 h1:q1kiSVscqoDeqTF27eQ2NnLLDmqF0I373qQNXYMy0fo=
|
||||
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 h1:a221mAAEAzq4Lz6ZWRkcS8ptb2mxoxYSt4N68aRyQHM=
|
||||
google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
|
||||
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
|
|
@ -838,8 +844,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
|
|||
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
|
||||
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
|
||||
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
|
||||
google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
|
||||
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ=
|
||||
google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ import (
|
|||
"reflect"
|
||||
|
||||
"github.com/databricks/bricks/folders"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/clusters"
|
||||
"github.com/databricks/databricks-sdk-go/service/clusters"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
)
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ type Project struct {
|
|||
Isolation Isolation `json:"isolation,omitempty"`
|
||||
|
||||
// development-time vs deployment-time resources
|
||||
DevCluster *clusters.Cluster `json:"dev_cluster,omitempty"`
|
||||
DevCluster *clusters.ClusterInfo `json:"dev_cluster,omitempty"`
|
||||
|
||||
// Assertions defines a list of configurations expected to be applied
|
||||
// to the workspace by a higher-privileged user (or service principal)
|
||||
|
|
@ -61,7 +62,7 @@ func (p *Project) IsDevClusterJustReference() bool {
|
|||
if p.DevCluster.ClusterName == "" {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(p.DevCluster, &clusters.Cluster{
|
||||
return reflect.DeepEqual(p.DevCluster, &clusters.ClusterInfo{
|
||||
ClusterName: p.DevCluster.ClusterName,
|
||||
})
|
||||
}
|
||||
|
|
@ -94,7 +95,7 @@ func loadProjectConf() (prj Project, err error) {
|
|||
}
|
||||
|
||||
func validateAndApplyProjectDefaults(prj Project) (Project, error) {
|
||||
// defaultCluster := clusters.Cluster{
|
||||
// defaultCluster := clusters.ClusterInfo{
|
||||
// NodeTypeID: "smallest",
|
||||
// SparkVersion: "latest",
|
||||
// AutoterminationMinutes: 30,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ func toTheRootAndBack() func() {
|
|||
}
|
||||
|
||||
var _ Flavor = PythonNotebooks{}
|
||||
|
||||
type PythonNotebooks struct{}
|
||||
|
||||
func (n PythonNotebooks) Name() (string, string) {
|
||||
|
|
@ -71,7 +72,7 @@ func (n PythonNotebooks) Name() (string, string) {
|
|||
func (n PythonNotebooks) Detected() bool {
|
||||
// TODO: Steps:
|
||||
// - get all filenames
|
||||
// - read first X bytes from random 10 files and check
|
||||
// - read first X bytes from random 10 files and check
|
||||
// if they're "Databricks Notebook Source"
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@ import (
|
|||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/databrickslabs/terraform-provider-databricks/clusters"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/commands"
|
||||
"github.com/databricks/databricks-sdk-go/service/clusters"
|
||||
"github.com/databricks/databricks-sdk-go/service/commands"
|
||||
"github.com/databricks/databricks-sdk-go/workspaces"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/common"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/scim"
|
||||
)
|
||||
|
|
@ -19,6 +20,7 @@ type inner struct {
|
|||
once sync.Once
|
||||
|
||||
project *Project
|
||||
wsc *workspaces.WorkspacesClient
|
||||
client *common.DatabricksClient
|
||||
me *scim.User
|
||||
}
|
||||
|
|
@ -28,11 +30,8 @@ func (i *inner) init() {
|
|||
defer i.mu.Unlock()
|
||||
i.once.Do(func() {
|
||||
client := &common.DatabricksClient{}
|
||||
client.WithCommandExecutor(func(
|
||||
ctx context.Context, c *common.DatabricksClient) common.CommandExecutor {
|
||||
return commands.NewCommandsAPI(ctx, c)
|
||||
})
|
||||
i.client = client
|
||||
i.wsc = workspaces.New()
|
||||
prj, err := loadProjectConf()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -42,7 +41,6 @@ func (i *inner) init() {
|
|||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
i.project = &prj
|
||||
})
|
||||
}
|
||||
|
|
@ -57,6 +55,13 @@ func (i *inner) Project() *Project {
|
|||
return i.project
|
||||
}
|
||||
|
||||
// Make sure to initialize the workspaces client on project init
|
||||
func (i *inner) WorkspacesClient() *workspaces.WorkspacesClient {
|
||||
i.init()
|
||||
return i.wsc
|
||||
}
|
||||
|
||||
// We can replace this with go sdk once https://github.com/databricks/databricks-sdk-go/issues/56 is fixed
|
||||
func (i *inner) Me() *scim.User {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
|
@ -82,6 +87,28 @@ func (i *inner) DeploymentIsolationPrefix() string {
|
|||
panic(fmt.Errorf("unknow project isolation: %s", i.project.Isolation))
|
||||
}
|
||||
|
||||
func getClusterIdFromClusterName(ctx context.Context,
|
||||
wsc *workspaces.WorkspacesClient,
|
||||
clusterName string,
|
||||
) (clusterId string, err error) {
|
||||
clusterId = ""
|
||||
clustersList, err := wsc.Clusters.List(ctx, clusters.ListRequest{})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, cluster := range clustersList.Clusters {
|
||||
if cluster.ClusterName == clusterName {
|
||||
clusterId = cluster.ClusterId
|
||||
return
|
||||
}
|
||||
}
|
||||
err = fmt.Errorf("could not find cluster with name: %s", clusterName)
|
||||
return
|
||||
}
|
||||
|
||||
// Old version of getting development cluster details with isolation implemented.
|
||||
// Kept just for reference. Remove once isolation is implemented properly
|
||||
/*
|
||||
func (i *inner) DevelopmentCluster(ctx context.Context) (cluster clusters.ClusterInfo, err error) {
|
||||
api := clusters.NewClustersAPI(ctx, i.Client()) // TODO: rewrite with normal SDK
|
||||
if i.project.DevCluster == nil {
|
||||
|
|
@ -117,3 +144,37 @@ func runCommandOnDev(ctx context.Context, language, command string) common.Comma
|
|||
func RunPythonOnDev(ctx context.Context, command string) common.CommandResults {
|
||||
return runCommandOnDev(ctx, "python", command)
|
||||
}
|
||||
*/
|
||||
|
||||
// TODO: Add safe access to i.project and i.project.DevCluster that throws errors if
|
||||
// the fields are not defined properly
|
||||
func (i *inner) GetDevelopmentClusterId(ctx context.Context) (clusterId string, err error) {
|
||||
i.init()
|
||||
clusterId = i.project.DevCluster.ClusterId
|
||||
clusterName := i.project.DevCluster.ClusterName
|
||||
if clusterId != "" {
|
||||
return
|
||||
} else if clusterName != "" {
|
||||
// Add workspaces client on init
|
||||
return getClusterIdFromClusterName(ctx, i.wsc, clusterName)
|
||||
} else {
|
||||
// TODO: Add the project config file location used to error message
|
||||
err = fmt.Errorf("please define either development cluster's cluster_id or cluster_name in your project config")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func runCommandOnDev(ctx context.Context, language, command string) commands.CommandResults {
|
||||
clusterId, err := Current.GetDevelopmentClusterId(ctx)
|
||||
if err != nil {
|
||||
return commands.CommandResults{
|
||||
ResultType: "error",
|
||||
Summary: err.Error(),
|
||||
}
|
||||
}
|
||||
return Current.wsc.Commands.Execute(ctx, clusterId, language, command)
|
||||
}
|
||||
|
||||
func RunPythonOnDev(ctx context.Context, command string) commands.CommandResults {
|
||||
return runCommandOnDev(ctx, "python", command)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/storage"
|
||||
"github.com/databricks/bricks/utilities"
|
||||
)
|
||||
|
||||
func BuildWheel(ctx context.Context, dir string) (string, error) {
|
||||
|
|
@ -19,7 +19,6 @@ func BuildWheel(ctx context.Context, dir string) (string, error) {
|
|||
os.RemoveAll("dist")
|
||||
// remove all other irrelevant traces
|
||||
silentlyCleanupWheelFolder(".")
|
||||
|
||||
// call simple wheel builder. we may need to pip install wheel as well
|
||||
out, err := Py(ctx, "setup.py", "bdist_wheel")
|
||||
if err != nil {
|
||||
|
|
@ -49,37 +48,41 @@ const DBFSWheelLocation = "dbfs:/FileStore/wheels/simple"
|
|||
func UploadWheelToDBFSWithPEP503(ctx context.Context, dir string) (string, error) {
|
||||
wheel, err := BuildWheel(ctx, dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", err
|
||||
}
|
||||
defer chdirAndBack(dir)()
|
||||
dist, err := ReadDistribution(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", err
|
||||
}
|
||||
// TODO: figure out wheel naming criteria for Soft project isolation to allow multiple
|
||||
// TODO: figure out wheel naming criteria for Soft project isolation to allow multiple
|
||||
// people workin on the same project to upload wheels and let them be deployed as independent jobs.
|
||||
// we should also consider multiple PEP503 index stacking: per enterprise, per project, per developer.
|
||||
// PEP503 indexes can be rolled out to clusters via checksummed global init script, that creates
|
||||
// a driver/worker `/etc/pip.conf` with FUSE-mounted file:///dbfs/FileStore/wheels/simple/..
|
||||
// extra index URLs. See more pointers at https://stackoverflow.com/q/30889494/277035
|
||||
dbfsLoc := fmt.Sprintf("%s/%s/%s", DBFSWheelLocation, dist.NormalizedName(), path.Base(wheel))
|
||||
dbfs := storage.NewDbfsAPI(ctx, project.Current.Client()) // circular dep?..
|
||||
|
||||
wsc := project.Current.WorkspacesClient()
|
||||
wf, err := os.Open(wheel)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", err
|
||||
}
|
||||
defer wf.Close()
|
||||
raw, err := io.ReadAll(wf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
err = dbfs.Create(dbfsLoc, raw, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", err
|
||||
}
|
||||
// err = dbfs.Create(dbfsLoc, raw, true)
|
||||
err = utilities.CreateDbfsFile(ctx,
|
||||
wsc,
|
||||
dbfsLoc,
|
||||
raw,
|
||||
true,
|
||||
)
|
||||
// TODO: maintain PEP503 compliance and update meta-files:
|
||||
// ${DBFSWheelLocation}/index.html and ${DBFSWheelLocation}/${NormalizedName}/index.html
|
||||
return dbfsLoc, nil
|
||||
return dbfsLoc, err
|
||||
}
|
||||
|
||||
func silentlyCleanupWheelFolder(dir string) {
|
||||
|
|
|
|||
|
|
@ -21,4 +21,4 @@ func TestWheel(t *testing.T) {
|
|||
func noFile(t *testing.T, name string) {
|
||||
_, err := os.Stat(name)
|
||||
assert.Error(t, err, "file %s should exist", name)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ import (
|
|||
"os"
|
||||
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/databrickslabs/terraform-provider-databricks/storage"
|
||||
"github.com/databricks/bricks/utilities"
|
||||
"github.com/hashicorp/go-version"
|
||||
"github.com/hashicorp/hc-install/product"
|
||||
"github.com/hashicorp/hc-install/releases"
|
||||
|
|
@ -87,8 +87,10 @@ func (d *TerraformDeployer) remoteTfstateLoc() string {
|
|||
|
||||
// returns structured representation of terraform state on DBFS.
|
||||
func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) {
|
||||
dbfs := storage.NewDbfsAPI(ctx, project.Current.Client())
|
||||
raw, err := dbfs.Read(d.remoteTfstateLoc())
|
||||
raw, err := utilities.ReadDbfsFile(ctx,
|
||||
project.Current.WorkspacesClient(),
|
||||
d.remoteTfstateLoc(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
|
@ -102,8 +104,8 @@ func (d *TerraformDeployer) openLocalState() (*os.File, error) {
|
|||
return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir))
|
||||
}
|
||||
|
||||
// returns structured representation of terraform state on local machine. as part of
|
||||
// the optimistic concurrency control, please make sure to always compare the serial
|
||||
// returns structured representation of terraform state on local machine. as part of
|
||||
// the optimistic concurrency control, please make sure to always compare the serial
|
||||
// number of local and remote states before proceeding with deployment.
|
||||
func (d *TerraformDeployer) localState() (*tfjson.State, int, error) {
|
||||
local, err := d.openLocalState()
|
||||
|
|
@ -114,7 +116,7 @@ func (d *TerraformDeployer) localState() (*tfjson.State, int, error) {
|
|||
return d.tfstateFromReader(local)
|
||||
}
|
||||
|
||||
// converts input stream into structured representation of terraform state and deployment
|
||||
// converts input stream into structured representation of terraform state and deployment
|
||||
// serial number, that helps controlling versioning and synchronisation via optimistic locking.
|
||||
func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) {
|
||||
var state tfjson.State
|
||||
|
|
@ -142,7 +144,6 @@ func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State,
|
|||
|
||||
// uploads terraform state from local directory to designated DBFS location.
|
||||
func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error {
|
||||
dbfs := storage.NewDbfsAPI(ctx, project.Current.Client())
|
||||
local, err := d.openLocalState()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -153,7 +154,12 @@ func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error {
|
|||
return err
|
||||
}
|
||||
// TODO: make sure that deployment locks are implemented
|
||||
return dbfs.Create(d.remoteTfstateLoc(), raw, true)
|
||||
return utilities.CreateDbfsFile(ctx,
|
||||
project.Current.WorkspacesClient(),
|
||||
d.remoteTfstateLoc(),
|
||||
raw,
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
// downloads terraform state from DBFS to local working directory.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
package utilities
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/databricks-sdk-go/service/dbfs"
|
||||
"github.com/databricks/databricks-sdk-go/workspaces"
|
||||
)
|
||||
|
||||
// move to go sdk / replace with utility function once
|
||||
// https://github.com/databricks/databricks-sdk-go/issues/57 is Done
|
||||
// Tracked in https://github.com/databricks/bricks/issues/25
|
||||
func CreateDbfsFile(ctx context.Context,
|
||||
wsc *workspaces.WorkspacesClient,
|
||||
path string,
|
||||
contents []byte,
|
||||
overwrite bool,
|
||||
) error {
|
||||
// see https://docs.databricks.com/dev-tools/api/latest/dbfs.html#add-block
|
||||
const WRITE_BYTE_CHUNK_SIZE = 1e6
|
||||
createResponse, err := wsc.Dbfs.Create(ctx,
|
||||
dbfs.CreateRequest{
|
||||
Overwrite: overwrite,
|
||||
Path: path,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
handle := createResponse.Handle
|
||||
buffer := bytes.NewBuffer(contents)
|
||||
for {
|
||||
byteChunk := buffer.Next(WRITE_BYTE_CHUNK_SIZE)
|
||||
if len(byteChunk) == 0 {
|
||||
break
|
||||
}
|
||||
b64Data := base64.StdEncoding.EncodeToString(byteChunk)
|
||||
err := wsc.Dbfs.AddBlock(ctx,
|
||||
dbfs.AddBlockRequest{
|
||||
Data: b64Data,
|
||||
Handle: handle,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot add block: %w", err)
|
||||
}
|
||||
}
|
||||
err = wsc.Dbfs.Close(ctx,
|
||||
dbfs.CloseRequest{
|
||||
Handle: handle,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot close handle: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ReadDbfsFile(ctx context.Context,
|
||||
wsc *workspaces.WorkspacesClient,
|
||||
path string,
|
||||
) (content []byte, err error) {
|
||||
// see https://docs.databricks.com/dev-tools/api/latest/dbfs.html#read
|
||||
const READ_BYTE_CHUNK_SIZE = 1e6
|
||||
fetchLoop := true
|
||||
offSet := 0
|
||||
length := int(READ_BYTE_CHUNK_SIZE)
|
||||
for fetchLoop {
|
||||
dbfsReadReponse, err := wsc.Dbfs.Read(ctx,
|
||||
dbfs.ReadRequest{
|
||||
Path: path,
|
||||
Offset: offSet,
|
||||
Length: length,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return content, fmt.Errorf("cannot read %s: %w", path, err)
|
||||
}
|
||||
if dbfsReadReponse.BytesRead == 0 || dbfsReadReponse.BytesRead < int64(length) {
|
||||
fetchLoop = false
|
||||
}
|
||||
decodedBytes, err := base64.StdEncoding.DecodeString(dbfsReadReponse.Data)
|
||||
if err != nil {
|
||||
return content, err
|
||||
}
|
||||
content = append(content, decodedBytes...)
|
||||
offSet += length
|
||||
}
|
||||
return content, err
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package utilities
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/databricks-sdk-go/service/repos"
|
||||
"github.com/databricks/databricks-sdk-go/workspaces"
|
||||
)
|
||||
|
||||
// Remove once this function is in go sdk
|
||||
// https://github.com/databricks/databricks-sdk-go/issues/58
|
||||
// Tracked in : https://github.com/databricks/bricks/issues/26
|
||||
func GetAllRepos(ctx context.Context, wsc *workspaces.WorkspacesClient, pathPrefix string) (resultRepos []repos.RepoInfo, err error) {
|
||||
nextPageToken := ""
|
||||
for {
|
||||
listReposResponse, err := wsc.Repos.List(ctx, repos.ListRequest{
|
||||
PathPrefix: pathPrefix,
|
||||
NextPageToken: nextPageToken,
|
||||
})
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
resultRepos = append(resultRepos, listReposResponse.Repos...)
|
||||
if nextPageToken == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue