diff --git a/.gitignore b/.gitignore index 1e35c543..97421b75 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ __pycache__ *.pyc .terraform -.terraform.lock.hcl \ No newline at end of file +.terraform.lock.hcl + +.vscode/launch.json \ No newline at end of file diff --git a/bundle/deployer/deployer.go b/bundle/deployer/deployer.go new file mode 100644 index 00000000..ae355c4a --- /dev/null +++ b/bundle/deployer/deployer.go @@ -0,0 +1,186 @@ +package deployer + +import ( + "context" + "fmt" + "log" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/databricks-sdk-go" + "github.com/hashicorp/terraform-exec/tfexec" +) + +type DeploymentStatus int + +const ( + // Empty plan produced on terraform plan. No changes need to be applied + NoChanges DeploymentStatus = iota + + // Deployment failed. No databricks assets were deployed + Failed + + // Deployment failed/partially succeeded. failed to update remote terraform + // state file. + // The partially deployed resources are thus untracked and in most cases + // will need to be cleaned up manually + PartialButUntracked + + // Deployment failed/partially succeeded. Remote terraform state file is + // updated with any partially deployed resources + Partial + + // Deployment succeeded however the remote terraform state was not updated. + // The deployed resources are thus untracked and in most cases will need to + // be cleaned up manually + CompleteButUntracked + + // Deployment succeeeded with remote terraform state file updated + Complete +) + +// Deployer is a struct to deploy a DAB to a databricks workspace +// +// Here's a high level description of what a deploy looks like: +// +// 1. Client compiles the bundle configuration to a terraform HCL config file +// +// 2. Client tries to acquire a lock on the remote root of the project. +// -- If FAIL: print details about current holder of the deployment lock on +// remote root and terminate deployment +// +// 3. Client reads terraform state from remote root +// +// 4. Client applies the diff in terraform config to the databricks workspace +// +// 5. Client updates terraform state file in remote root +// +// 6. Client releases the deploy lock on remote root +type Deployer struct { + localRoot string + remoteRoot string + env string + locker *Locker + wsc *databricks.WorkspaceClient +} + +func Create(ctx context.Context, env, localRoot, remoteRoot string, wsc *databricks.WorkspaceClient) (*Deployer, error) { + user, err := wsc.CurrentUser.Me(ctx) + if err != nil { + return nil, err + } + newLocker := CreateLocker(user.UserName, remoteRoot) + if err != nil { + return nil, err + } + return &Deployer{ + localRoot: localRoot, + remoteRoot: remoteRoot, + env: env, + locker: newLocker, + wsc: wsc, + }, nil +} + +func (b *Deployer) DefaultTerraformRoot() string { + return filepath.Join(b.localRoot, ".databricks/bundle", b.env) +} + +func (b *Deployer) tfStateRemotePath() string { + return path.Join(b.remoteRoot, ".bundle", "terraform.tfstate") +} + +func (b *Deployer) tfStateLocalPath() string { + return filepath.Join(b.DefaultTerraformRoot(), "terraform.tfstate") +} + +func (b *Deployer) LoadTerraformState(ctx context.Context) error { + bytes, err := b.locker.GetRawJsonFileContent(ctx, b.wsc, b.tfStateRemotePath()) + if err != nil { + // If remote tf state is absent, use local tf state + if strings.Contains(err.Error(), "File not found.") { + return nil + } else { + return err + } + } + err = os.MkdirAll(b.DefaultTerraformRoot(), os.ModeDir) + if err != nil { + return err + } + err = os.WriteFile(b.tfStateLocalPath(), bytes, os.ModePerm) + return err +} + +func (b *Deployer) SaveTerraformState(ctx context.Context) error { + bytes, err := os.ReadFile(b.tfStateLocalPath()) + if err != nil { + return err + } + return b.locker.PutFile(ctx, b.wsc, b.tfStateRemotePath(), bytes) +} + +func (d *Deployer) Lock(ctx context.Context, isForced bool) error { + return d.locker.Lock(ctx, d.wsc, isForced) +} + +func (d *Deployer) Unlock(ctx context.Context) error { + return d.locker.Unlock(ctx, d.wsc) +} + +func (d *Deployer) ApplyTerraformConfig(ctx context.Context, configPath, terraformBinaryPath string, isForced bool) (DeploymentStatus, error) { + applyErr := d.Lock(ctx, isForced) + if applyErr != nil { + return Failed, applyErr + } + defer func() { + applyErr = d.Unlock(ctx) + if applyErr != nil { + log.Printf("[ERROR] failed to unlock deployment mutex: %s", applyErr) + } + }() + + applyErr = d.LoadTerraformState(ctx) + if applyErr != nil { + log.Printf("[DEBUG] failed to load terraform state from workspace: %s", applyErr) + return Failed, applyErr + } + + tf, applyErr := tfexec.NewTerraform(configPath, terraformBinaryPath) + if applyErr != nil { + log.Printf("[DEBUG] failed to construct terraform object: %s", applyErr) + return Failed, applyErr + } + + isPlanNotEmpty, applyErr := tf.Plan(ctx) + if applyErr != nil { + log.Printf("[DEBUG] failed to compute terraform plan: %s", applyErr) + return Failed, applyErr + } + + if !isPlanNotEmpty { + log.Printf("[DEBUG] terraform plan returned a empty diff") + return NoChanges, nil + } + + applyErr = tf.Apply(ctx) + // upload state even if apply fails to handle partial deployments + saveStateErr := d.SaveTerraformState(ctx) + + if applyErr != nil && saveStateErr != nil { + log.Printf("[ERROR] terraform apply failed: %s", applyErr) + log.Printf("[ERROR] failed to upload terraform state after partial terraform apply: %s", saveStateErr) + return PartialButUntracked, fmt.Errorf("deploymented failed: %s", applyErr) + } + if applyErr != nil { + log.Printf("[ERROR] terraform apply failed: %s", applyErr) + return Partial, fmt.Errorf("deploymented failed: %s", applyErr) + } + if saveStateErr != nil { + log.Printf("[ERROR] failed to upload terraform state after completing terraform apply: %s", saveStateErr) + return CompleteButUntracked, fmt.Errorf("failed to upload terraform state file: %s", saveStateErr) + } + return Complete, nil +} diff --git a/bundle/deployer/locker.go b/bundle/deployer/locker.go new file mode 100644 index 00000000..5af976bb --- /dev/null +++ b/bundle/deployer/locker.go @@ -0,0 +1,190 @@ +package deployer + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "path" + "strings" + "time" + + "github.com/databricks/bricks/utilities" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/google/uuid" +) + +// Locker object enables exclusive access to TargetDir's scope for a client. This +// enables multiple clients to deploy to the same scope (ie TargetDir) in an atomic +// manner +// +// Here are some of the details of the locking protocol used here: +// +// 1. Potentially multiple clients race to create a deploy.lock file in +// TargetDir/.bundle directory with unique ID. The deploy.lock file +// is a json file containing the State from the locker +// +// 2. Clients read the remote deploy.lock file and if it's ID matches, the client +// assumes it has the lock on TargetDir. The client is now free to read/write code +// asserts and deploy databricks assets scoped under TargetDir +// +// 3. To sidestep clients failing to relinquish a lock during a failed deploy attempt +// we allow clients to forcefully acquire a lock on TargetDir. However forcefully acquired +// locks come with the following caveats: +// +// a. a forcefully acquired lock does not guarentee exclusive access to +// TargetDir's scope +// b. forcefully acquiring a lock(s) on TargetDir can break the assumption +// of exclusive access that other clients with non forcefully acquired +// locks might have +type Locker struct { + // scope of the locker + TargetDir string + // Active == true implies exclusive access to TargetDir for the client. + // This implication break down if locks are forcefully acquired by a user + Active bool + // if locker is active, this information about the locker is uploaded onto + // the workspace so as to let other clients details about the active locker + State *LockState +} + +type LockState struct { + // unique identifier for the locker + ID uuid.UUID + // last timestamp when locker was active + AcquisitionTime time.Time + // Only relevant for active lockers + // IsForced == true implies the lock was acquired forcefully + IsForced bool + // creator of this locker + User string +} + +// don't need to hold lock on TargetDir to read locker state +func GetActiveLockState(ctx context.Context, wsc *databricks.WorkspaceClient, path string) (*LockState, error) { + bytes, err := utilities.GetRawJsonFileContent(ctx, wsc, path) + if err != nil { + return nil, err + } + remoteLock := LockState{} + err = json.Unmarshal(bytes, &remoteLock) + if err != nil { + return nil, err + } + return &remoteLock, nil +} + +// asserts whether lock is held by locker. Returns descriptive error with current +// holder details if locker does not hold the lock +func (locker *Locker) assertLockHeld(ctx context.Context, wsc *databricks.WorkspaceClient) error { + activeLockState, err := GetActiveLockState(ctx, wsc, locker.RemotePath()) + if err != nil && strings.Contains(err.Error(), "File not found.") { + return fmt.Errorf("no active lock on target dir: %s", err) + } + if err != nil { + return err + } + if activeLockState.ID != locker.State.ID && !activeLockState.IsForced { + return fmt.Errorf("deploy lock acquired by %s at %v. Use --force to override", activeLockState.User, activeLockState.AcquisitionTime) + } + if activeLockState.ID != locker.State.ID && activeLockState.IsForced { + return fmt.Errorf("deploy lock force acquired by %s at %v. Use --force to override", activeLockState.User, activeLockState.AcquisitionTime) + } + return nil +} + +// idempotent function since overwrite is set to true +func (locker *Locker) PutFile(ctx context.Context, wsc *databricks.WorkspaceClient, pathToFile string, content []byte) error { + if !locker.Active { + return fmt.Errorf("failed to put file. deploy lock not held") + } + apiClient, err := client.New(wsc.Config) + if err != nil { + return err + } + apiPath := fmt.Sprintf( + "/api/2.0/workspace-files/import-file/%s?overwrite=true", + strings.TrimLeft(pathToFile, "/")) + + err = apiClient.Do(ctx, http.MethodPost, apiPath, bytes.NewReader(content), nil) + if err != nil { + // retry after creating parent dirs + err = wsc.Workspace.MkdirsByPath(ctx, path.Dir(pathToFile)) + if err != nil { + return fmt.Errorf("could not mkdir to put file: %s", err) + } + err = apiClient.Do(ctx, http.MethodPost, apiPath, bytes.NewReader(content), nil) + } + return err +} + +func (locker *Locker) GetRawJsonFileContent(ctx context.Context, wsc *databricks.WorkspaceClient, path string) ([]byte, error) { + if !locker.Active { + return nil, fmt.Errorf("failed to get file. deploy lock not held") + } + return utilities.GetRawJsonFileContent(ctx, wsc, path) +} + +func (locker *Locker) Lock(ctx context.Context, wsc *databricks.WorkspaceClient, isForced bool) error { + newLockerState := LockState{ + ID: locker.State.ID, + AcquisitionTime: time.Now(), + IsForced: isForced, + User: locker.State.User, + } + bytes, err := json.Marshal(newLockerState) + if err != nil { + return err + } + err = utilities.WriteFile(ctx, wsc, locker.RemotePath(), bytes, isForced) + if err != nil && !strings.Contains(err.Error(), fmt.Sprintf("%s already exists", locker.RemotePath())) { + return err + } + err = locker.assertLockHeld(ctx, wsc) + if err != nil { + return err + } + + locker.State = &newLockerState + locker.Active = true + return nil +} + +func (locker *Locker) Unlock(ctx context.Context, wsc *databricks.WorkspaceClient) error { + if !locker.Active { + return fmt.Errorf("unlock called when lock is not held") + } + err := locker.assertLockHeld(ctx, wsc) + if err != nil { + return fmt.Errorf("unlock called when lock is not held: %s", err) + } + err = wsc.Workspace.Delete(ctx, + workspace.Delete{ + Path: locker.RemotePath(), + Recursive: false, + }, + ) + if err != nil { + return err + } + locker.Active = false + return nil +} + +func (locker *Locker) RemotePath() string { + return path.Join(locker.TargetDir, ".bundle/deploy.lock") +} + +func CreateLocker(user string, targetDir string) *Locker { + return &Locker{ + TargetDir: targetDir, + Active: false, + State: &LockState{ + ID: uuid.New(), + User: user, + }, + } +} diff --git a/cmd/bundle/debug/debug.go b/cmd/bundle/debug/debug.go index 9750b7fb..1786f71f 100644 --- a/cmd/bundle/debug/debug.go +++ b/cmd/bundle/debug/debug.go @@ -10,6 +10,10 @@ var debugCmd = &cobra.Command{ Use: "debug", } +func AddCommand(cmd *cobra.Command) { + debugCmd.AddCommand(cmd) +} + func init() { parent.AddCommand(debugCmd) } diff --git a/cmd/bundle/debug/deploy/deploy.go b/cmd/bundle/debug/deploy/deploy.go new file mode 100644 index 00000000..91f0f865 --- /dev/null +++ b/cmd/bundle/debug/deploy/deploy.go @@ -0,0 +1,101 @@ +package deploy + +import ( + "log" + "os" + "path/filepath" + + "github.com/databricks/bricks/bundle/deployer" + "github.com/databricks/bricks/cmd/bundle/debug" + "github.com/databricks/databricks-sdk-go" + "github.com/hashicorp/go-version" + "github.com/hashicorp/hc-install/product" + "github.com/hashicorp/hc-install/releases" + "github.com/spf13/cobra" +) + +// TODO: will add integration test once terraform binary is bundled with bricks +var deployTerraformCmd = &cobra.Command{ + Use: "deploy", + Short: "deploys resources defined in a terraform config to a Databricks workspace", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + // default to cwd + if *localRoot == "" { + cwd, err := os.Getwd() + if err != nil { + return err + } + *localRoot = cwd + } + + if *terraformBinaryPath == "" { + installer := releases.ExactVersion{ + Product: product.Terraform, + Version: version.Must(version.NewVersion("1.2.4")), + } + execPath, err := installer.Install(ctx) + if err != nil { + log.Printf("[ERROR] error installing Terraform: %s", err) + } + *terraformBinaryPath = execPath + defer installer.Remove(ctx) + } + + // TODO: load bundle and get the workspace client from there once bundles + // are stable + wsc, err := databricks.NewWorkspaceClient() + if err != nil { + return err + } + + d, err := deployer.Create(ctx, *env, *localRoot, *remoteRoot, wsc) + if err != nil { + return err + } + + if *terraformHcl == "" { + *terraformHcl = filepath.Join(d.DefaultTerraformRoot()) + } + + status, err := d.ApplyTerraformConfig(ctx, *terraformHcl, *terraformBinaryPath, *isForced) + switch status { + case deployer.Failed: + log.Printf("[ERROR] failed to initiate deployment") + case deployer.NoChanges: + log.Printf("[INFO] no changes detected") + case deployer.Partial: + log.Printf("[ERROR] started deployment, but failed to complete") + case deployer.PartialButUntracked: + log.Printf("[ERROR] started deployment, but failed to complete. Any partially deployed resources in this run are untracked in the databricks workspace and might not be cleaned up on future deployments") + case deployer.CompleteButUntracked: + log.Printf("[ERROR] deployment complete. Failed to track deployed resources. Any deployed resources in this run are untracked in the databricks workspace and might not be cleaned up on future deployments") + case deployer.Complete: + log.Printf("[INFO] deployment complete") + } + return err + }, +} + +var remoteRoot *string +var localRoot *string +var env *string +var isForced *bool + +var terraformHcl *string + +// TODO: remove this arguement once we package a terraform binary with the bricks cli +var terraformBinaryPath *string + +func init() { + remoteRoot = deployTerraformCmd.Flags().String("remote-root", "", "workspace root of the project eg: /Repos/me@example.com/test-repo") + localRoot = deployTerraformCmd.Flags().String("local-root", "", "path to the root directory of the DAB project. default: current working dir") + terraformBinaryPath = deployTerraformCmd.Flags().String("terraform-cli-binary", "", "path to a terraform CLI executable binary") + env = deployTerraformCmd.Flags().String("env", "development", "environment to deploy on. default: development") + isForced = deployTerraformCmd.Flags().Bool("force", false, "force deploy your DAB to the workspace. default: false") + terraformHcl = deployTerraformCmd.Flags().String("terraform-hcl", "", "path to the terraform config file from project root") + + deployTerraformCmd.MarkFlagRequired("remote-root") + debug.AddCommand(deployTerraformCmd) +} diff --git a/go.mod b/go.mod index 5a41cbf8..daec6eba 100644 --- a/go.mod +++ b/go.mod @@ -18,10 +18,21 @@ require ( ) require ( + github.com/google/uuid v1.3.0 + github.com/hashicorp/go-version v1.6.0 + github.com/hashicorp/hc-install v0.4.0 + github.com/hashicorp/terraform-exec v0.17.3 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 golang.org/x/sync v0.1.0 ) +require ( + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/terraform-json v0.14.0 // indirect + github.com/zclconf/go-cty v1.11.0 // indirect + golang.org/x/crypto v0.1.0 // indirect +) + require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect @@ -33,11 +44,10 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/imdario/mergo v0.3.13 github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect + golang.org/x/net v0.1.0 // indirect golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect @@ -47,7 +57,6 @@ require ( google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect google.golang.org/grpc v1.50.1 // indirect google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 260960b9..00e2a1f4 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,17 @@ cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -22,19 +33,33 @@ github.com/databricks/databricks-sdk-go v0.1.0/go.mod h1:n4we8UoagEFH0VQuxkhQ3WH github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -53,54 +78,112 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hc-install v0.4.0 h1:cZkRFr1WVa0Ty6x5fTvL1TuO1flul231rWkGH92oYYk= +github.com/hashicorp/hc-install v0.4.0/go.mod h1:5d155H8EC5ewegao9A4PUTMNPZaq+TbOzkJJZ4vrXeI= +github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU= +github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI= +github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s= +github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= +github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0= +github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= @@ -110,14 +193,18 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= @@ -127,16 +214,27 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= @@ -151,6 +249,7 @@ google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -179,10 +278,18 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/locker_test.go b/internal/locker_test.go new file mode 100644 index 00000000..bd7c6a27 --- /dev/null +++ b/internal/locker_test.go @@ -0,0 +1,168 @@ +package internal + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "os/exec" + "path" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/databricks/bricks/bundle/deployer" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/repos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TODO: create a utility function to create an empty test repo for tests and refactor sync_test integration test + +const EmptyRepoUrl = "https://github.com/shreyas-goenka/empty-repo.git" + +func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databricks.WorkspaceClient) string { + ctx := context.TODO() + me, err := wsc.CurrentUser.Me(ctx) + assert.NoError(t, err) + + remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix)) + repoInfo, err := wsc.Repos.Create(ctx, repos.CreateRepo{ + Path: remoteProjectRoot, + Url: EmptyRepoUrl, + Provider: "gitHub", + }) + assert.NoError(t, err) + t.Cleanup(func() { + err := wsc.Repos.DeleteByRepoId(ctx, repoInfo.Id) + assert.NoError(t, err) + }) + + return remoteProjectRoot +} + +func createLocalTestProject(t *testing.T) string { + tempDir := t.TempDir() + + cmd := exec.Command("git", "clone", EmptyRepoUrl) + cmd.Dir = tempDir + err := cmd.Run() + assert.NoError(t, err) + + localProjectRoot := filepath.Join(tempDir, "empty-repo") + err = os.Chdir(localProjectRoot) + assert.NoError(t, err) + return localProjectRoot +} + +func TestAccLock(t *testing.T) { + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + ctx := context.TODO() + wsc := databricks.Must(databricks.NewWorkspaceClient()) + createLocalTestProject(t) + remoteProjectRoot := createRemoteTestProject(t, "lock-acc-", wsc) + + // 50 lockers try to acquire a lock at the same time + numConcurrentLocks := 50 + + var err error + lockerErrs := make([]error, numConcurrentLocks) + lockers := make([]*deployer.Locker, numConcurrentLocks) + + for i := 0; i < numConcurrentLocks; i++ { + lockers[i] = deployer.CreateLocker("humpty.dumpty@databricks.com", remoteProjectRoot) + } + + var wg sync.WaitGroup + for i := 0; i < numConcurrentLocks; i++ { + wg.Add(1) + currentIndex := i + go func() { + defer wg.Done() + time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) + lockerErrs[currentIndex] = lockers[currentIndex].Lock(ctx, wsc, false) + }() + } + wg.Wait() + + countActive := 0 + indexOfActiveLocker := 0 + indexOfAnInactiveLocker := -1 + for i := 0; i < numConcurrentLocks; i++ { + if lockers[i].Active { + countActive += 1 + assert.NoError(t, lockerErrs[i]) + indexOfActiveLocker = i + } else { + if indexOfAnInactiveLocker == -1 { + indexOfAnInactiveLocker = i + } + assert.ErrorContains(t, lockerErrs[i], "lock acquired by") + assert.ErrorContains(t, lockerErrs[i], "Use --force to override") + } + } + assert.Equal(t, 1, countActive, "Exactly one locker should successfull acquire the lock") + + // test remote lock matches active lock + remoteLocker, err := deployer.GetActiveLockState(ctx, wsc, lockers[indexOfActiveLocker].RemotePath()) + require.NoError(t, err) + assert.Equal(t, remoteLocker.ID, lockers[indexOfActiveLocker].State.ID, "remote locker id does not match active locker") + assert.True(t, remoteLocker.AcquisitionTime.Equal(lockers[indexOfActiveLocker].State.AcquisitionTime), "remote locker acquisition time does not match active locker") + + // test all other locks (inactive ones) do not match the remote lock and Unlock fails + for i := 0; i < numConcurrentLocks; i++ { + if i == indexOfActiveLocker { + continue + } + assert.NotEqual(t, remoteLocker.ID, lockers[i].State.ID) + err := lockers[i].Unlock(ctx, wsc) + assert.ErrorContains(t, err, "unlock called when lock is not held") + } + + // test inactive locks fail to write a file + for i := 0; i < numConcurrentLocks; i++ { + if i == indexOfActiveLocker { + continue + } + err := lockers[i].PutFile(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"), []byte(`'{"surname":"Khan", "name":"Shah Rukh"}`)) + assert.ErrorContains(t, err, "failed to put file. deploy lock not held") + } + + // active locker file write succeeds + err = lockers[indexOfActiveLocker].PutFile(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"), []byte(`{"surname":"Khan", "name":"Shah Rukh"}`)) + assert.NoError(t, err) + + // active locker file read succeeds with expected results + bytes, err := lockers[indexOfActiveLocker].GetRawJsonFileContent(ctx, wsc, path.Join(remoteProjectRoot, "foo.json")) + var res map[string]string + json.Unmarshal(bytes, &res) + assert.NoError(t, err) + assert.Equal(t, "Khan", res["surname"]) + assert.Equal(t, "Shah Rukh", res["name"]) + + // inactive locker file reads fail + for i := 0; i < numConcurrentLocks; i++ { + if i == indexOfActiveLocker { + continue + } + _, err = lockers[i].GetRawJsonFileContent(ctx, wsc, path.Join(remoteProjectRoot, "foo.json")) + assert.ErrorContains(t, err, "failed to get file. deploy lock not held") + } + + // Unlock active lock and check it becomes inactive + err = lockers[indexOfActiveLocker].Unlock(ctx, wsc) + assert.NoError(t, err) + remoteLocker, err = deployer.GetActiveLockState(ctx, wsc, lockers[indexOfActiveLocker].RemotePath()) + assert.ErrorContains(t, err, "File not found.", "remote lock file not deleted on unlock") + assert.Nil(t, remoteLocker) + assert.False(t, lockers[indexOfActiveLocker].Active) + + // A locker that failed to acquire the lock should now be able to acquire it + assert.False(t, lockers[indexOfAnInactiveLocker].Active) + err = lockers[indexOfAnInactiveLocker].Lock(ctx, wsc, false) + assert.NoError(t, err) + assert.True(t, lockers[indexOfAnInactiveLocker].Active) +} diff --git a/main.go b/main.go index fc927e75..d089e211 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( _ "github.com/databricks/bricks/cmd/api" _ "github.com/databricks/bricks/cmd/bundle" _ "github.com/databricks/bricks/cmd/bundle/debug" + _ "github.com/databricks/bricks/cmd/bundle/debug/deploy" _ "github.com/databricks/bricks/cmd/configure" _ "github.com/databricks/bricks/cmd/fs" _ "github.com/databricks/bricks/cmd/init" diff --git a/utilities/workspace_files.go b/utilities/workspace_files.go new file mode 100644 index 00000000..73e96c7a --- /dev/null +++ b/utilities/workspace_files.go @@ -0,0 +1,59 @@ +package utilities + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "path" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" +) + +// NOTE: This API is only available for files in /Repos if a workspace has repos +// in workspace enabled and files in workspace not enabled +// +// Get the file contents of a json file in workspace +// TODO(Nov 2022): add method in go sdk to get the raw bytes from response of an API +// +// TODO(Nov 2022): talk to eng-files team about what the response structure would look like. +// This function would have to be modfified probably in the future once this +// API goes to public preview +func GetRawJsonFileContent(ctx context.Context, wsc *databricks.WorkspaceClient, path string) ([]byte, error) { + apiClient, err := client.New(wsc.Config) + if err != nil { + return nil, err + } + exportApiPath := fmt.Sprintf( + "/api/2.0/workspace-files/%s", + strings.TrimLeft(path, "/")) + + var res json.RawMessage + + err = apiClient.Do(ctx, http.MethodGet, exportApiPath, nil, &res) + if err != nil { + return nil, fmt.Errorf("failed to fetch file %s: %s", path, err) + } + return res, nil +} + +func WriteFile(ctx context.Context, wsc *databricks.WorkspaceClient, pathToFile string, content []byte, overwrite bool) error { + apiClient, err := client.New(wsc.Config) + if err != nil { + return err + } + err = wsc.Workspace.MkdirsByPath(ctx, path.Dir(pathToFile)) + if err != nil { + return fmt.Errorf("could not mkdir to post file: %s", err) + } + + importApiPath := fmt.Sprintf( + "/api/2.0/workspace-files/import-file/%s?overwrite=%s", + strings.TrimLeft(pathToFile, "/"), strconv.FormatBool(overwrite)) + + return apiClient.Do(ctx, http.MethodPost, importApiPath, bytes.NewReader(content), nil) +}