Implement Terraform state synchronization and deploy (#98)

https://user-images.githubusercontent.com/88374338/203669797-abebf99e-8fa6-4d6e-b57a-abd172d8020d.mov

.gitignore

@@ -25,4 +25,6 @@ __pycache__
 *.pyc
 
 .terraform
-.terraform.lock.hcl
+.terraform.lock.hcl
+
+.vscode/launch.json

bundle/deployer/deployer.go

@@ -0,0 +1,186 @@
+package deployer
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/hashicorp/terraform-exec/tfexec"
+)
+
+type DeploymentStatus int
+
+const (
+	// Empty plan produced on terraform plan. No changes need to be applied
+	NoChanges DeploymentStatus = iota
+
+	// Deployment failed. No databricks assets were deployed
+	Failed
+
+	// Deployment failed/partially succeeded. failed to update remote terraform
+	// state file.
+	// The partially deployed resources are thus untracked and in most cases
+	// will need to be cleaned up manually
+	PartialButUntracked
+
+	// Deployment failed/partially succeeded. Remote terraform state file is
+	// updated with any partially deployed resources
+	Partial
+
+	// Deployment succeeded however the remote terraform state was not updated.
+	// The deployed resources are thus untracked and in most cases will need to
+	// be cleaned up manually
+	CompleteButUntracked
+
+	// Deployment succeeeded with remote terraform state file updated
+	Complete
+)
+
+// Deployer is a struct to deploy a DAB to a databricks workspace
+//
+// Here's a high level description of what a deploy looks like:
+//
+// 1. Client compiles the bundle configuration to a terraform HCL config file
+//
+// 2. Client tries to acquire a lock on the remote root of the project.
+//  	-- If FAIL: print details about current holder of the deployment lock on
+//					remote root and terminate deployment
+//
+// 3. Client reads terraform state from remote root
+//
+// 4. Client applies the diff in terraform config to the databricks workspace
+//
+// 5. Client updates terraform state file in remote root
+//
+// 6. Client releases the deploy lock on remote root
+type Deployer struct {
+	localRoot  string
+	remoteRoot string
+	env        string
+	locker     *Locker
+	wsc        *databricks.WorkspaceClient
+}
+
+func Create(ctx context.Context, env, localRoot, remoteRoot string, wsc *databricks.WorkspaceClient) (*Deployer, error) {
+	user, err := wsc.CurrentUser.Me(ctx)
+	if err != nil {
+		return nil, err
+	}
+	newLocker := CreateLocker(user.UserName, remoteRoot)
+	if err != nil {
+		return nil, err
+	}
+	return &Deployer{
+		localRoot:  localRoot,
+		remoteRoot: remoteRoot,
+		env:        env,
+		locker:     newLocker,
+		wsc:        wsc,
+	}, nil
+}
+
+func (b *Deployer) DefaultTerraformRoot() string {
+	return filepath.Join(b.localRoot, ".databricks/bundle", b.env)
+}
+
+func (b *Deployer) tfStateRemotePath() string {
+	return path.Join(b.remoteRoot, ".bundle", "terraform.tfstate")
+}
+
+func (b *Deployer) tfStateLocalPath() string {
+	return filepath.Join(b.DefaultTerraformRoot(), "terraform.tfstate")
+}
+
+func (b *Deployer) LoadTerraformState(ctx context.Context) error {
+	bytes, err := b.locker.GetRawJsonFileContent(ctx, b.wsc, b.tfStateRemotePath())
+	if err != nil {
+		// If remote tf state is absent, use local tf state
+		if strings.Contains(err.Error(), "File not found.") {
+			return nil
+		} else {
+			return err
+		}
+	}
+	err = os.MkdirAll(b.DefaultTerraformRoot(), os.ModeDir)
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(b.tfStateLocalPath(), bytes, os.ModePerm)
+	return err
+}
+
+func (b *Deployer) SaveTerraformState(ctx context.Context) error {
+	bytes, err := os.ReadFile(b.tfStateLocalPath())
+	if err != nil {
+		return err
+	}
+	return b.locker.PutFile(ctx, b.wsc, b.tfStateRemotePath(), bytes)
+}
+
+func (d *Deployer) Lock(ctx context.Context, isForced bool) error {
+	return d.locker.Lock(ctx, d.wsc, isForced)
+}
+
+func (d *Deployer) Unlock(ctx context.Context) error {
+	return d.locker.Unlock(ctx, d.wsc)
+}
+
+func (d *Deployer) ApplyTerraformConfig(ctx context.Context, configPath, terraformBinaryPath string, isForced bool) (DeploymentStatus, error) {
+	applyErr := d.Lock(ctx, isForced)
+	if applyErr != nil {
+		return Failed, applyErr
+	}
+	defer func() {
+		applyErr = d.Unlock(ctx)
+		if applyErr != nil {
+			log.Printf("[ERROR] failed to unlock deployment mutex: %s", applyErr)
+		}
+	}()
+
+	applyErr = d.LoadTerraformState(ctx)
+	if applyErr != nil {
+		log.Printf("[DEBUG] failed to load terraform state from workspace: %s", applyErr)
+		return Failed, applyErr
+	}
+
+	tf, applyErr := tfexec.NewTerraform(configPath, terraformBinaryPath)
+	if applyErr != nil {
+		log.Printf("[DEBUG] failed to construct terraform object: %s", applyErr)
+		return Failed, applyErr
+	}
+
+	isPlanNotEmpty, applyErr := tf.Plan(ctx)
+	if applyErr != nil {
+		log.Printf("[DEBUG] failed to compute terraform plan: %s", applyErr)
+		return Failed, applyErr
+	}
+
+	if !isPlanNotEmpty {
+		log.Printf("[DEBUG] terraform plan returned a empty diff")
+		return NoChanges, nil
+	}
+
+	applyErr = tf.Apply(ctx)
+	// upload state even if apply fails to handle partial deployments
+	saveStateErr := d.SaveTerraformState(ctx)
+
+	if applyErr != nil && saveStateErr != nil {
+		log.Printf("[ERROR] terraform apply failed: %s", applyErr)
+		log.Printf("[ERROR] failed to upload terraform state after partial terraform apply: %s", saveStateErr)
+		return PartialButUntracked, fmt.Errorf("deploymented failed: %s", applyErr)
+	}
+	if applyErr != nil {
+		log.Printf("[ERROR] terraform apply failed: %s", applyErr)
+		return Partial, fmt.Errorf("deploymented failed: %s", applyErr)
+	}
+	if saveStateErr != nil {
+		log.Printf("[ERROR] failed to upload terraform state after completing terraform apply: %s", saveStateErr)
+		return CompleteButUntracked, fmt.Errorf("failed to upload terraform state file: %s", saveStateErr)
+	}
+	return Complete, nil
+}

bundle/deployer/locker.go

@@ -0,0 +1,190 @@
+package deployer
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/databricks/bricks/utilities"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/client"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/google/uuid"
+)
+
+// Locker object enables exclusive access to TargetDir's scope for a client. This
+// enables multiple clients to deploy to the same scope (ie TargetDir) in an atomic
+// manner
+//
+// Here are some of the details of the locking protocol used here:
+//
+// 1. Potentially multiple clients race to create a deploy.lock file in
+//    TargetDir/.bundle directory with unique ID. The deploy.lock file
+//    is a json file containing the State from the locker
+//
+// 2. Clients read the remote deploy.lock file and if it's ID matches, the client
+//    assumes it has the lock on TargetDir. The client is now free to read/write code
+//    asserts and deploy databricks assets scoped under TargetDir
+//
+// 3. To sidestep clients failing to relinquish a lock during a failed deploy attempt
+//    we allow clients to forcefully acquire a lock on TargetDir. However forcefully acquired
+//    locks come with the following caveats:
+//
+//       a.  a forcefully acquired lock does not guarentee exclusive access to
+//           TargetDir's scope
+//       b.  forcefully acquiring a lock(s) on TargetDir can break the assumption
+//           of exclusive access that other clients with non forcefully acquired
+//           locks might have
+type Locker struct {
+	// scope of the locker
+	TargetDir string
+	// Active == true implies exclusive access to TargetDir for the client.
+	// This implication break down if locks are forcefully acquired by a user
+	Active bool
+	// if locker is active, this information about the locker is uploaded onto
+	// the workspace so as to let other clients details about the active locker
+	State *LockState
+}
+
+type LockState struct {
+	// unique identifier for the locker
+	ID uuid.UUID
+	// last timestamp when locker was active
+	AcquisitionTime time.Time
+	// Only relevant for active lockers
+	// IsForced == true implies the lock was acquired forcefully
+	IsForced bool
+	// creator of this locker
+	User string
+}
+
+// don't need to hold lock on TargetDir to read locker state
+func GetActiveLockState(ctx context.Context, wsc *databricks.WorkspaceClient, path string) (*LockState, error) {
+	bytes, err := utilities.GetRawJsonFileContent(ctx, wsc, path)
+	if err != nil {
+		return nil, err
+	}
+	remoteLock := LockState{}
+	err = json.Unmarshal(bytes, &remoteLock)
+	if err != nil {
+		return nil, err
+	}
+	return &remoteLock, nil
+}
+
+// asserts whether lock is held by locker. Returns descriptive error with current
+// holder details if locker does not hold the lock
+func (locker *Locker) assertLockHeld(ctx context.Context, wsc *databricks.WorkspaceClient) error {
+	activeLockState, err := GetActiveLockState(ctx, wsc, locker.RemotePath())
+	if err != nil && strings.Contains(err.Error(), "File not found.") {
+		return fmt.Errorf("no active lock on target dir: %s", err)
+	}
+	if err != nil {
+		return err
+	}
+	if activeLockState.ID != locker.State.ID && !activeLockState.IsForced {
+		return fmt.Errorf("deploy lock acquired by %s at %v. Use --force to override", activeLockState.User, activeLockState.AcquisitionTime)
+	}
+	if activeLockState.ID != locker.State.ID && activeLockState.IsForced {
+		return fmt.Errorf("deploy lock force acquired by %s at %v. Use --force to override", activeLockState.User, activeLockState.AcquisitionTime)
+	}
+	return nil
+}
+
+// idempotent function since overwrite is set to true
+func (locker *Locker) PutFile(ctx context.Context, wsc *databricks.WorkspaceClient, pathToFile string, content []byte) error {
+	if !locker.Active {
+		return fmt.Errorf("failed to put file. deploy lock not held")
+	}
+	apiClient, err := client.New(wsc.Config)
+	if err != nil {
+		return err
+	}
+	apiPath := fmt.Sprintf(
+		"/api/2.0/workspace-files/import-file/%s?overwrite=true",
+		strings.TrimLeft(pathToFile, "/"))
+
+	err = apiClient.Do(ctx, http.MethodPost, apiPath, bytes.NewReader(content), nil)
+	if err != nil {
+		// retry after creating parent dirs
+		err = wsc.Workspace.MkdirsByPath(ctx, path.Dir(pathToFile))
+		if err != nil {
+			return fmt.Errorf("could not mkdir to put file: %s", err)
+		}
+		err = apiClient.Do(ctx, http.MethodPost, apiPath, bytes.NewReader(content), nil)
+	}
+	return err
+}
+
+func (locker *Locker) GetRawJsonFileContent(ctx context.Context, wsc *databricks.WorkspaceClient, path string) ([]byte, error) {
+	if !locker.Active {
+		return nil, fmt.Errorf("failed to get file. deploy lock not held")
+	}
+	return utilities.GetRawJsonFileContent(ctx, wsc, path)
+}
+
+func (locker *Locker) Lock(ctx context.Context, wsc *databricks.WorkspaceClient, isForced bool) error {
+	newLockerState := LockState{
+		ID:              locker.State.ID,
+		AcquisitionTime: time.Now(),
+		IsForced:        isForced,
+		User:            locker.State.User,
+	}
+	bytes, err := json.Marshal(newLockerState)
+	if err != nil {
+		return err
+	}
+	err = utilities.WriteFile(ctx, wsc, locker.RemotePath(), bytes, isForced)
+	if err != nil && !strings.Contains(err.Error(), fmt.Sprintf("%s already exists", locker.RemotePath())) {
+		return err
+	}
+	err = locker.assertLockHeld(ctx, wsc)
+	if err != nil {
+		return err
+	}
+
+	locker.State = &newLockerState
+	locker.Active = true
+	return nil
+}
+
+func (locker *Locker) Unlock(ctx context.Context, wsc *databricks.WorkspaceClient) error {
+	if !locker.Active {
+		return fmt.Errorf("unlock called when lock is not held")
+	}
+	err := locker.assertLockHeld(ctx, wsc)
+	if err != nil {
+		return fmt.Errorf("unlock called when lock is not held: %s", err)
+	}
+	err = wsc.Workspace.Delete(ctx,
+		workspace.Delete{
+			Path:      locker.RemotePath(),
+			Recursive: false,
+		},
+	)
+	if err != nil {
+		return err
+	}
+	locker.Active = false
+	return nil
+}
+
+func (locker *Locker) RemotePath() string {
+	return path.Join(locker.TargetDir, ".bundle/deploy.lock")
+}
+
+func CreateLocker(user string, targetDir string) *Locker {
+	return &Locker{
+		TargetDir: targetDir,
+		Active:    false,
+		State: &LockState{
+			ID:   uuid.New(),
+			User: user,
+		},
+	}
+}

cmd/bundle/debug/debug.go

@@ -10,6 +10,10 @@ var debugCmd = &cobra.Command{
 	Use: "debug",
 }
 
+func AddCommand(cmd *cobra.Command) {
+	debugCmd.AddCommand(cmd)
+}
+
 func init() {
 	parent.AddCommand(debugCmd)
 }

cmd/bundle/debug/deploy/deploy.go

@@ -0,0 +1,101 @@
+package deploy
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/databricks/bricks/bundle/deployer"
+	"github.com/databricks/bricks/cmd/bundle/debug"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/hc-install/product"
+	"github.com/hashicorp/hc-install/releases"
+	"github.com/spf13/cobra"
+)
+
+// TODO: will add integration test once terraform binary is bundled with bricks
+var deployTerraformCmd = &cobra.Command{
+	Use:   "deploy",
+	Short: "deploys resources defined in a terraform config to a Databricks workspace",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Context()
+
+		// default to cwd
+		if *localRoot == "" {
+			cwd, err := os.Getwd()
+			if err != nil {
+				return err
+			}
+			*localRoot = cwd
+		}
+
+		if *terraformBinaryPath == "" {
+			installer := releases.ExactVersion{
+				Product: product.Terraform,
+				Version: version.Must(version.NewVersion("1.2.4")),
+			}
+			execPath, err := installer.Install(ctx)
+			if err != nil {
+				log.Printf("[ERROR] error installing Terraform: %s", err)
+			}
+			*terraformBinaryPath = execPath
+			defer installer.Remove(ctx)
+		}
+
+		// TODO: load bundle and get the workspace client from there once bundles
+		// are stable
+		wsc, err := databricks.NewWorkspaceClient()
+		if err != nil {
+			return err
+		}
+
+		d, err := deployer.Create(ctx, *env, *localRoot, *remoteRoot, wsc)
+		if err != nil {
+			return err
+		}
+
+		if *terraformHcl == "" {
+			*terraformHcl = filepath.Join(d.DefaultTerraformRoot())
+		}
+
+		status, err := d.ApplyTerraformConfig(ctx, *terraformHcl, *terraformBinaryPath, *isForced)
+		switch status {
+		case deployer.Failed:
+			log.Printf("[ERROR] failed to initiate deployment")
+		case deployer.NoChanges:
+			log.Printf("[INFO] no changes detected")
+		case deployer.Partial:
+			log.Printf("[ERROR] started deployment, but failed to complete")
+		case deployer.PartialButUntracked:
+			log.Printf("[ERROR] started deployment, but failed to complete. Any partially deployed resources in this run are untracked in the databricks workspace and might not be cleaned up on future deployments")
+		case deployer.CompleteButUntracked:
+			log.Printf("[ERROR] deployment complete. Failed to track deployed resources. Any deployed resources in this run are untracked in the databricks workspace and might not be cleaned up on future deployments")
+		case deployer.Complete:
+			log.Printf("[INFO] deployment complete")
+		}
+		return err
+	},
+}
+
+var remoteRoot *string
+var localRoot *string
+var env *string
+var isForced *bool
+
+var terraformHcl *string
+
+// TODO: remove this arguement once we package a terraform binary with the bricks cli
+var terraformBinaryPath *string
+
+func init() {
+	remoteRoot = deployTerraformCmd.Flags().String("remote-root", "", "workspace root of the project eg: /Repos/me@example.com/test-repo")
+	localRoot = deployTerraformCmd.Flags().String("local-root", "", "path to the root directory of the DAB project. default: current working dir")
+	terraformBinaryPath = deployTerraformCmd.Flags().String("terraform-cli-binary", "", "path to a terraform CLI executable binary")
+	env = deployTerraformCmd.Flags().String("env", "development", "environment to deploy on. default: development")
+	isForced = deployTerraformCmd.Flags().Bool("force", false, "force deploy your DAB to the workspace. default: false")
+	terraformHcl = deployTerraformCmd.Flags().String("terraform-hcl", "", "path to the terraform config file from project root")
+
+	deployTerraformCmd.MarkFlagRequired("remote-root")
+	debug.AddCommand(deployTerraformCmd)
+}

go.mod

@@ -18,10 +18,21 @@ require (
 )
 
 require (
+	github.com/google/uuid v1.3.0
+	github.com/hashicorp/go-version v1.6.0
+	github.com/hashicorp/hc-install v0.4.0
+	github.com/hashicorp/terraform-exec v0.17.3
 	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
 	golang.org/x/sync v0.1.0
 )
 
+require (
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/terraform-json v0.14.0 // indirect
+	github.com/zclconf/go-cty v1.11.0 // indirect
+	golang.org/x/crypto v0.1.0 // indirect
+)
+
 require (
 	cloud.google.com/go/compute v1.12.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
@@ -33,11 +44,10 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/imdario/mergo v0.3.13
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
+	golang.org/x/net v0.1.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	golang.org/x/sys v0.1.0 // indirect
 	golang.org/x/text v0.4.0 // indirect
@@ -47,7 +57,6 @@ require (
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
 	google.golang.org/grpc v1.50.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -4,6 +4,17 @@ cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x
 cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
 cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
+github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
+github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ=
+github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
+github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
+github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -22,19 +33,33 @@ github.com/databricks/databricks-sdk-go v0.1.0/go.mod h1:n4we8UoagEFH0VQuxkhQ3WH
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
+github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0=
+github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4=
+github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -53,54 +78,112 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/hc-install v0.4.0 h1:cZkRFr1WVa0Ty6x5fTvL1TuO1flul231rWkGH92oYYk=
+github.com/hashicorp/hc-install v0.4.0/go.mod h1:5d155H8EC5ewegao9A4PUTMNPZaq+TbOzkJJZ4vrXeI=
+github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU=
+github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI=
+github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e17dKDpqV7s=
+github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
+github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
 github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
+github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
+github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
 github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
 github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
+github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
+github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
+github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
+github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0=
+github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA=
+github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
@@ -110,14 +193,18 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
@@ -127,16 +214,27 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
@@ -151,6 +249,7 @@ google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
 google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -179,10 +278,18 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/locker_test.go

@@ -0,0 +1,168 @@
+package internal
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/databricks/bricks/bundle/deployer"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/repos"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TODO: create a utility function to create an empty test repo for tests and refactor sync_test integration test
+
+const EmptyRepoUrl = "https://github.com/shreyas-goenka/empty-repo.git"
+
+func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databricks.WorkspaceClient) string {
+	ctx := context.TODO()
+	me, err := wsc.CurrentUser.Me(ctx)
+	assert.NoError(t, err)
+
+	remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix))
+	repoInfo, err := wsc.Repos.Create(ctx, repos.CreateRepo{
+		Path:     remoteProjectRoot,
+		Url:      EmptyRepoUrl,
+		Provider: "gitHub",
+	})
+	assert.NoError(t, err)
+	t.Cleanup(func() {
+		err := wsc.Repos.DeleteByRepoId(ctx, repoInfo.Id)
+		assert.NoError(t, err)
+	})
+
+	return remoteProjectRoot
+}
+
+func createLocalTestProject(t *testing.T) string {
+	tempDir := t.TempDir()
+
+	cmd := exec.Command("git", "clone", EmptyRepoUrl)
+	cmd.Dir = tempDir
+	err := cmd.Run()
+	assert.NoError(t, err)
+
+	localProjectRoot := filepath.Join(tempDir, "empty-repo")
+	err = os.Chdir(localProjectRoot)
+	assert.NoError(t, err)
+	return localProjectRoot
+}
+
+func TestAccLock(t *testing.T) {
+	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+	ctx := context.TODO()
+	wsc := databricks.Must(databricks.NewWorkspaceClient())
+	createLocalTestProject(t)
+	remoteProjectRoot := createRemoteTestProject(t, "lock-acc-", wsc)
+
+	// 50 lockers try to acquire a lock at the same time
+	numConcurrentLocks := 50
+
+	var err error
+	lockerErrs := make([]error, numConcurrentLocks)
+	lockers := make([]*deployer.Locker, numConcurrentLocks)
+
+	for i := 0; i < numConcurrentLocks; i++ {
+		lockers[i] = deployer.CreateLocker("humpty.dumpty@databricks.com", remoteProjectRoot)
+	}
+
+	var wg sync.WaitGroup
+	for i := 0; i < numConcurrentLocks; i++ {
+		wg.Add(1)
+		currentIndex := i
+		go func() {
+			defer wg.Done()
+			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
+			lockerErrs[currentIndex] = lockers[currentIndex].Lock(ctx, wsc, false)
+		}()
+	}
+	wg.Wait()
+
+	countActive := 0
+	indexOfActiveLocker := 0
+	indexOfAnInactiveLocker := -1
+	for i := 0; i < numConcurrentLocks; i++ {
+		if lockers[i].Active {
+			countActive += 1
+			assert.NoError(t, lockerErrs[i])
+			indexOfActiveLocker = i
+		} else {
+			if indexOfAnInactiveLocker == -1 {
+				indexOfAnInactiveLocker = i
+			}
+			assert.ErrorContains(t, lockerErrs[i], "lock acquired by")
+			assert.ErrorContains(t, lockerErrs[i], "Use --force to override")
+		}
+	}
+	assert.Equal(t, 1, countActive, "Exactly one locker should successfull acquire the lock")
+
+	// test remote lock matches active lock
+	remoteLocker, err := deployer.GetActiveLockState(ctx, wsc, lockers[indexOfActiveLocker].RemotePath())
+	require.NoError(t, err)
+	assert.Equal(t, remoteLocker.ID, lockers[indexOfActiveLocker].State.ID, "remote locker id does not match active locker")
+	assert.True(t, remoteLocker.AcquisitionTime.Equal(lockers[indexOfActiveLocker].State.AcquisitionTime), "remote locker acquisition time does not match active locker")
+
+	// test all other locks (inactive ones) do not match the remote lock and Unlock fails
+	for i := 0; i < numConcurrentLocks; i++ {
+		if i == indexOfActiveLocker {
+			continue
+		}
+		assert.NotEqual(t, remoteLocker.ID, lockers[i].State.ID)
+		err := lockers[i].Unlock(ctx, wsc)
+		assert.ErrorContains(t, err, "unlock called when lock is not held")
+	}
+
+	// test inactive locks fail to write a file
+	for i := 0; i < numConcurrentLocks; i++ {
+		if i == indexOfActiveLocker {
+			continue
+		}
+		err := lockers[i].PutFile(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"), []byte(`'{"surname":"Khan", "name":"Shah Rukh"}`))
+		assert.ErrorContains(t, err, "failed to put file. deploy lock not held")
+	}
+
+	// active locker file write succeeds
+	err = lockers[indexOfActiveLocker].PutFile(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"), []byte(`{"surname":"Khan", "name":"Shah Rukh"}`))
+	assert.NoError(t, err)
+
+	// active locker file read succeeds with expected results
+	bytes, err := lockers[indexOfActiveLocker].GetRawJsonFileContent(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"))
+	var res map[string]string
+	json.Unmarshal(bytes, &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "Khan", res["surname"])
+	assert.Equal(t, "Shah Rukh", res["name"])
+
+	// inactive locker file reads fail
+	for i := 0; i < numConcurrentLocks; i++ {
+		if i == indexOfActiveLocker {
+			continue
+		}
+		_, err = lockers[i].GetRawJsonFileContent(ctx, wsc, path.Join(remoteProjectRoot, "foo.json"))
+		assert.ErrorContains(t, err, "failed to get file. deploy lock not held")
+	}
+
+	// Unlock active lock and check it becomes inactive
+	err = lockers[indexOfActiveLocker].Unlock(ctx, wsc)
+	assert.NoError(t, err)
+	remoteLocker, err = deployer.GetActiveLockState(ctx, wsc, lockers[indexOfActiveLocker].RemotePath())
+	assert.ErrorContains(t, err, "File not found.", "remote lock file not deleted on unlock")
+	assert.Nil(t, remoteLocker)
+	assert.False(t, lockers[indexOfActiveLocker].Active)
+
+	// A locker that failed to acquire the lock should now be able to acquire it
+	assert.False(t, lockers[indexOfAnInactiveLocker].Active)
+	err = lockers[indexOfAnInactiveLocker].Lock(ctx, wsc, false)
+	assert.NoError(t, err)
+	assert.True(t, lockers[indexOfAnInactiveLocker].Active)
+}

main.go

@@ -4,6 +4,7 @@ import (
 	_ "github.com/databricks/bricks/cmd/api"
 	_ "github.com/databricks/bricks/cmd/bundle"
 	_ "github.com/databricks/bricks/cmd/bundle/debug"
+	_ "github.com/databricks/bricks/cmd/bundle/debug/deploy"
 	_ "github.com/databricks/bricks/cmd/configure"
 	_ "github.com/databricks/bricks/cmd/fs"
 	_ "github.com/databricks/bricks/cmd/init"

utilities/workspace_files.go

@@ -0,0 +1,59 @@
+package utilities
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/client"
+)
+
+// NOTE: This API is only available for files in /Repos if a workspace has repos
+// in workspace enabled and files in workspace not enabled
+//
+// Get the file contents of a json file in workspace
+// TODO(Nov 2022): add method in go sdk to get the raw bytes from response of an API
+//
+// TODO(Nov 2022): talk to eng-files team about what the response structure would look like.
+//       This function would have to be modfified probably in the future once this
+//       API goes to public preview
+func GetRawJsonFileContent(ctx context.Context, wsc *databricks.WorkspaceClient, path string) ([]byte, error) {
+	apiClient, err := client.New(wsc.Config)
+	if err != nil {
+		return nil, err
+	}
+	exportApiPath := fmt.Sprintf(
+		"/api/2.0/workspace-files/%s",
+		strings.TrimLeft(path, "/"))
+
+	var res json.RawMessage
+
+	err = apiClient.Do(ctx, http.MethodGet, exportApiPath, nil, &res)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch file %s: %s", path, err)
+	}
+	return res, nil
+}
+
+func WriteFile(ctx context.Context, wsc *databricks.WorkspaceClient, pathToFile string, content []byte, overwrite bool) error {
+	apiClient, err := client.New(wsc.Config)
+	if err != nil {
+		return err
+	}
+	err = wsc.Workspace.MkdirsByPath(ctx, path.Dir(pathToFile))
+	if err != nil {
+		return fmt.Errorf("could not mkdir to post file: %s", err)
+	}
+
+	importApiPath := fmt.Sprintf(
+		"/api/2.0/workspace-files/import-file/%s?overwrite=%s",
+		strings.TrimLeft(pathToFile, "/"), strconv.FormatBool(overwrite))
+
+	return apiClient.Do(ctx, http.MethodPost, importApiPath, bytes.NewReader(content), nil)
+}