Refactor planning of deployments and destroys

bundle/bundle.go

@@ -20,8 +20,8 @@ import (
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/locker"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/plan"
 	"github.com/databricks/cli/libs/tags"
-	"github.com/databricks/cli/libs/terraform"
 	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
@@ -63,7 +63,7 @@ type Bundle struct {
 	// Stores the locker responsible for acquiring/releasing a deployment lock.
 	Locker *locker.Locker
 
-	Plan *terraform.Plan
+	Plan plan.Plan
 
 	// if true, we skip approval checks for deploy, destroy resources and delete
 	// files

bundle/deploy/files/delete.go

@@ -12,7 +12,6 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/sync"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
-	"github.com/fatih/color"
 )
 
 type delete struct{}
@@ -22,24 +21,7 @@ func (m *delete) Name() string {
 }
 
 func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	// Do not delete files if terraform destroy was not consented
+	cmdio.LogString(ctx, "Deleting files...")
-	if !b.Plan.IsEmpty && !b.Plan.ConfirmApply {
-		return nil
-	}
-
-	cmdio.LogString(ctx, "Starting deletion of remote bundle files")
-	cmdio.LogString(ctx, fmt.Sprintf("Bundle remote directory is %s", b.Config.Workspace.RootPath))
-
-	red := color.New(color.FgRed).SprintFunc()
-	if !b.AutoApprove {
-		proceed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\n%s and all files in it will be %s Proceed?", b.Config.Workspace.RootPath, red("deleted permanently!")))
-		if err != nil {
-			return diag.FromErr(err)
-		}
-		if !proceed {
-			return nil
-		}
-	}
 
 	err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{
 		Path:      b.Config.Workspace.RootPath,
@@ -54,8 +36,6 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if err != nil {
 		return diag.FromErr(err)
 	}
-
-	cmdio.LogString(ctx, "Successfully deleted files!")
 	return nil
 }
 

bundle/deploy/terraform/apply.go

@@ -1,45 +0,0 @@
-package terraform
-
-import (
-	"context"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/log"
-	"github.com/hashicorp/terraform-exec/tfexec"
-)
-
-type apply struct{}
-
-func (w *apply) Name() string {
-	return "terraform.Apply"
-}
-
-func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	tf := b.Terraform
-	if tf == nil {
-		return diag.Errorf("terraform not initialized")
-	}
-
-	cmdio.LogString(ctx, "Deploying resources...")
-
-	err := tf.Init(ctx, tfexec.Upgrade(true))
-	if err != nil {
-		return diag.Errorf("terraform init: %v", err)
-	}
-
-	err = tf.Apply(ctx)
-	if err != nil {
-		return diag.Errorf("terraform apply: %v", err)
-	}
-
-	log.Infof(ctx, "Resource deployment completed")
-	return nil
-}
-
-// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply`
-// from the bundle's ephemeral working directory for Terraform.
-func Apply() bundle.Mutator {
-	return &apply{}
-}

bundle/deploy/terraform/deploy.go

@@ -0,0 +1,42 @@
+package terraform
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/log"
+)
+
+// TODO: Release lock on error?
+type tfDeploy struct{}
+
+func (w *tfDeploy) Name() string {
+	return "terraform.Deploy"
+}
+
+// TODO: Document in PR why the approval logic is being moved to the plan mutator.
+func (w *tfDeploy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	// return early if plan is empty
+	if b.Plan.IsEmpty() {
+		cmdio.LogString(ctx, "No changes to deploy...")
+		return nil
+	}
+
+	cmdio.LogString(ctx, "Deploying resources...")
+
+	err := b.Plan.Apply()
+	if err != nil {
+		return diag.Errorf("terraform apply: %v", err)
+	}
+
+	log.Infof(ctx, "Resource deployment completed")
+	return nil
+}
+
+// Deploy returns a [bundle.Mutator] that runs the equivalent of `terraform apply`
+// from the bundle's ephemeral working directory for Terraform.
+func Deploy() bundle.Mutator {
+	return &tfDeploy{}
+}

bundle/deploy/terraform/destroy.go

@@ -2,61 +2,12 @@ package terraform
 
 import (
 	"context"
-	"fmt"
-	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/fatih/color"
-	"github.com/hashicorp/terraform-exec/tfexec"
-	tfjson "github.com/hashicorp/terraform-json"
 )
 
-type PlanResourceChange struct {
-	ResourceType string `json:"resource_type"`
-	Action       string `json:"action"`
-	ResourceName string `json:"resource_name"`
-}
-
-func (c *PlanResourceChange) String() string {
-	result := strings.Builder{}
-	switch c.Action {
-	case "delete":
-		result.WriteString("  delete ")
-	default:
-		result.WriteString(c.Action + " ")
-	}
-	switch c.ResourceType {
-	case "databricks_job":
-		result.WriteString("job ")
-	case "databricks_pipeline":
-		result.WriteString("pipeline ")
-	default:
-		result.WriteString(c.ResourceType + " ")
-	}
-	result.WriteString(c.ResourceName)
-	return result.String()
-}
-
-func (c *PlanResourceChange) IsInplaceSupported() bool {
-	return false
-}
-
-func logDestroyPlan(ctx context.Context, changes []*tfjson.ResourceChange) error {
-	cmdio.LogString(ctx, "The following resources will be removed:")
-	for _, c := range changes {
-		if c.Change.Actions.Delete() {
-			cmdio.Log(ctx, &PlanResourceChange{
-				ResourceType: c.Type,
-				Action:       "delete",
-				ResourceName: c.Name,
-			})
-		}
-	}
-	return nil
-}
-
 type destroy struct{}
 
 func (w *destroy) Name() string {
@@ -65,55 +16,17 @@ func (w *destroy) Name() string {
 
 func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// return early if plan is empty
-	if b.Plan.IsEmpty {
+	if b.Plan.IsEmpty() {
-		cmdio.LogString(ctx, "No resources to destroy in plan. Skipping destroy!")
+		cmdio.LogString(ctx, "No resources to destroy...")
 		return nil
 	}
 
-	tf := b.Terraform
+	cmdio.LogString(ctx, "Destroying resources...")
-	if tf == nil {
-		return diag.Errorf("terraform not initialized")
-	}
 
-	// read plan file
+	err := b.Plan.Apply()
-	plan, err := tf.ShowPlanFile(ctx, b.Plan.Path)
 	if err != nil {
-		return diag.FromErr(err)
+		return diag.Errorf("terraform apply: %v", err)
 	}
-
-	// print the resources that will be destroyed
-	err = logDestroyPlan(ctx, plan.ResourceChanges)
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
-	// Ask for confirmation, if needed
-	if !b.Plan.ConfirmApply {
-		red := color.New(color.FgRed).SprintFunc()
-		b.Plan.ConfirmApply, err = cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nThis will permanently %s resources! Proceed?", red("destroy")))
-		if err != nil {
-			return diag.FromErr(err)
-		}
-	}
-
-	// return if confirmation was not provided
-	if !b.Plan.ConfirmApply {
-		return nil
-	}
-
-	if b.Plan.Path == "" {
-		return diag.Errorf("no plan found")
-	}
-
-	cmdio.LogString(ctx, "Starting to destroy resources")
-
-	// Apply terraform according to the computed destroy plan
-	err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path))
-	if err != nil {
-		return diag.Errorf("terraform destroy: %v", err)
-	}
-
-	cmdio.LogString(ctx, "Successfully destroyed resources!")
 	return nil
 }
 

bundle/deploy/terraform/plan.go

@@ -1,73 +0,0 @@
-package terraform
-
-import (
-	"context"
-	"fmt"
-	"path/filepath"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/terraform"
-	"github.com/hashicorp/terraform-exec/tfexec"
-)
-
-type PlanGoal string
-
-var (
-	PlanDeploy  = PlanGoal("deploy")
-	PlanDestroy = PlanGoal("destroy")
-)
-
-type plan struct {
-	goal PlanGoal
-}
-
-func (p *plan) Name() string {
-	return "terraform.Plan"
-}
-
-func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	tf := b.Terraform
-	if tf == nil {
-		return diag.Errorf("terraform not initialized")
-	}
-
-	cmdio.LogString(ctx, "Starting plan computation")
-
-	err := tf.Init(ctx, tfexec.Upgrade(true))
-	if err != nil {
-		return diag.Errorf("terraform init: %v", err)
-	}
-
-	// Persist computed plan
-	tfDir, err := Dir(ctx, b)
-	if err != nil {
-		return diag.FromErr(err)
-	}
-	planPath := filepath.Join(tfDir, "plan")
-	destroy := p.goal == PlanDestroy
-
-	notEmpty, err := tf.Plan(ctx, tfexec.Destroy(destroy), tfexec.Out(planPath))
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
-	// Set plan in main bundle struct for downstream mutators
-	b.Plan = &terraform.Plan{
-		Path:         planPath,
-		ConfirmApply: b.AutoApprove,
-		IsEmpty:      !notEmpty,
-	}
-
-	cmdio.LogString(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath))
-	return nil
-}
-
-// Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
-// from the bundle's ephemeral working directory for Terraform.
-func Plan(goal PlanGoal) bundle.Mutator {
-	return &plan{
-		goal: goal,
-	}
-}

bundle/deploy/terraform/plan_deploy.go

@@ -0,0 +1,78 @@
+package terraform
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/plan"
+)
+
+type planDeploy struct{}
+
+func (p *planDeploy) Name() string {
+	return "terraform.PlanDeploy"
+}
+
+// TODO: We need end to end tests for the approval flow.
+func (p *planDeploy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	tf := b.Terraform
+	if tf == nil {
+		return diag.Errorf("terraform not initialized")
+	}
+
+	// Get terraform project root.
+	tfDir, err := Dir(ctx, b)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// Set plan in main bundle struct for downstream mutators
+	b.Plan, err = plan.NewTerraformPlan(ctx, plan.TerraformPlanOpts{
+		Executable: tf,
+		Root:       tfDir,
+		IsDestroy:  false,
+	})
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// If the `--auto-approve` flag is set, we don't need to ask for approval.
+	if b.AutoApprove {
+		return nil
+	}
+
+	destructiveActions := b.Plan.ActionsByTypes(plan.ActionTypeRecreate, plan.ActionTypeDelete)
+
+	// If there are no deletes or recreates, we don't need to ask for approval.
+	if len(destructiveActions) == 0 {
+		return nil
+	}
+
+	// display information the user needs to know about the deploy plan.
+	cmdio.LogString(ctx, "The following resources will be deleted or recreated:")
+	for _, a := range destructiveActions {
+		cmdio.Log(ctx, a)
+	}
+
+	approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?")
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// TODO: Remove the previous flag for deployment in the bundle tree?
+	if !approved {
+		// We error here to terminate the control flow and prevent the current
+		// process from modifying any deployment state.
+		return diag.Errorf("deployment terminated. No changes were made.")
+	}
+
+	return nil
+}
+
+// PlanDeploy returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
+// from the bundle's ephemeral working directory for Terraform.
+func PlanDeploy() bundle.Mutator {
+	return &planDeploy{}
+}

bundle/deploy/terraform/plan_destroy.go

@@ -0,0 +1,86 @@
+package terraform
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/plan"
+)
+
+type planDestroy struct{}
+
+func (p *planDestroy) Name() string {
+	return "terraform.PlanDestroy"
+}
+
+// TODO: We need end to end tests for the approval flow.
+func (p *planDestroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	tf := b.Terraform
+	if tf == nil {
+		return diag.Errorf("terraform not initialized")
+	}
+
+	// Get terraform project root.
+	tfDir, err := Dir(ctx, b)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	b.Plan, err = plan.NewTerraformPlan(ctx, plan.TerraformPlanOpts{
+		Executable: tf,
+		Root:       tfDir,
+		IsDestroy:  true,
+	})
+
+	// TODO: Note this PR also makes destroy idempotent.
+
+	// Display information needs to know about the destroy plan.
+	deleteActions := b.Plan.ActionsByTypes(plan.ActionTypeDelete)
+
+	// If there are delete actions, show that information to the user.
+	if len(deleteActions) != 0 {
+		cmdio.LogString(ctx, "The following resources will be deleted:")
+		for _, a := range deleteActions {
+			cmdio.Log(ctx, a)
+		}
+	}
+
+	// TODO: write e2e integration tests / unit tests to check this property.
+	// TODO: Also test this manually.
+	f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.RootPath)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// If the root path exists, show warning.
+	if _, err := f.Stat(ctx, ""); err == nil {
+		cmdio.LogString(ctx, fmt.Sprintf("All files and directories at %s will be deleted.", b.Config.Workspace.RootPath))
+	}
+
+	// If the `--auto-approve` flag is set, we don't need to ask for approval.
+	if b.AutoApprove {
+		return nil
+	}
+
+	approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?")
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	// TODO: Remove the previous flag for deployment in the bundle tree?
+	if !approved {
+		return diag.Errorf("destroy terminated. No changes were made.")
+	}
+
+	return nil
+}
+
+// Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
+// from the bundle's ephemeral working directory for Terraform.
+func PlanDestroy() bundle.Mutator {
+	return &planDestroy{}
+}

bundle/phases/deploy.go

@@ -37,8 +37,9 @@ func Deploy() bundle.Mutator {
 				terraform.Interpolate(),
 				terraform.Write(),
 				terraform.CheckRunningResource(),
+				terraform.PlanDeploy(),
 				bundle.Defer(
-					terraform.Apply(),
+					terraform.Deploy(),
 					bundle.Seq(
 						terraform.StatePush(),
 						terraform.Load(),

bundle/phases/destroy.go

@@ -17,7 +17,7 @@ func Destroy() bundle.Mutator {
 				terraform.StatePull(),
 				terraform.Interpolate(),
 				terraform.Write(),
-				terraform.Plan(terraform.PlanGoal("destroy")),
+				terraform.PlanDestroy(),
 				terraform.Destroy(),
 				terraform.StatePush(),
 				files.Delete(),

libs/plan/plan.go

@@ -0,0 +1,43 @@
+package plan
+
+import "strings"
+
+type Plan interface {
+	// Path where the plan is persisted
+	// Path() string
+
+	// Get all actions of the given types
+	ActionsByTypes(...ActionType) []Action
+
+	// Whether the plan is empty, i.e. no actions
+	IsEmpty() bool
+
+	// Apply the plan, changing the underlying resources.
+	Apply() error
+}
+
+type Action struct {
+	rtype string
+	rname string
+
+	atype ActionType
+}
+
+func (a Action) String() string {
+	return strings.Join([]string{" ", string(a.atype), a.rtype, a.rname}, " ")
+}
+
+func (c Action) IsInplaceSupported() bool {
+	return false
+}
+
+type ActionType string
+
+const (
+	ActionTypeCreate   ActionType = "create"
+	ActionTypeDelete   ActionType = "delete"
+	ActionTypeUpdate   ActionType = "update"
+	ActionTypeNoOp     ActionType = "no-op"
+	ActionTypeRead     ActionType = "read"
+	ActionTypeRecreate ActionType = "recreate"
+)

libs/plan/terraform.go

@@ -0,0 +1,138 @@
+package plan
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strings"
+
+	"github.com/databricks/cli/libs/log"
+	"github.com/hashicorp/terraform-exec/tfexec"
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+const planDirName = "plan"
+
+// TODO: start using this. Also clean up the fields in the struct below.
+type TerraformPlanOpts struct {
+	// If true, the plan will be a IsDestroy plan.
+	IsDestroy bool
+
+	// Path where the terraform project we'll compute the plan for is rooted at.
+	Root string
+
+	// Executable executable to compute the plan with.
+	Executable *tfexec.Terraform
+}
+
+// TODO: Remove any unnecessary fields in here.
+type terraformPlan struct {
+	ctx context.Context
+
+	opts TerraformPlanOpts
+
+	// Path to the computed plan.
+	planPath string
+
+	// In memory representation of the computed plan, obtained by running terraform show.
+	tfPlan *tfjson.Plan
+
+	// TODO: Can we invert this?
+	notEmpty bool
+}
+
+// NewTerraformPlan creates a new Terraform plan for a terraform project rooted
+// at the given path. If destroy is true, the plan will be a destroy plan.
+func NewTerraformPlan(ctx context.Context, opts TerraformPlanOpts) (Plan, error) {
+	plan := &terraformPlan{
+		ctx:  ctx,
+		opts: opts,
+	}
+
+	if plan.opts.Executable == nil {
+		return nil, fmt.Errorf("terraform executable not provided")
+	}
+	if plan.opts.Root == "" {
+		return nil, fmt.Errorf("root path to compute the terraform plan not provided")
+	}
+
+	// Initialize the terraform project
+	err := opts.Executable.Init(ctx, tfexec.Upgrade(true))
+	if err != nil {
+		return nil, fmt.Errorf("terraform init: %w", err)
+	}
+
+	// Path where the computed plan will be persisted.
+	planPath := filepath.Join(opts.Root, planDirName)
+
+	log.Debugf(ctx, "Computing terraform plan")
+	plan.notEmpty, err = opts.Executable.Plan(ctx, tfexec.Destroy(opts.IsDestroy), tfexec.Out(planPath))
+	if err != nil {
+		return nil, fmt.Errorf("terraform plan: %w", err)
+	}
+
+	log.Debugf(ctx, "Loading terraform plan by running terraform show")
+	plan.tfPlan, err = opts.Executable.ShowPlanFile(ctx, planPath)
+	if err != nil {
+		return nil, fmt.Errorf("terraform show: %w", err)
+	}
+
+	// Persist the plan path
+	plan.planPath = planPath
+
+	return plan, nil
+}
+
+func (p *terraformPlan) Path() string {
+	return p.planPath
+}
+
+// TODO: Add tests for this functionality.
+func (p *terraformPlan) ActionsByTypes(query ...ActionType) []Action {
+	actions := make([]Action, 0)
+	for _, rc := range p.tfPlan.ResourceChanges {
+		tfActions := rc.Change.Actions
+		switch {
+		case slices.Contains(query, ActionTypeDelete) && tfActions.Delete():
+			actions = append(actions, Action{
+				rtype: strings.TrimPrefix(rc.Type, "databricks_"),
+				rname: rc.Name,
+				atype: ActionTypeDelete,
+			})
+		case slices.Contains(query, ActionTypeRecreate) && tfActions.Replace():
+			actions = append(actions, Action{
+				rtype: strings.TrimPrefix(rc.Type, "databricks_"),
+				rname: rc.Name,
+				atype: ActionTypeRecreate,
+			})
+		default:
+			// We don't need to track other action types yet.
+		}
+	}
+
+	// Sort by action type first, then by resource type, and finally by resource name
+	sort.Slice(actions, func(i, j int) bool {
+		if actions[i].atype != actions[j].atype {
+			return actions[i].atype < actions[j].atype
+		}
+		if actions[i].rtype != actions[j].rtype {
+			return actions[i].rtype < actions[j].rtype
+		}
+		return actions[i].rname < actions[j].rname
+	})
+	return actions
+}
+
+func (p *terraformPlan) IsEmpty() bool {
+	return !p.notEmpty
+}
+
+func (p *terraformPlan) Apply() error {
+	err := p.opts.Executable.Apply(p.ctx, tfexec.DirOrPlan(p.planPath))
+	if err != nil {
+		return fmt.Errorf("terraform apply: %w", err)
+	}
+	return nil
+}

libs/terraform/plan.go

@@ -1,13 +0,0 @@
-package terraform
-
-type Plan struct {
-	// Path to the plan
-	Path string
-
-	// Holds whether the user can consented to destruction. Either by interactive
-	// confirmation or by passing a command line flag
-	ConfirmApply bool
-
-	// If true, the plan is empty and applying it will not do anything
-	IsEmpty bool
-}