Add bundle destroy command (#300)

Adds bundle destroy capability to bricks

bundle/bundle.go

@@ -16,6 +16,7 @@ import (
 	"github.com/databricks/bricks/folders"
 	"github.com/databricks/bricks/libs/git"
 	"github.com/databricks/bricks/libs/locker"
+	"github.com/databricks/bricks/libs/terraform"
 	"github.com/databricks/databricks-sdk-go"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -34,6 +35,12 @@ type Bundle struct {
 
 	// Stores the locker responsible for acquiring/releasing a deployment lock.
 	Locker *locker.Locker
+
+	Plan *terraform.Plan
+
+	// if true, we skip approval checks for deploy, destroy resources and delete
+	// files
+	AutoApprove bool
 }
 
 func Load(path string) (*Bundle, error) {

bundle/deploy/files/delete.go

@@ -0,0 +1,58 @@
+package files
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/libs/cmdio"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/fatih/color"
+)
+
+type delete struct{}
+
+func (m *delete) Name() string {
+	return "files.Delete"
+}
+
+func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
+	// Do not delete files if terraform destroy was not consented
+	if !b.Plan.IsEmpty && !b.Plan.ConfirmApply {
+		return nil, nil
+	}
+
+	// interface to io with the user
+	logger, ok := cmdio.FromContext(ctx)
+	if !ok {
+		return nil, fmt.Errorf("no logger found")
+	}
+	red := color.New(color.FgRed).SprintFunc()
+
+	fmt.Fprintf(os.Stderr, "\nRemote directory %s will be deleted\n", b.Config.Workspace.Root)
+	if !b.AutoApprove {
+		proceed, err := logger.Ask(fmt.Sprintf("%s and all files in it will be %s Proceed?: ", b.Config.Workspace.Root, red("deleted permanently!")))
+		if err != nil {
+			return nil, err
+		}
+		if !proceed {
+			return nil, nil
+		}
+	}
+
+	err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{
+		Path:      b.Config.Workspace.Root,
+		Recursive: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("Successfully deleted files!")
+	return nil, nil
+}
+
+func Delete() bundle.Mutator {
+	return &delete{}
+}

bundle/deploy/terraform/destroy.go

@@ -0,0 +1,126 @@
+package terraform
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/libs/cmdio"
+	"github.com/fatih/color"
+	"github.com/hashicorp/terraform-exec/tfexec"
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+// TODO: This is temporary. Come up with a robust way to log mutator progress and
+// status events
+type PlanResourceChange struct {
+	ResourceType string `json:"resource_type"`
+	Action       string `json:"action"`
+	ResourceName string `json:"resource_name"`
+}
+
+func (c *PlanResourceChange) String() string {
+	result := strings.Builder{}
+	switch c.Action {
+	case "delete":
+		result.WriteString("  delete ")
+	default:
+		result.WriteString(c.Action + " ")
+	}
+	switch c.ResourceType {
+	case "databricks_job":
+		result.WriteString("job ")
+	case "databricks_pipeline":
+		result.WriteString("pipeline ")
+	default:
+		result.WriteString(c.ResourceType + " ")
+	}
+	result.WriteString(c.ResourceName)
+	return result.String()
+}
+
+func logDestroyPlan(l *cmdio.Logger, changes []*tfjson.ResourceChange) error {
+	// TODO: remove once we have mutator logging in place
+	fmt.Fprintln(os.Stderr, "The following resources will be removed: ")
+	for _, c := range changes {
+		if c.Change.Actions.Delete() {
+			l.Log(&PlanResourceChange{
+				ResourceType: c.Type,
+				Action:       "delete",
+				ResourceName: c.Name,
+			})
+		}
+	}
+	return nil
+}
+
+type destroy struct{}
+
+func (w *destroy) Name() string {
+	return "terraform.Destroy"
+}
+
+func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
+	// interface to io with the user
+	logger, ok := cmdio.FromContext(ctx)
+	if !ok {
+		return nil, fmt.Errorf("no logger found")
+	}
+
+	if b.Plan.IsEmpty {
+		fmt.Fprintln(os.Stderr, "No resources to destroy!")
+		return nil, nil
+	}
+
+	tf := b.Terraform
+	if tf == nil {
+		return nil, fmt.Errorf("terraform not initialized")
+	}
+
+	// read plan file
+	plan, err := tf.ShowPlanFile(ctx, b.Plan.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	// print the resources that will be destroyed
+	err = logDestroyPlan(logger, plan.ResourceChanges)
+	if err != nil {
+		return nil, err
+	}
+
+	// Ask for confirmation, if needed
+	if !b.Plan.ConfirmApply {
+		red := color.New(color.FgRed).SprintFunc()
+		b.Plan.ConfirmApply, err = logger.Ask(fmt.Sprintf("\nThis will permanently %s resources! Proceed? [y/n]: ", red("destroy")))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// return if confirmation was not provided
+	if !b.Plan.ConfirmApply {
+		return nil, nil
+	}
+
+	if b.Plan.Path == "" {
+		return nil, fmt.Errorf("no plan found")
+	}
+
+	// Apply terraform according to the computed destroy plan
+	err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path))
+	if err != nil {
+		return nil, fmt.Errorf("terraform destroy: %w", err)
+	}
+
+	fmt.Fprintln(os.Stderr, "Successfully destroyed resources!")
+	return nil, nil
+}
+
+// Destroy returns a [bundle.Mutator] that runs the conceptual equivalent of
+// `terraform destroy ./plan` from the bundle's ephemeral working directory for Terraform.
+func Destroy() bundle.Mutator {
+	return &destroy{}
+}

bundle/deploy/terraform/plan.go

@@ -0,0 +1,66 @@
+package terraform
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/libs/terraform"
+	"github.com/hashicorp/terraform-exec/tfexec"
+)
+
+type PlanGoal string
+
+var (
+	PlanDeploy  = PlanGoal("deploy")
+	PlanDestroy = PlanGoal("destroy")
+)
+
+type plan struct {
+	goal PlanGoal
+}
+
+func (p *plan) Name() string {
+	return "terraform.Plan"
+}
+
+func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
+	tf := b.Terraform
+	if tf == nil {
+		return nil, fmt.Errorf("terraform not initialized")
+	}
+
+	err := tf.Init(ctx, tfexec.Upgrade(true))
+	if err != nil {
+		return nil, fmt.Errorf("terraform init: %w", err)
+	}
+
+	// Persist computed plan
+	tfDir, err := Dir(b)
+	if err != nil {
+		return nil, err
+	}
+	planPath := filepath.Join(tfDir, "plan")
+	destroy := p.goal == PlanDestroy
+	notEmpty, err := tf.Plan(ctx, tfexec.Destroy(destroy), tfexec.Out(planPath))
+	if err != nil {
+		return nil, err
+	}
+
+	// Set plan in main bundle struct for downstream mutators
+	b.Plan = &terraform.Plan{
+		Path:         planPath,
+		ConfirmApply: b.AutoApprove,
+		IsEmpty:      !notEmpty,
+	}
+	return nil, nil
+}
+
+// Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
+// from the bundle's ephemeral working directory for Terraform.
+func Plan(goal PlanGoal) bundle.Mutator {
+	return &plan{
+		goal: goal,
+	}
+}

bundle/phases/destroy.go

@@ -0,0 +1,24 @@
+package phases
+
+import (
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/bundle/deploy/files"
+	"github.com/databricks/bricks/bundle/deploy/lock"
+	"github.com/databricks/bricks/bundle/deploy/terraform"
+)
+
+// The destroy phase deletes artifacts and resources.
+func Destroy() bundle.Mutator {
+	return newPhase(
+		"destroy",
+		[]bundle.Mutator{
+			lock.Acquire(),
+			terraform.StatePull(),
+			terraform.Plan(terraform.PlanGoal("destroy")),
+			terraform.Destroy(),
+			terraform.StatePush(),
+			lock.Release(),
+			files.Delete(),
+		},
+	)
+}

bundle/run/job.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/databricks/bricks/bundle"
 	"github.com/databricks/bricks/bundle/config/resources"
+	"github.com/databricks/bricks/libs/cmdio"
 	"github.com/databricks/bricks/libs/log"
-	"github.com/databricks/bricks/libs/progress"
 	"github.com/databricks/databricks-sdk-go/retries"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/fatih/color"
@@ -177,7 +177,7 @@ func logDebugCallback(ctx context.Context, runId *int64) func(info *retries.Info
 	}
 }
 
-func logProgressCallback(ctx context.Context, progressLogger *progress.Logger) func(info *retries.Info[jobs.Run]) {
+func logProgressCallback(ctx context.Context, progressLogger *cmdio.Logger) func(info *retries.Info[jobs.Run]) {
 	var prevState *jobs.RunState
 	return func(info *retries.Info[jobs.Run]) {
 		i := info.Info
@@ -241,7 +241,7 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (RunOutput, error) {
 	logDebug := logDebugCallback(ctx, runId)
 
 	// callback to log progress events. Called on every poll request
-	progressLogger, ok := progress.FromContext(ctx)
+	progressLogger, ok := cmdio.FromContext(ctx)
 	if !ok {
 		return nil, fmt.Errorf("no progress logger found")
 	}

bundle/run/pipeline.go

@@ -9,9 +9,9 @@ import (
 	"github.com/databricks/bricks/bundle"
 	"github.com/databricks/bricks/bundle/config/resources"
 	"github.com/databricks/bricks/bundle/run/pipeline"
+	"github.com/databricks/bricks/libs/cmdio"
 	"github.com/databricks/bricks/libs/flags"
 	"github.com/databricks/bricks/libs/log"
-	"github.com/databricks/bricks/libs/progress"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	flag "github.com/spf13/pflag"
 )
@@ -162,7 +162,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (RunOutput, err
 
 	// setup progress logger and tracker to query events
 	updateTracker := pipeline.NewUpdateTracker(pipelineID, updateID, w)
-	progressLogger, ok := progress.FromContext(ctx)
+	progressLogger, ok := cmdio.FromContext(ctx)
 	if !ok {
 		return nil, fmt.Errorf("no progress logger found")
 	}

cmd/bundle/destroy.go

@@ -0,0 +1,50 @@
+package bundle
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/databricks/bricks/bundle"
+	"github.com/databricks/bricks/bundle/phases"
+	"github.com/databricks/bricks/cmd/root"
+	"github.com/databricks/bricks/libs/cmdio"
+	"github.com/databricks/bricks/libs/flags"
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+)
+
+var destroyCmd = &cobra.Command{
+	Use:   "destroy",
+	Short: "Destroy deployed bundle resources",
+
+	PreRunE: root.MustConfigureBundle,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		b := bundle.Get(cmd.Context())
+
+		// If `--force` is specified, force acquisition of the deployment lock.
+		b.Config.Bundle.Lock.Force = force
+
+		// If `--auto-approve`` is specified, we skip confirmation checks
+		b.AutoApprove = autoApprove
+
+		// we require auto-approve for non tty terminals since interactive consent
+		// is not possible
+		if !term.IsTerminal(int(os.Stderr.Fd())) && !autoApprove {
+			return fmt.Errorf("please specify --auto-approve to skip interactive confirmation checks for non tty consoles")
+		}
+
+		ctx := cmdio.NewContext(cmd.Context(), cmdio.NewLogger(flags.ModeAppend))
+		return bundle.Apply(ctx, b, []bundle.Mutator{
+			phases.Initialize(),
+			phases.Build(),
+			phases.Destroy(),
+		})
+	},
+}
+
+var autoApprove bool
+
+func init() {
+	AddCommand(destroyCmd)
+	destroyCmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals for deleting resources and files")
+}

cmd/root/progress_logger.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/databricks/bricks/libs/cmdio"
 	"github.com/databricks/bricks/libs/flags"
-	"github.com/databricks/bricks/libs/progress"
 	"golang.org/x/term"
 )
 
@@ -31,8 +31,8 @@ func initializeProgressLogger(ctx context.Context) (context.Context, error) {
 		format = resolveModeDefault(format)
 	}
 
-	progressLogger := progress.NewLogger(format)
-	return progress.NewContext(ctx, progressLogger), nil
+	progressLogger := cmdio.NewLogger(format)
+	return cmdio.NewContext(ctx, progressLogger), nil
 }
 
 var progressFormat = flags.NewProgressLogFormat()

cmd/root/progress_logger_test.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"testing"
 
+	"github.com/databricks/bricks/libs/cmdio"
 	"github.com/databricks/bricks/libs/flags"
-	"github.com/databricks/bricks/libs/progress"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,7 +39,7 @@ func TestDefaultLoggerModeResolution(t *testing.T) {
 	require.Equal(t, progressFormat, flags.ModeDefault)
 	ctx, err := initializeProgressLogger(context.Background())
 	require.NoError(t, err)
-	logger, ok := progress.FromContext(ctx)
+	logger, ok := cmdio.FromContext(ctx)
 	assert.True(t, ok)
 	assert.Equal(t, logger.Mode, flags.ModeAppend)
 }

libs/cmdio/context.go

@@ -1,4 +1,4 @@
-package progress
+package cmdio
 
 import (
 	"context"

libs/cmdio/event.go

@@ -1,4 +1,4 @@
-package progress
+package cmdio
 
 type Event interface {
 	String() string

libs/cmdio/logger.go

@@ -1,6 +1,7 @@
-package progress
+package cmdio
 
 import (
+	"bufio"
 	"encoding/json"
 	"io"
 	"os"
@@ -10,6 +11,8 @@ import (
 
 type Logger struct {
 	Mode flags.ProgressLogFormat
+
+	Reader bufio.Reader
 	Writer io.Writer
 
 	isFirstEvent bool
@@ -19,10 +22,26 @@ func NewLogger(mode flags.ProgressLogFormat) *Logger {
 	return &Logger{
 		Mode:         mode,
 		Writer:       os.Stderr,
+		Reader:       *bufio.NewReader(os.Stdin),
 		isFirstEvent: true,
 	}
 }
 
+func (l *Logger) Ask(question string) (bool, error) {
+	l.Writer.Write([]byte(question))
+	ans, err := l.Reader.ReadString('\n')
+
+	if err != nil {
+		return false, err
+	}
+
+	if ans == "y\n" {
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
 func (l *Logger) Log(event Event) {
 	switch l.Mode {
 	case flags.ModeInplace:

libs/terraform/plan.go

@@ -0,0 +1,13 @@
+package terraform
+
+type Plan struct {
+	// Path to the plan
+	Path string
+
+	// Holds whether the user can consented to destruction. Either by interactive
+	// confirmation or by passing a command line flag
+	ConfirmApply bool
+
+	// If true, the plan is empty and applying it will not do anything
+	IsEmpty bool
+}