steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor planning of deployments and destroys

This commit is contained in:
Shreyas Goenka 2024-07-08 17:35:47 +02:00
parent 040b374430
commit 2ec5475d97
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
13 changed files with 398 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bundle/bundle.go
View File

@ -20,8 +20,8 @@ import (
"github.com/databricks/cli/libs/git"
"github.com/databricks/cli/libs/locker"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/plan"
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/terraform"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go"
sdkconfig "github.com/databricks/databricks-sdk-go/config"
@ -63,7 +63,7 @@ type Bundle struct {
// Stores the locker responsible for acquiring/releasing a deployment lock.
Locker *locker.Locker
Plan *terraform.Plan
Plan plan.Plan
// if true, we skip approval checks for deploy, destroy resources and delete
// files

22
bundle/deploy/files/delete.go
View File

@ -12,7 +12,6 @@ import (
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/sync"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/fatih/color"
)
type delete struct{}
@ -22,24 +21,7 @@ func (m *delete) Name() string {
}
func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// Do not delete files if terraform destroy was not consented
if !b.Plan.IsEmpty && !b.Plan.ConfirmApply {
return nil
}
cmdio.LogString(ctx, "Starting deletion of remote bundle files")
cmdio.LogString(ctx, fmt.Sprintf("Bundle remote directory is %s", b.Config.Workspace.RootPath))
red := color.New(color.FgRed).SprintFunc()
if !b.AutoApprove {
proceed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\n%s and all files in it will be %s Proceed?", b.Config.Workspace.RootPath, red("deleted permanently!")))
if err != nil {
return diag.FromErr(err)
}
if !proceed {
return nil
}
}
cmdio.LogString(ctx, "Deleting files...")
err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{
Path: b.Config.Workspace.RootPath,
@ -54,8 +36,6 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
if err != nil {
return diag.FromErr(err)
}
cmdio.LogString(ctx, "Successfully deleted files!")
return nil
}

45
bundle/deploy/terraform/apply.go
View File

@ -1,45 +0,0 @@
package terraform
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
"github.com/hashicorp/terraform-exec/tfexec"
)
type apply struct{}
func (w *apply) Name() string {
return "terraform.Apply"
}
func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
cmdio.LogString(ctx, "Deploying resources...")
err := tf.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return diag.Errorf("terraform init: %v", err)
}
err = tf.Apply(ctx)
if err != nil {
return diag.Errorf("terraform apply: %v", err)
}
log.Infof(ctx, "Resource deployment completed")
return nil
}
// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply`
// from the bundle's ephemeral working directory for Terraform.
func Apply() bundle.Mutator {
return &apply{}
}

42
bundle/deploy/terraform/deploy.go Normal file
View File

@ -0,0 +1,42 @@
package terraform
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
)
// TODO: Release lock on error?
type tfDeploy struct{}
func (w *tfDeploy) Name() string {
return "terraform.Deploy"
}
// TODO: Document in PR why the approval logic is being moved to the plan mutator.
func (w *tfDeploy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// return early if plan is empty
if b.Plan.IsEmpty() {
cmdio.LogString(ctx, "No changes to deploy...")
return nil
}
cmdio.LogString(ctx, "Deploying resources...")
err := b.Plan.Apply()
if err != nil {
return diag.Errorf("terraform apply: %v", err)
}
log.Infof(ctx, "Resource deployment completed")
return nil
}
// Deploy returns a [bundle.Mutator] that runs the equivalent of `terraform apply`
// from the bundle's ephemeral working directory for Terraform.
func Deploy() bundle.Mutator {
return &tfDeploy{}
}

97
bundle/deploy/terraform/destroy.go
View File

@ -2,61 +2,12 @@ package terraform
import (
"context"
"fmt"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/fatih/color"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
)
type PlanResourceChange struct {
ResourceType string `json:"resource_type"`
Action string `json:"action"`
ResourceName string `json:"resource_name"`
}
func (c *PlanResourceChange) String() string {
result := strings.Builder{}
switch c.Action {
case "delete":
result.WriteString(" delete ")
default:
result.WriteString(c.Action + " ")
}
switch c.ResourceType {
case "databricks_job":
result.WriteString("job ")
case "databricks_pipeline":
result.WriteString("pipeline ")
default:
result.WriteString(c.ResourceType + " ")
}
result.WriteString(c.ResourceName)
return result.String()
}
func (c *PlanResourceChange) IsInplaceSupported() bool {
return false
}
func logDestroyPlan(ctx context.Context, changes []*tfjson.ResourceChange) error {
cmdio.LogString(ctx, "The following resources will be removed:")
for _, c := range changes {
if c.Change.Actions.Delete() {
cmdio.Log(ctx, &PlanResourceChange{
ResourceType: c.Type,
Action: "delete",
ResourceName: c.Name,
})
}
}
return nil
}
type destroy struct{}
func (w *destroy) Name() string {
@ -65,55 +16,17 @@ func (w *destroy) Name() string {
func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// return early if plan is empty
if b.Plan.IsEmpty {
cmdio.LogString(ctx, "No resources to destroy in plan. Skipping destroy!")
if b.Plan.IsEmpty() {
cmdio.LogString(ctx, "No resources to destroy...")
return nil
}
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
cmdio.LogString(ctx, "Destroying resources...")
// read plan file
plan, err := tf.ShowPlanFile(ctx, b.Plan.Path)
err := b.Plan.Apply()
if err != nil {
return diag.FromErr(err)
return diag.Errorf("terraform apply: %v", err)
}
// print the resources that will be destroyed
err = logDestroyPlan(ctx, plan.ResourceChanges)
if err != nil {
return diag.FromErr(err)
}
// Ask for confirmation, if needed
if !b.Plan.ConfirmApply {
red := color.New(color.FgRed).SprintFunc()
b.Plan.ConfirmApply, err = cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nThis will permanently %s resources! Proceed?", red("destroy")))
if err != nil {
return diag.FromErr(err)
}
}
// return if confirmation was not provided
if !b.Plan.ConfirmApply {
return nil
}
if b.Plan.Path == "" {
return diag.Errorf("no plan found")
}
cmdio.LogString(ctx, "Starting to destroy resources")
// Apply terraform according to the computed destroy plan
err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path))
if err != nil {
return diag.Errorf("terraform destroy: %v", err)
}
cmdio.LogString(ctx, "Successfully destroyed resources!")
return nil
}

73
bundle/deploy/terraform/plan.go
View File

@ -1,73 +0,0 @@
package terraform
import (
"context"
"fmt"
"path/filepath"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/terraform"
"github.com/hashicorp/terraform-exec/tfexec"
)
type PlanGoal string
var (
PlanDeploy = PlanGoal("deploy")
PlanDestroy = PlanGoal("destroy")
)
type plan struct {
goal PlanGoal
}
func (p *plan) Name() string {
return "terraform.Plan"
}
func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
cmdio.LogString(ctx, "Starting plan computation")
err := tf.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return diag.Errorf("terraform init: %v", err)
}
// Persist computed plan
tfDir, err := Dir(ctx, b)
if err != nil {
return diag.FromErr(err)
}
planPath := filepath.Join(tfDir, "plan")
destroy := p.goal == PlanDestroy
notEmpty, err := tf.Plan(ctx, tfexec.Destroy(destroy), tfexec.Out(planPath))
if err != nil {
return diag.FromErr(err)
}
// Set plan in main bundle struct for downstream mutators
b.Plan = &terraform.Plan{
Path: planPath,
ConfirmApply: b.AutoApprove,
IsEmpty: !notEmpty,
}
cmdio.LogString(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath))
return nil
}
// Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
// from the bundle's ephemeral working directory for Terraform.
func Plan(goal PlanGoal) bundle.Mutator {
return &plan{
goal: goal,
}
}

78
bundle/deploy/terraform/plan_deploy.go Normal file
View File

@ -0,0 +1,78 @@
package terraform
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/plan"
)
type planDeploy struct{}
func (p *planDeploy) Name() string {
return "terraform.PlanDeploy"
}
// TODO: We need end to end tests for the approval flow.
func (p *planDeploy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
// Get terraform project root.
tfDir, err := Dir(ctx, b)
if err != nil {
return diag.FromErr(err)
}
// Set plan in main bundle struct for downstream mutators
b.Plan, err = plan.NewTerraformPlan(ctx, plan.TerraformPlanOpts{
Executable: tf,
Root: tfDir,
IsDestroy: false,
})
if err != nil {
return diag.FromErr(err)
}
// If the `--auto-approve` flag is set, we don't need to ask for approval.
if b.AutoApprove {
return nil
}
destructiveActions := b.Plan.ActionsByTypes(plan.ActionTypeRecreate, plan.ActionTypeDelete)
// If there are no deletes or recreates, we don't need to ask for approval.
if len(destructiveActions) == 0 {
return nil
}
// display information the user needs to know about the deploy plan.
cmdio.LogString(ctx, "The following resources will be deleted or recreated:")
for _, a := range destructiveActions {
cmdio.Log(ctx, a)
}
approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?")
if err != nil {
return diag.FromErr(err)
}
// TODO: Remove the previous flag for deployment in the bundle tree?
if !approved {
// We error here to terminate the control flow and prevent the current
// process from modifying any deployment state.
return diag.Errorf("deployment terminated. No changes were made.")
}
return nil
}
// PlanDeploy returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
// from the bundle's ephemeral working directory for Terraform.
func PlanDeploy() bundle.Mutator {
return &planDeploy{}
}

86
bundle/deploy/terraform/plan_destroy.go Normal file
View File

@ -0,0 +1,86 @@
package terraform
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/plan"
)
type planDestroy struct{}
func (p *planDestroy) Name() string {
return "terraform.PlanDestroy"
}
// TODO: We need end to end tests for the approval flow.
func (p *planDestroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
// Get terraform project root.
tfDir, err := Dir(ctx, b)
if err != nil {
return diag.FromErr(err)
}
b.Plan, err = plan.NewTerraformPlan(ctx, plan.TerraformPlanOpts{
Executable: tf,
Root: tfDir,
IsDestroy: true,
})
// TODO: Note this PR also makes destroy idempotent.
// Display information needs to know about the destroy plan.
deleteActions := b.Plan.ActionsByTypes(plan.ActionTypeDelete)
// If there are delete actions, show that information to the user.
if len(deleteActions) != 0 {
cmdio.LogString(ctx, "The following resources will be deleted:")
for _, a := range deleteActions {
cmdio.Log(ctx, a)
}
}
// TODO: write e2e integration tests / unit tests to check this property.
// TODO: Also test this manually.
f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.RootPath)
if err != nil {
return diag.FromErr(err)
}
// If the root path exists, show warning.
if _, err := f.Stat(ctx, ""); err == nil {
cmdio.LogString(ctx, fmt.Sprintf("All files and directories at %s will be deleted.", b.Config.Workspace.RootPath))
}
// If the `--auto-approve` flag is set, we don't need to ask for approval.
if b.AutoApprove {
return nil
}
approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?")
if err != nil {
return diag.FromErr(err)
}
// TODO: Remove the previous flag for deployment in the bundle tree?
if !approved {
return diag.Errorf("destroy terminated. No changes were made.")
}
return nil
}
// Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan`
// from the bundle's ephemeral working directory for Terraform.
func PlanDestroy() bundle.Mutator {
return &planDestroy{}
}

3
bundle/phases/deploy.go
View File

@ -37,8 +37,9 @@ func Deploy() bundle.Mutator {
terraform.Interpolate(),
terraform.Write(),
terraform.CheckRunningResource(),
terraform.PlanDeploy(),
bundle.Defer(
terraform.Apply(),
terraform.Deploy(),
bundle.Seq(
terraform.StatePush(),
terraform.Load(),

2
bundle/phases/destroy.go
View File

@ -17,7 +17,7 @@ func Destroy() bundle.Mutator {
terraform.StatePull(),
terraform.Interpolate(),
terraform.Write(),
terraform.Plan(terraform.PlanGoal("destroy")),
terraform.PlanDestroy(),
terraform.Destroy(),
terraform.StatePush(),
files.Delete(),

43
libs/plan/plan.go Normal file
View File

@ -0,0 +1,43 @@
package plan
import "strings"
type Plan interface {
// Path where the plan is persisted
// Path() string
// Get all actions of the given types
ActionsByTypes(...ActionType) []Action
// Whether the plan is empty, i.e. no actions
IsEmpty() bool
// Apply the plan, changing the underlying resources.
Apply() error
}
type Action struct {
rtype string
rname string
atype ActionType
}
func (a Action) String() string {
return strings.Join([]string{" ", string(a.atype), a.rtype, a.rname}, " ")
}
func (c Action) IsInplaceSupported() bool {
return false
}
type ActionType string
const (
ActionTypeCreate ActionType = "create"
ActionTypeDelete ActionType = "delete"
ActionTypeUpdate ActionType = "update"
ActionTypeNoOp ActionType = "no-op"
ActionTypeRead ActionType = "read"
ActionTypeRecreate ActionType = "recreate"
)

138
libs/plan/terraform.go Normal file
View File

@ -0,0 +1,138 @@
package plan
import (
"context"
"fmt"
"path/filepath"
"slices"
"sort"
"strings"
"github.com/databricks/cli/libs/log"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
)
const planDirName = "plan"
// TODO: start using this. Also clean up the fields in the struct below.
type TerraformPlanOpts struct {
// If true, the plan will be a IsDestroy plan.
IsDestroy bool
// Path where the terraform project we'll compute the plan for is rooted at.
Root string
// Executable executable to compute the plan with.
Executable *tfexec.Terraform
}
// TODO: Remove any unnecessary fields in here.
type terraformPlan struct {
ctx context.Context
opts TerraformPlanOpts
// Path to the computed plan.
planPath string
// In memory representation of the computed plan, obtained by running terraform show.
tfPlan *tfjson.Plan
// TODO: Can we invert this?
notEmpty bool
}
// NewTerraformPlan creates a new Terraform plan for a terraform project rooted
// at the given path. If destroy is true, the plan will be a destroy plan.
func NewTerraformPlan(ctx context.Context, opts TerraformPlanOpts) (Plan, error) {
plan := &terraformPlan{
ctx: ctx,
opts: opts,
}
if plan.opts.Executable == nil {
return nil, fmt.Errorf("terraform executable not provided")
}
if plan.opts.Root == "" {
return nil, fmt.Errorf("root path to compute the terraform plan not provided")
}
// Initialize the terraform project
err := opts.Executable.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return nil, fmt.Errorf("terraform init: %w", err)
}
// Path where the computed plan will be persisted.
planPath := filepath.Join(opts.Root, planDirName)
log.Debugf(ctx, "Computing terraform plan")
plan.notEmpty, err = opts.Executable.Plan(ctx, tfexec.Destroy(opts.IsDestroy), tfexec.Out(planPath))
if err != nil {
return nil, fmt.Errorf("terraform plan: %w", err)
}
log.Debugf(ctx, "Loading terraform plan by running terraform show")
plan.tfPlan, err = opts.Executable.ShowPlanFile(ctx, planPath)
if err != nil {
return nil, fmt.Errorf("terraform show: %w", err)
}
// Persist the plan path
plan.planPath = planPath
return plan, nil
}
func (p *terraformPlan) Path() string {
return p.planPath
}
// TODO: Add tests for this functionality.
func (p *terraformPlan) ActionsByTypes(query ...ActionType) []Action {
actions := make([]Action, 0)
for _, rc := range p.tfPlan.ResourceChanges {
tfActions := rc.Change.Actions
switch {
case slices.Contains(query, ActionTypeDelete) && tfActions.Delete():
actions = append(actions, Action{
rtype: strings.TrimPrefix(rc.Type, "databricks_"),
rname: rc.Name,
atype: ActionTypeDelete,
})
case slices.Contains(query, ActionTypeRecreate) && tfActions.Replace():
actions = append(actions, Action{
rtype: strings.TrimPrefix(rc.Type, "databricks_"),
rname: rc.Name,
atype: ActionTypeRecreate,
})
default:
// We don't need to track other action types yet.
}
}
// Sort by action type first, then by resource type, and finally by resource name
sort.Slice(actions, func(i, j int) bool {
if actions[i].atype != actions[j].atype {
return actions[i].atype < actions[j].atype
}
if actions[i].rtype != actions[j].rtype {
return actions[i].rtype < actions[j].rtype
}
return actions[i].rname < actions[j].rname
})
return actions
}
func (p *terraformPlan) IsEmpty() bool {
return !p.notEmpty
}
func (p *terraformPlan) Apply() error {
err := p.opts.Executable.Apply(p.ctx, tfexec.DirOrPlan(p.planPath))
if err != nil {
return fmt.Errorf("terraform apply: %w", err)
}
return nil
}

13
libs/terraform/plan.go
View File

@ -1,13 +0,0 @@
package terraform
type Plan struct {
// Path to the plan
Path string
// Holds whether the user can consented to destruction. Either by interactive
// confirmation or by passing a command line flag
ConfirmApply bool
// If true, the plan is empty and applying it will not do anything
IsEmpty bool
}