Added ability for deferred mutator execution (#380)

## Changes
Added `DeferredMutator` and `bundle.Defer` function which allows to
always execute some mutators either in the end of execution chain or
after error occurs in the middle of execution chain.

Usage as follows:

```
deferredMutator := bundle.Defer([]bundle.Mutator{
    lock.Acquire()
    transform.DoSomething(),
    //...
}, []bundle.Mutator{
    lock.Release(),
})
```
In such case `lock.Release()` will always be executed: either when all
operations above succeed or when any of them fails

## Tests
Before the change

```
andrew.nester@HFW9Y94129 multiples-tasks % bricks bundle deploy
Starting upload of bundle files
Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/simple-task/development/files!

Error: terraform not initialized
andrew.nester@HFW9Y94129 multiples-tasks % bricks bundle deploy
Error: deploy lock acquired by andrew.nester@databricks.com at 2023-05-10 16:41:22.902659 +0200 CEST. Use --force to override

```

After the change
```
andrew.nester@HFW9Y94129 multiples-tasks % bricks bundle deploy 
Starting upload of bundle files
Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/simple-task/development/files!

Error: terraform not initialized
andrew.nester@HFW9Y94129 multiples-tasks % bricks bundle deploy
Starting upload of bundle files
Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/simple-task/development/files!

Error: terraform not initialized
```

bundle/deferred.go

@@ -0,0 +1,35 @@
+package bundle
+
+import (
+	"context"
+
+	"github.com/databricks/bricks/libs/errs"
+)
+
+type DeferredMutator struct {
+	mutators []Mutator
+	finally  []Mutator
+}
+
+func (d *DeferredMutator) Name() string {
+	return "deferred"
+}
+
+func Defer(mutators []Mutator, finally []Mutator) []Mutator {
+	return []Mutator{
+		&DeferredMutator{
+			mutators: mutators,
+			finally:  finally,
+		},
+	}
+}
+
+func (d *DeferredMutator) Apply(ctx context.Context, b *Bundle) ([]Mutator, error) {
+	mainErr := Apply(ctx, b, d.mutators)
+	errOnFinish := Apply(ctx, b, d.finally)
+	if mainErr != nil || errOnFinish != nil {
+		return nil, errs.FromMany(mainErr, errOnFinish)
+	}
+
+	return nil, nil
+}

bundle/deferred_test.go

@@ -0,0 +1,108 @@
+package bundle
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mutatorWithError struct {
+	applyCalled int
+	errorMsg    string
+}
+
+func (t *mutatorWithError) Name() string {
+	return "mutatorWithError"
+}
+
+func (t *mutatorWithError) Apply(_ context.Context, b *Bundle) ([]Mutator, error) {
+	t.applyCalled++
+	return nil, fmt.Errorf(t.errorMsg)
+}
+
+func TestDeferredMutatorWhenAllMutatorsSucceed(t *testing.T) {
+	m1 := &testMutator{}
+	m2 := &testMutator{}
+	m3 := &testMutator{}
+	cleanup := &testMutator{}
+	deferredMutator := Defer([]Mutator{m1, m2, m3}, []Mutator{cleanup})
+
+	bundle := &Bundle{}
+	err := Apply(context.Background(), bundle, deferredMutator)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1, m1.applyCalled)
+	assert.Equal(t, 1, m2.applyCalled)
+	assert.Equal(t, 1, m3.applyCalled)
+	assert.Equal(t, 1, cleanup.applyCalled)
+}
+
+func TestDeferredMutatorWhenFirstFails(t *testing.T) {
+	m1 := &testMutator{}
+	m2 := &testMutator{}
+	mErr := &mutatorWithError{errorMsg: "mutator error occurred"}
+	cleanup := &testMutator{}
+	deferredMutator := Defer([]Mutator{mErr, m1, m2}, []Mutator{cleanup})
+
+	bundle := &Bundle{}
+	err := Apply(context.Background(), bundle, deferredMutator)
+	assert.ErrorContains(t, err, "mutator error occurred")
+
+	assert.Equal(t, 1, mErr.applyCalled)
+	assert.Equal(t, 0, m1.applyCalled)
+	assert.Equal(t, 0, m2.applyCalled)
+	assert.Equal(t, 1, cleanup.applyCalled)
+}
+
+func TestDeferredMutatorWhenMiddleOneFails(t *testing.T) {
+	m1 := &testMutator{}
+	m2 := &testMutator{}
+	mErr := &mutatorWithError{errorMsg: "mutator error occurred"}
+	cleanup := &testMutator{}
+	deferredMutator := Defer([]Mutator{m1, mErr, m2}, []Mutator{cleanup})
+
+	bundle := &Bundle{}
+	err := Apply(context.Background(), bundle, deferredMutator)
+	assert.ErrorContains(t, err, "mutator error occurred")
+
+	assert.Equal(t, 1, m1.applyCalled)
+	assert.Equal(t, 1, mErr.applyCalled)
+	assert.Equal(t, 0, m2.applyCalled)
+	assert.Equal(t, 1, cleanup.applyCalled)
+}
+
+func TestDeferredMutatorWhenLastOneFails(t *testing.T) {
+	m1 := &testMutator{}
+	m2 := &testMutator{}
+	mErr := &mutatorWithError{errorMsg: "mutator error occurred"}
+	cleanup := &testMutator{}
+	deferredMutator := Defer([]Mutator{m1, m2, mErr}, []Mutator{cleanup})
+
+	bundle := &Bundle{}
+	err := Apply(context.Background(), bundle, deferredMutator)
+	assert.ErrorContains(t, err, "mutator error occurred")
+
+	assert.Equal(t, 1, m1.applyCalled)
+	assert.Equal(t, 1, m2.applyCalled)
+	assert.Equal(t, 1, mErr.applyCalled)
+	assert.Equal(t, 1, cleanup.applyCalled)
+}
+
+func TestDeferredMutatorCombinesErrorMessages(t *testing.T) {
+	m1 := &testMutator{}
+	m2 := &testMutator{}
+	mErr := &mutatorWithError{errorMsg: "mutator error occurred"}
+	cleanupErr := &mutatorWithError{errorMsg: "cleanup error occurred"}
+	deferredMutator := Defer([]Mutator{m1, m2, mErr}, []Mutator{cleanupErr})
+
+	bundle := &Bundle{}
+	err := Apply(context.Background(), bundle, deferredMutator)
+	assert.ErrorContains(t, err, "mutator error occurred\ncleanup error occurred")
+
+	assert.Equal(t, 1, m1.applyCalled)
+	assert.Equal(t, 1, m2.applyCalled)
+	assert.Equal(t, 1, mErr.applyCalled)
+	assert.Equal(t, 1, cleanupErr.applyCalled)
+}

bundle/phases/deploy.go

@@ -10,18 +10,21 @@ import (
 
 // The deploy phase deploys artifacts and resources.
 func Deploy() bundle.Mutator {
+	deployPhase := bundle.Defer([]bundle.Mutator{
+		lock.Acquire(),
+		files.Upload(),
+		artifacts.UploadAll(),
+		terraform.Interpolate(),
+		terraform.Write(),
+		terraform.StatePull(),
+		terraform.Apply(),
+		terraform.StatePush(),
+	}, []bundle.Mutator{
+		lock.Release(),
+	})
+
 	return newPhase(
 		"deploy",
-		[]bundle.Mutator{
-			lock.Acquire(),
-			files.Upload(),
-			artifacts.UploadAll(),
-			terraform.Interpolate(),
-			terraform.Write(),
-			terraform.StatePull(),
-			terraform.Apply(),
-			terraform.StatePush(),
-			lock.Release(),
-		},
+		deployPhase,
 	)
 }

bundle/phases/destroy.go

@@ -9,16 +9,19 @@ import (
 
 // The destroy phase deletes artifacts and resources.
 func Destroy() bundle.Mutator {
+	destroyPhase := bundle.Defer([]bundle.Mutator{
+		lock.Acquire(),
+		terraform.StatePull(),
+		terraform.Plan(terraform.PlanGoal("destroy")),
+		terraform.Destroy(),
+		terraform.StatePush(),
+		files.Delete(),
+	}, []bundle.Mutator{
+		lock.Release(),
+	})
+
 	return newPhase(
 		"destroy",
-		[]bundle.Mutator{
-			lock.Acquire(),
-			terraform.StatePull(),
-			terraform.Plan(terraform.PlanGoal("destroy")),
-			terraform.Destroy(),
-			terraform.StatePush(),
-			lock.Release(),
-			files.Delete(),
-		},
+		destroyPhase,
 	)
 }

libs/errs/aggregate.go

@@ -0,0 +1,68 @@
+package errs
+
+import "errors"
+
+type aggregateError struct {
+	errors []error
+}
+
+func FromMany(errors ...error) error {
+	n := 0
+	for _, err := range errors {
+		if err != nil {
+			n++
+		}
+	}
+	if n == 0 {
+		return nil
+	}
+	aggregateErr := &aggregateError{
+		errors: make([]error, 0, n),
+	}
+	for _, err := range errors {
+		if err != nil {
+			aggregateErr.errors = append(aggregateErr.errors, err)
+		}
+	}
+	return aggregateErr
+}
+
+func (ce *aggregateError) Error() string {
+	var b []byte
+	for i, err := range ce.errors {
+		if i > 0 {
+			b = append(b, '\n')
+		}
+		b = append(b, err.Error()...)
+	}
+	return string(b)
+}
+
+func (ce *aggregateError) Unwrap() error {
+	return errorChain(ce.errors)
+}
+
+// Represents chained list of errors.
+// Implements Error interface so that chain of errors
+// can correctly work with errors.Is/As method
+type errorChain []error
+
+func (ec errorChain) Error() string {
+	return ec[0].Error()
+}
+
+func (ec errorChain) Unwrap() error {
+	if len(ec) == 1 {
+		return nil
+	}
+
+	return ec[1:]
+}
+
+func (ec errorChain) As(target interface{}) bool {
+	return errors.As(ec[0], target)
+}
+
+func (ec errorChain) Is(target error) bool {
+	return errors.Is(ec[0], target)
+}

libs/errs/aggregate_test.go

@@ -0,0 +1,37 @@
+package errs
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFromManyErrors(t *testing.T) {
+	e1 := fmt.Errorf("Error 1")
+	e2 := fmt.Errorf("Error 2")
+	e3 := fmt.Errorf("Error 3")
+	err := FromMany(e1, e2, e3)
+
+	assert.True(t, errors.Is(err, e1))
+	assert.True(t, errors.Is(err, e2))
+	assert.True(t, errors.Is(err, e3))
+
+	assert.Equal(t, err.Error(), `Error 1
+Error 2
+Error 3`)
+}
+
+func TestFromManyErrorsWihtNil(t *testing.T) {
+	e1 := fmt.Errorf("Error 1")
+	var e2 error = nil
+	e3 := fmt.Errorf("Error 3")
+	err := FromMany(e1, e2, e3)
+
+	assert.True(t, errors.Is(err, e1))
+	assert.True(t, errors.Is(err, e3))
+
+	assert.Equal(t, err.Error(), `Error 1
+Error 3`)
+}