Added support for experimental scripts section (#632)

## Changes
Added support for experimental scripts section

It allows execution of arbitrary bash commands during certain bundle
lifecycle steps.

## Tests
Example of configuration

```yaml
bundle:
  name: wheel-task


workspace:
  host: ***

experimental:
  scripts:
    prebuild: |
      echo 'Prebuild 1'
      echo 'Prebuild 2'
    postbuild: "echo 'Postbuild 1' && echo 'Postbuild 2'" 
    predeploy: |
      echo 'Checking go version...'
      go version
    postdeploy: |
      echo 'Checking python version...'
      python --version

resources:
  jobs:
    test_job:
      name: "[${bundle.environment}] My Wheel Job"
      tasks:
        - task_key: TestTask
          existing_cluster_id: "***"
          python_wheel_task:
            package_name: "my_test_code"
            entry_point: "run"
          libraries:
          - whl: ./dist/*.whl
```

Output
```bash
andrew.nester@HFW9Y94129 wheel % databricks bundle deploy
artifacts.whl.AutoDetect: Detecting Python wheel project...
artifacts.whl.AutoDetect: Found Python wheel project at /Users/andrew.nester/dabs/wheel
'Prebuild 1'
'Prebuild 2'

artifacts.whl.Build(my_test_code): Building...
artifacts.whl.Build(my_test_code): Build succeeded
'Postbuild 1'
'Postbuild 2'

'Checking go version...'
go version go1.19.9 darwin/arm64

Starting upload of bundle files
Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/wheel-task/default/files!

artifacts.Upload(my_test_code-0.0.0a0-py3-none-any.whl): Uploading...
artifacts.Upload(my_test_code-0.0.0a0-py3-none-any.whl): Upload succeeded
Starting resource deployment
Resource deployment completed!
'Checking python version...'
Python 2.7.18
```

bundle/config/experimental.go

@@ -0,0 +1,18 @@
+package config
+
+type Experimental struct {
+	Scripts map[ScriptHook]Command `json:"scripts,omitempty"`
+}
+
+type Command string
+type ScriptHook string
+
+// These hook names are subject to change and currently experimental
+const (
+	ScriptPreInit    ScriptHook = "preinit"
+	ScriptPostInit   ScriptHook = "postinit"
+	ScriptPreBuild   ScriptHook = "prebuild"
+	ScriptPostBuild  ScriptHook = "postbuild"
+	ScriptPreDeploy  ScriptHook = "predeploy"
+	ScriptPostDeploy ScriptHook = "postdeploy"
+)

bundle/config/mutator/mutator.go

@@ -2,10 +2,13 @@ package mutator
 
 import (
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/scripts"
 )
 
 func DefaultMutators() []bundle.Mutator {
 	return []bundle.Mutator{
+		scripts.Execute(config.ScriptPreInit),
 		ProcessRootIncludes(),
 		DefineDefaultTarget(),
 		LoadGitDetails(),

bundle/config/root.go

@@ -84,6 +84,8 @@ type Root struct {
 
 	// RunAs section allows to define an execution identity for jobs and pipelines runs
 	RunAs *jobs.JobRunAs `json:"run_as,omitempty"`
+
+	Experimental *Experimental `json:"experimental,omitempty"`
 }
 
 func Load(path string) (*Root, error) {

bundle/phases/build.go

@@ -3,7 +3,9 @@ package phases
 import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/interpolation"
+	"github.com/databricks/cli/bundle/scripts"
 )
 
 // The build phase builds artifacts.
@@ -11,9 +13,11 @@ func Build() bundle.Mutator {
 	return newPhase(
 		"build",
 		[]bundle.Mutator{
+			scripts.Execute(config.ScriptPreBuild),
 			artifacts.DetectPackages(),
 			artifacts.InferMissingProperties(),
 			artifacts.BuildAll(),
+			scripts.Execute(config.ScriptPostBuild),
 			interpolation.Interpolate(
 				interpolation.IncludeLookupsInPath("artifacts"),
 			),

bundle/phases/deploy.go

@@ -3,17 +3,20 @@ package phases
 import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/deploy/files"
 	"github.com/databricks/cli/bundle/deploy/lock"
 	"github.com/databricks/cli/bundle/deploy/terraform"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/bundle/python"
+	"github.com/databricks/cli/bundle/scripts"
 )
 
 // The deploy phase deploys artifacts and resources.
 func Deploy() bundle.Mutator {
 	deployMutator := bundle.Seq(
+		scripts.Execute(config.ScriptPreDeploy),
 		lock.Acquire(),
 		bundle.Defer(
 			bundle.Seq(
@@ -31,6 +34,7 @@ func Deploy() bundle.Mutator {
 			),
 			lock.Release(lock.GoalDeploy),
 		),
+		scripts.Execute(config.ScriptPostDeploy),
 	)
 
 	return newPhase(

bundle/phases/initialize.go

@@ -2,10 +2,12 @@ package phases
 
 import (
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/interpolation"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/bundle/deploy/terraform"
+	"github.com/databricks/cli/bundle/scripts"
 )
 
 // The initialize phase fills in defaults and connects to the workspace.
@@ -30,6 +32,7 @@ func Initialize() bundle.Mutator {
 			mutator.ProcessTargetMode(),
 			mutator.TranslatePaths(),
 			terraform.Initialize(),
+			scripts.Execute(config.ScriptPostInit),
 		},
 	)
 }

bundle/scripts/scripts.go

@@ -0,0 +1,91 @@
+package scripts
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/log"
+)
+
+func Execute(hook config.ScriptHook) bundle.Mutator {
+	return &script{
+		scriptHook: hook,
+	}
+}
+
+type script struct {
+	scriptHook config.ScriptHook
+}
+
+func (m *script) Name() string {
+	return fmt.Sprintf("scripts.%s", m.scriptHook)
+}
+
+func (m *script) Apply(ctx context.Context, b *bundle.Bundle) error {
+	cmd, out, err := executeHook(ctx, b, m.scriptHook)
+	if err != nil {
+		return err
+	}
+	if cmd == nil {
+		log.Debugf(ctx, "No script defined for %s, skipping", m.scriptHook)
+		return nil
+	}
+
+	cmdio.LogString(ctx, fmt.Sprintf("Executing '%s' script", m.scriptHook))
+
+	reader := bufio.NewReader(out)
+	line, err := reader.ReadString('\n')
+	for err == nil {
+		cmdio.LogString(ctx, strings.TrimSpace(line))
+		line, err = reader.ReadString('\n')
+	}
+
+	return cmd.Wait()
+}
+
+func executeHook(ctx context.Context, b *bundle.Bundle, hook config.ScriptHook) (*exec.Cmd, io.Reader, error) {
+	command := getCommmand(b, hook)
+	if command == "" {
+		return nil, nil, nil
+	}
+
+	interpreter, err := findInterpreter()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cmd := exec.CommandContext(ctx, interpreter, "-c", string(command))
+	cmd.Dir = b.Config.Path
+
+	outPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	errPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return cmd, io.MultiReader(outPipe, errPipe), cmd.Start()
+}
+
+func getCommmand(b *bundle.Bundle, hook config.ScriptHook) config.Command {
+	if b.Config.Experimental == nil || b.Config.Experimental.Scripts == nil {
+		return ""
+	}
+
+	return b.Config.Experimental.Scripts[hook]
+}
+
+func findInterpreter() (string, error) {
+	// At the moment we just return 'sh' on all platforms and use it to execute scripts
+	return "sh", nil
+}

bundle/scripts/scripts_test.go

@@ -0,0 +1,32 @@
+package scripts
+
+import (
+	"bufio"
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecutesHook(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				Scripts: map[config.ScriptHook]config.Command{
+					config.ScriptPreBuild: "echo 'Hello'",
+				},
+			},
+		},
+	}
+	_, out, err := executeHook(context.Background(), b, config.ScriptPreBuild)
+	require.NoError(t, err)
+
+	reader := bufio.NewReader(out)
+	line, err := reader.ReadString('\n')
+
+	require.NoError(t, err)
+	require.Equal(t, "Hello", strings.TrimSpace(line))
+}