Add 'experimental/python' support (#2052)

## Changes
Add `experimental/python` section replacing `experimental/pydabs`.

Add 2 new mutators into existing pipeline:
- `ApplyPythonMutator(load_resources)` - loads resources from Python
code
- `ApplyPythonMutator(apply_mutators)` - transforms existing resources
defined in Python/YAML

Example:
```yaml
experimental:
  python:
    resources:
    - "resources:load_resources"
    mutators:
    - "mutators:add_email_notifications"
```

## Tests
Unit tests and manually

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

bundle/config/experimental.go

@@ -27,9 +27,33 @@ type Experimental struct {
 	// PyDABs determines whether to load the 'databricks-pydabs' package.
 	//
 	// PyDABs allows to define bundle configuration using Python.
+	// PyDABs is deprecated use Python instead.
 	PyDABs PyDABs `json:"pydabs,omitempty"`
+
+	// Python configures loading of Python code defined with 'databricks-bundles' package.
+	Python Python `json:"python,omitempty"`
 }
 
+type Python struct {
+	// Resources contains a list of fully qualified function paths to load resources
+	// defined in Python code.
+	//
+	// Example: ["my_project.resources:load_resources"]
+	Resources []string `json:"resources"`
+
+	// Mutators contains a list of fully qualified function paths to mutator functions.
+	//
+	// Example: ["my_project.mutators:add_default_cluster"]
+	Mutators []string `json:"mutators"`
+
+	// VEnvPath is path to the virtual environment.
+	//
+	// If enabled, Python code will execute within this environment. If disabled,
+	// it defaults to using the Python interpreter available in the current shell.
+	VEnvPath string `json:"venv_path,omitempty"`
+}
+
+// PyDABs is deprecated use Python instead
 type PyDABs struct {
 	// Enabled is a flag to enable the feature.
 	Enabled bool `json:"enabled,omitempty"`

bundle/config/mutator/python/python_mutator.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strings"
 
 	"github.com/databricks/databricks-sdk-go/logger"
@@ -40,6 +41,8 @@ const (
 	// We also open for possibility of appending other sections of bundle configuration,
 	// for example, adding new variables. However, this is not supported yet, and CLI rejects
 	// such changes.
+	//
+	// Deprecated, left for backward-compatibility with PyDABs.
 	PythonMutatorPhaseLoad phase = "load"
 
 	// PythonMutatorPhaseInit is the phase after bundle configuration was loaded, and
@@ -59,7 +62,46 @@ const (
 	// PyDABs can output YAML containing references to variables, and CLI should resolve them.
 	//
 	// Existing resources can't be removed, and CLI rejects such changes.
+	//
+	// Deprecated, left for backward-compatibility with PyDABs.
 	PythonMutatorPhaseInit phase = "init"
+
+	// PythonMutatorPhaseLoadResources is the phase in which YAML configuration was loaded.
+	//
+	// At this stage, we execute Python code to load resources defined in Python.
+	//
+	// During this process, Python code can access:
+	// - selected deployment target
+	// - bundle variable values
+	// - variables provided through CLI argument or environment variables
+	//
+	// The following is not available:
+	// - variables referencing other variables are in unresolved format
+	//
+	// Python code can output YAML referencing variables, and CLI should resolve them.
+	//
+	// Existing resources can't be removed or modified, and CLI rejects such changes.
+	// While it's called 'load_resources', this phase is executed in 'init' phase of mutator pipeline.
+	PythonMutatorPhaseLoadResources phase = "load_resources"
+
+	// PythonMutatorPhaseApplyMutators is the phase in which resources defined in YAML or Python
+	// are already loaded.
+	//
+	// At this stage, we execute Python code to mutate resources defined in YAML or Python.
+	//
+	// During this process, Python code can access:
+	// - selected deployment target
+	// - bundle variable values
+	// - variables provided through CLI argument or environment variables
+	//
+	// The following is not available:
+	// - variables referencing other variables are in unresolved format
+	//
+	// Python code can output YAML referencing variables, and CLI should resolve them.
+	//
+	// Resources can't be added or removed, and CLI rejects such changes. Python code is
+	// allowed to modify existing resources, but not other parts of bundle configuration.
+	PythonMutatorPhaseApplyMutators phase = "apply_mutators"
 )
 
 type pythonMutator struct {
@@ -76,18 +118,64 @@ func (m *pythonMutator) Name() string {
 	return fmt.Sprintf("PythonMutator(%s)", m.phase)
 }
 
-func getExperimental(b *bundle.Bundle) config.Experimental {
+// opts is a common structure for deprecated PyDABs and upcoming Python
-	if b.Config.Experimental == nil {
+// configuration sections
-		return config.Experimental{}
+type opts struct {
+	enabled bool
+
+	venvPath string
 }
 
-	return *b.Config.Experimental
+// getOpts adapts deprecated PyDABs and upcoming Python configuration
+// into a common structure.
+func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
+	experimental := b.Config.Experimental
+	if experimental == nil {
+		return opts{}, nil
+	}
+
+	// using reflect.DeepEquals in case we add more fields
+	pydabsEnabled := !reflect.DeepEqual(experimental.PyDABs, config.PyDABs{})
+	pythonEnabled := !reflect.DeepEqual(experimental.Python, config.Python{})
+
+	if pydabsEnabled && pythonEnabled {
+		return opts{}, errors.New("both experimental/pydabs and experimental/python are enabled, only one can be enabled")
+	} else if pydabsEnabled {
+		if !experimental.PyDABs.Enabled {
+			return opts{}, nil
+		}
+
+		// don't execute for phases for 'python' section
+		if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad {
+			return opts{
+				enabled:  true,
+				venvPath: experimental.PyDABs.VEnvPath,
+			}, nil
+		} else {
+			return opts{}, nil
+		}
+	} else if pythonEnabled {
+		// don't execute for phases for 'pydabs' section
+		if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators {
+			return opts{
+				enabled:  true,
+				venvPath: experimental.Python.VEnvPath,
+			}, nil
+		} else {
+			return opts{}, nil
+		}
+	} else {
+		return opts{}, nil
+	}
 }
 
 func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	experimental := getExperimental(b)
+	opts, err := getOpts(b, m.phase)
+	if err != nil {
+		return diag.Errorf("failed to apply python mutator: %s", err)
+	}
 
-	if !experimental.PyDABs.Enabled {
+	if !opts.enabled {
 		return nil
 	}
 
@@ -95,8 +183,8 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 	var mutateDiags diag.Diagnostics
 	mutateDiagsHasError := errors.New("unexpected error")
 
-	err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
+	err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
-		pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
+		pythonPath, err := detectExecutable(ctx, opts.venvPath)
 		if err != nil {
 			return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
 		}
@@ -137,7 +225,7 @@ func createCacheDir(ctx context.Context) (string, error) {
 	// support the same env variable as in b.CacheDir
 	if tempDir, exists := env.TempDir(ctx); exists {
 		// use 'default' as target name
-		cacheDir := filepath.Join(tempDir, "default", "pydabs")
+		cacheDir := filepath.Join(tempDir, "default", "python")
 
 		err := os.MkdirAll(cacheDir, 0o700)
 		if err != nil {
@@ -147,7 +235,7 @@ func createCacheDir(ctx context.Context) (string, error) {
 		return cacheDir, nil
 	}
 
-	return os.MkdirTemp("", "-pydabs")
+	return os.MkdirTemp("", "-python")
 }
 
 func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
@@ -203,7 +291,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 	}
 
 	// process can fail without reporting errors in diagnostics file or creating it, for instance,
-	// venv doesn't have PyDABs library installed
+	// venv doesn't have 'databricks-bundles' library installed
 	if processErr != nil {
 		diagnostic := diag.Diagnostic{
 			Severity: diag.Error,
@@ -226,16 +314,15 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
 	return output, pythonDiagnostics
 }
 
-const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
+const pythonInstallExplanation = `Ensure that 'databricks-bundles' is installed in Python environment:
-and that the wheel is installed in the Python environment:
 
-  $ .venv/bin/pip install -e .
+  $ .venv/bin/pip install databricks-bundles
 
 If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:
-    pydabs:
+    python:
       venv_path: .venv
 `
 
@@ -245,9 +332,9 @@ or activate the environment before running CLI commands:
 func explainProcessErr(stderr string) string {
 	// implemented in cpython/Lib/runpy.py and portable across Python 3.x, including pypy
 	if strings.Contains(stderr, "Error while finding module specification for 'databricks.bundles.build'") {
-		summary := color.CyanString("Explanation: ") + "'databricks-pydabs' library is not installed in the Python environment.\n"
+		summary := color.CyanString("Explanation: ") + "'databricks-bundles' library is not installed in the Python environment.\n"
 
-		return stderr + "\n" + summary + "\n" + installExplanation
+		return stderr + "\n" + summary + "\n" + pythonInstallExplanation
 	}
 
 	return stderr
@@ -277,10 +364,10 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
 	//
 	// virtualPath has to stay in rootPath, because locations outside root path are not allowed:
 	//
-	//   Error: path /var/folders/.../pydabs/dist/*.whl is not contained in bundle root path
+	//   Error: path /var/folders/.../python/dist/*.whl is not contained in bundle root path
 	//
 	// for that, we pass virtualPath instead of outputPath as file location
-	virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_pydabs__.yml"))
+	virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_python__.yml"))
 	if err != nil {
 		return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err))
 	}
@@ -334,19 +421,23 @@ func loadDiagnosticsFile(path string) (diag.Diagnostics, error) {
 func createOverrideVisitor(ctx context.Context, phase phase) (merge.OverrideVisitor, error) {
 	switch phase {
 	case PythonMutatorPhaseLoad:
-		return createLoadOverrideVisitor(ctx), nil
+		return createLoadResourcesOverrideVisitor(ctx), nil
 	case PythonMutatorPhaseInit:
-		return createInitOverrideVisitor(ctx), nil
+		return createInitOverrideVisitor(ctx, insertResourceModeAllow), nil
+	case PythonMutatorPhaseLoadResources:
+		return createLoadResourcesOverrideVisitor(ctx), nil
+	case PythonMutatorPhaseApplyMutators:
+		return createInitOverrideVisitor(ctx, insertResourceModeDisallow), nil
 	default:
 		return merge.OverrideVisitor{}, fmt.Errorf("unknown phase: %s", phase)
 	}
 }
 
-// createLoadOverrideVisitor creates an override visitor for the load phase.
+// createLoadResourcesOverrideVisitor creates an override visitor for the load_resources phase.
 //
-// During load, it's only possible to create new resources, and not modify or
+// During load_resources, it's only possible to create new resources, and not modify or
 // delete existing ones.
-func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
+func createLoadResourcesOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 	resourcesPath := dyn.NewPath(dyn.Key("resources"))
 	jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
 
@@ -385,11 +476,21 @@ func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 	}
 }
 
+// insertResourceMode controls whether createInitOverrideVisitor allows or disallows inserting new resources.
+type insertResourceMode int
+
+const (
+	insertResourceModeDisallow insertResourceMode = iota
+	insertResourceModeAllow    insertResourceMode = iota
+)
+
 // createInitOverrideVisitor creates an override visitor for the init phase.
 //
 // During the init phase it's possible to create new resources, modify existing
 // resources, but not delete existing resources.
-func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
+//
+// If mode is insertResourceModeDisallow, it matching expected behaviour of apply_mutators
+func createInitOverrideVisitor(ctx context.Context, mode insertResourceMode) merge.OverrideVisitor {
 	resourcesPath := dyn.NewPath(dyn.Key("resources"))
 	jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
 
@@ -424,6 +525,11 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 				return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
 			}
 
+			insertResource := len(valuePath) == len(jobsPath)+1
+			if mode == insertResourceModeDisallow && insertResource {
+				return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
+			}
+
 			log.Debugf(ctx, "Insert value at %q", valuePath.String())
 
 			return right, nil
@@ -441,9 +547,9 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
 }
 
 func isOmitemptyDelete(left dyn.Value) bool {
-	// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
+	// Python output can omit empty sequences/mappings, because we don't track them as optional,
 	// there is no semantic difference between empty and missing, so we keep them as they were before
-	// PyDABs deleted them.
+	// Python mutator deleted them.
 
 	switch left.Kind() {
 	case dyn.KindMap:

bundle/config/mutator/python/python_mutator_test.go

@@ -40,13 +40,25 @@ func TestPythonMutator_Name_init(t *testing.T) {
 	assert.Equal(t, "PythonMutator(init)", mutator.Name())
 }
 
-func TestPythonMutator_load(t *testing.T) {
+func TestPythonMutator_Name_loadResources(t *testing.T) {
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
+
+	assert.Equal(t, "PythonMutator(load_resources)", mutator.Name())
+}
+
+func TestPythonMutator_Name_applyMutators(t *testing.T) {
+	mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
+
+	assert.Equal(t, "PythonMutator(apply_mutators)", mutator.Name())
+}
+
+func TestPythonMutator_loadResources(t *testing.T) {
 	withFakeVEnv(t, ".venv")
 
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
+        python:
-          enabled: true
+          resources: ["resources:load_resources"]
           venv_path: .venv
       resources:
         jobs:
@@ -60,12 +72,12 @@ func TestPythonMutator_load(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
+				"python": {
-					"enabled": true,
+					"resources": ["resources:load_resources"],
 					"venv_path": ".venv"
 				}
 			},
@@ -83,7 +95,7 @@ func TestPythonMutator_load(t *testing.T) {
 		`{"severity": "warning", "summary": "job doesn't have any tasks", "location": {"file": "src/examples/file.py", "line": 10, "column": 5}}`,
 	)
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diags := bundle.Apply(ctx, b, mutator)
 
 	assert.NoError(t, diags.Error())
@@ -109,13 +121,12 @@ func TestPythonMutator_load(t *testing.T) {
 	}, diags[0].Locations)
 }
 
-func TestPythonMutator_load_disallowed(t *testing.T) {
+func TestPythonMutator_loadResources_disallowed(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
+        python:
-          enabled: true
+          resources: ["resources:load_resources"]
           venv_path: .venv
       resources:
         jobs:
@@ -129,12 +140,12 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
+				"python": {
-					"enabled": true,
+					"resources": ["resources:load_resources"],
 					"venv_path": ".venv"
 				}
 			},
@@ -148,20 +159,20 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.EqualError(t, diag.Error(), "unexpected change at \"resources.jobs.job0.description\" (insert)")
 }
 
-func TestPythonMutator_init(t *testing.T) {
+func TestPythonMutator_applyMutators(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
+        python:
-          enabled: true
           venv_path: .venv
+          mutators:
+            - "mutators:add_description"
       resources:
         jobs:
           job0:
@@ -174,13 +185,13 @@ func TestPythonMutator_init(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"init",
+			"apply_mutators",
 		},
 		`{
 			"experimental": {
-				"pydabs": {
+				"python": {
-					"enabled": true,
+					"venv_path": ".venv",
-					"venv_path": ".venv"
+					"mutators": ["mutators:add_description"]
 				}
 			},
 			"resources": {
@@ -193,7 +204,7 @@ func TestPythonMutator_init(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseInit)
+	mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.NoError(t, diag.Error())
@@ -208,12 +219,12 @@ func TestPythonMutator_init(t *testing.T) {
 		require.NoError(t, err)
 		assert.Equal(t, "databricks.yml", name.Location().File)
 
-		// 'description' was updated by PyDABs and has location of generated file until
+		// 'description' was updated by Python code and has location of generated file until
 		// we implement source maps
 		description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description"))
 		require.NoError(t, err)
 
-		expectedVirtualPath, err := filepath.Abs("__generated_by_pydabs__.yml")
+		expectedVirtualPath, err := filepath.Abs("__generated_by_python__.yml")
 		require.NoError(t, err)
 		assert.Equal(t, expectedVirtualPath, description.Location().File)
 
@@ -224,12 +235,12 @@ func TestPythonMutator_init(t *testing.T) {
 
 func TestPythonMutator_badOutput(t *testing.T) {
 	withFakeVEnv(t, ".venv")
-
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
+        python:
-          enabled: true
           venv_path: .venv
+          resources:
+            - "resources:load_resources"
       resources:
         jobs:
           job0:
@@ -242,7 +253,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
 			"-m",
 			"databricks.bundles.build",
 			"--phase",
-			"load",
+			"load_resources",
 		},
 		`{
 			"resources": {
@@ -254,7 +265,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
 			}
 		}`, "")
 
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(ctx, b, mutator)
 
 	assert.EqualError(t, diag.Error(), "unknown field: unknown_property")
@@ -270,34 +281,63 @@ func TestPythonMutator_disabled(t *testing.T) {
 	assert.NoError(t, diag.Error())
 }
 
-func TestPythonMutator_venvRequired(t *testing.T) {
-	b := loadYaml("databricks.yml", `
-      experimental:
-        pydabs:
-          enabled: true`)
-
-	ctx := context.Background()
-	mutator := PythonMutator(PythonMutatorPhaseLoad)
-	diag := bundle.Apply(ctx, b, mutator)
-
-	assert.Error(t, diag.Error(), "\"experimental.enable_pydabs\" is enabled, but \"experimental.venv.path\" is not set")
-}
-
 func TestPythonMutator_venvNotFound(t *testing.T) {
 	expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path"))
 
 	b := loadYaml("databricks.yml", `
       experimental:
-        pydabs:
+        python:
-          enabled: true
+          venv_path: bad_path
-          venv_path: bad_path`)
+          resources:
+            - "resources:load_resources"`)
 
-	mutator := PythonMutator(PythonMutatorPhaseInit)
+	mutator := PythonMutator(PythonMutatorPhaseLoadResources)
 	diag := bundle.Apply(context.Background(), b, mutator)
 
 	assert.EqualError(t, diag.Error(), expectedError)
 }
 
+func TestGetOps_Python(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				Python: config.Python{
+					VEnvPath: ".venv",
+					Resources: []string{
+						"resources:load_resources",
+					},
+				},
+			},
+		},
+	}, PythonMutatorPhaseLoadResources)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+}
+
+func TestGetOps_PyDABs(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{
+		Config: config.Root{
+			Experimental: &config.Experimental{
+				PyDABs: config.PyDABs{
+					VEnvPath: ".venv",
+					Enabled:  true,
+				},
+			},
+		},
+	}, PythonMutatorPhaseInit)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
+}
+
+func TestGetOps_empty(t *testing.T) {
+	actual, err := getOpts(&bundle.Bundle{}, PythonMutatorPhaseLoadResources)
+
+	assert.NoError(t, err)
+	assert.Equal(t, opts{enabled: false}, actual)
+}
+
 type createOverrideVisitorTestCase struct {
 	name        string
 	updatePath  dyn.Path
@@ -315,8 +355,8 @@ func TestCreateOverrideVisitor(t *testing.T) {
 
 	testCases := []createOverrideVisitorTestCase{
 		{
-			name:        "load: can't change an existing job",
+			name:        "load_resources: can't change an existing job",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			updatePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0.name"),
 			insertPath:  dyn.MustPathFromString("resources.jobs.job0.name"),
@@ -325,32 +365,32 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			updateError: errors.New("unexpected change at \"resources.jobs.job0.name\" (update)"),
 		},
 		{
-			name:        "load: can't delete an existing job",
+			name:        "load_resources: can't delete an existing job",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
 			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
 		},
 		{
-			name:        "load: can insert 'resources'",
+			name:        "load_resources: can insert 'resources'",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can insert 'resources.jobs'",
+			name:        "load_resources: can insert 'resources.jobs'",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources.jobs"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can insert a job",
+			name:        "load_resources: can insert a job",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			insertPath:  dyn.MustPathFromString("resources.jobs.job0"),
 			insertError: nil,
 		},
 		{
-			name:        "load: can't change include",
+			name:        "load_resources: can't change include",
-			phase:       PythonMutatorPhaseLoad,
+			phase:       PythonMutatorPhaseLoadResources,
 			deletePath:  dyn.MustPathFromString("include[0]"),
 			insertPath:  dyn.MustPathFromString("include[0]"),
 			updatePath:  dyn.MustPathFromString("include[0]"),
@@ -402,6 +442,40 @@ func TestCreateOverrideVisitor(t *testing.T) {
 			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
 			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
 		},
+		{
+			name:        "apply_mutators: can't delete an existing job",
+			phase:       PythonMutatorPhaseInit,
+			deletePath:  dyn.MustPathFromString("resources.jobs.job0"),
+			deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
+		},
+		{
+			name:        "apply_mutators: can insert 'resources'",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources"),
+			insertError: nil,
+		},
+		{
+			name:        "apply_mutators: can insert 'resources.jobs'",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources.jobs"),
+			insertError: nil,
+		},
+		{
+			name:        "apply_mutators: can't insert a job",
+			phase:       PythonMutatorPhaseApplyMutators,
+			insertPath:  dyn.MustPathFromString("resources.jobs.job0"),
+			insertError: errors.New("unexpected change at \"resources.jobs.job0\" (insert)"),
+		},
+		{
+			name:        "apply_mutators: can't change include",
+			phase:       PythonMutatorPhaseApplyMutators,
+			deletePath:  dyn.MustPathFromString("include[0]"),
+			insertPath:  dyn.MustPathFromString("include[0]"),
+			updatePath:  dyn.MustPathFromString("include[0]"),
+			deleteError: errors.New("unexpected change at \"include[0]\" (delete)"),
+			insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
+			updateError: errors.New("unexpected change at \"include[0]\" (update)"),
+		},
 	}
 
 	for _, tc := range testCases {
@@ -459,9 +533,9 @@ type overrideVisitorOmitemptyTestCase struct {
 }
 
 func TestCreateOverrideVisitor_omitempty(t *testing.T) {
-	// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
+	// Python output can omit empty sequences/mappings in output, because we don't track them as optional,
 	// there is no semantic difference between empty and missing, so we keep them as they were before
-	// PyDABs deleted them.
+	// Python code deleted them.
 
 	allPhases := []phase{PythonMutatorPhaseLoad, PythonMutatorPhaseInit}
 	location := dyn.Location{
@@ -568,18 +642,17 @@ func TestExplainProcessErr(t *testing.T) {
 	stderr := "/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')\n"
 	expected := `/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')
 
-Explanation: 'databricks-pydabs' library is not installed in the Python environment.
+Explanation: 'databricks-bundles' library is not installed in the Python environment.
 
-If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
+Ensure that 'databricks-bundles' is installed in Python environment:
-and that the wheel is installed in the Python environment:
 
-  $ .venv/bin/pip install -e .
+  $ .venv/bin/pip install databricks-bundles
 
 If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
 or activate the environment before running CLI commands:
 
   experimental:
-    pydabs:
+    python:
       venv_path: .venv
 `
 

bundle/internal/schema/annotations.yml

@@ -69,6 +69,9 @@ github.com/databricks/cli/bundle/config.Experimental:
   "pydabs":
     "description": |-
       The PyDABs configuration.
+  "python":
+    "description": |-
+      Configures loading of Python code defined with 'databricks-bundles' package.
   "python_wheel_wrapper":
     "description": |-
       Whether to use a Python wheel wrapper
@@ -125,6 +128,24 @@ github.com/databricks/cli/bundle/config.PyDABs:
   "venv_path":
     "description": |-
       The Python virtual environment path
+github.com/databricks/cli/bundle/config.Python:
+  "mutators":
+    "description": |-
+      Mutators contains a list of fully qualified function paths to mutator functions.
+
+      Example: ["my_project.mutators:add_default_cluster"]
+  "resources":
+    "description": |-
+      Resources contains a list of fully qualified function paths to load resources
+      defined in Python code.
+
+      Example: ["my_project.resources:load_resources"]
+  "venv_path":
+    "description": |-
+      VEnvPath is path to the virtual environment.
+
+      If enabled, Python code will execute within this environment. If disabled,
+      it defaults to using the Python interpreter available in the current shell.
 github.com/databricks/cli/bundle/config.Resources:
   "clusters":
     "description": |-

bundle/phases/initialize.go

@@ -55,6 +55,8 @@ func Initialize() bundle.Mutator {
 			// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
 			// See what is expected in PythonMutatorPhaseInit doc
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
+			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoadResources),
+			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
 			mutator.ResolveVariableReferencesInLookup(),
 			mutator.ResolveResourceReferences(),
 			mutator.ResolveVariableReferencesInComplexVariables(),

bundle/schema/jsonschema.json

@@ -1100,6 +1100,10 @@
                       "description": "The PyDABs configuration.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs"
                     },
+                    "python": {
+                      "description": "Configures loading of Python code defined with 'databricks-bundles' package.",
+                      "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python"
+                    },
                     "python_wheel_wrapper": {
                       "description": "Whether to use a Python wheel wrapper",
                       "$ref": "#/$defs/bool"
@@ -1234,6 +1238,36 @@
                 }
               ]
             },
+            "config.Python": {
+              "oneOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "mutators": {
+                      "description": "Mutators contains a list of fully qualified function paths to mutator functions.\n\nExample: [\"my_project.mutators:add_default_cluster\"]",
+                      "$ref": "#/$defs/slice/string"
+                    },
+                    "resources": {
+                      "description": "Resources contains a list of fully qualified function paths to load resources\ndefined in Python code.\n\nExample: [\"my_project.resources:load_resources\"]",
+                      "$ref": "#/$defs/slice/string"
+                    },
+                    "venv_path": {
+                      "description": "VEnvPath is path to the virtual environment.\n\nIf enabled, Python code will execute within this environment. If disabled,\nit defaults to using the Python interpreter available in the current shell.",
+                      "$ref": "#/$defs/string"
+                    }
+                  },
+                  "additionalProperties": false,
+                  "required": [
+                    "resources",
+                    "mutators"
+                  ]
+                },
+                {
+                  "type": "string",
+                  "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
+                }
+              ]
+            },
             "config.Resources": {
               "oneOf": [
                 {