steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add 'experimental/python' support (#2052) 

## Changes
Add `experimental/python` section replacing `experimental/pydabs`.

Add 2 new mutators into existing pipeline:
- `ApplyPythonMutator(load_resources)` - loads resources from Python
code
- `ApplyPythonMutator(apply_mutators)` - transforms existing resources
defined in Python/YAML

Example:
```yaml
experimental:
  python:
    resources:
    - "resources:load_resources"
    mutators:
    - "mutators:add_email_notifications"
```

## Tests
Unit tests and manually

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
This commit is contained in:
Gleb Kanterov 2025-01-08 10:29:45 +01:00 committed by GitHub
parent 43420d01ad
commit 02c7df39f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 354 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bundle/config/experimental.go
View File

@ -27,9 +27,33 @@ type Experimental struct {
// PyDABs determines whether to load the 'databricks-pydabs' package.
//
// PyDABs allows to define bundle configuration using Python.
// PyDABs is deprecated use Python instead.
PyDABs PyDABs `json:"pydabs,omitempty"`
// Python configures loading of Python code defined with 'databricks-bundles' package.
Python Python `json:"python,omitempty"`
}
type Python struct {
// Resources contains a list of fully qualified function paths to load resources
// defined in Python code.
//
// Example: ["my_project.resources:load_resources"]
Resources []string `json:"resources"`
// Mutators contains a list of fully qualified function paths to mutator functions.
//
// Example: ["my_project.mutators:add_default_cluster"]
Mutators []string `json:"mutators"`
// VEnvPath is path to the virtual environment.
//
// If enabled, Python code will execute within this environment. If disabled,
// it defaults to using the Python interpreter available in the current shell.
VEnvPath string `json:"venv_path,omitempty"`
}
// PyDABs is deprecated use Python instead
type PyDABs struct {
// Enabled is a flag to enable the feature.
Enabled bool `json:"enabled,omitempty"`

160
bundle/config/mutator/python/python_mutator.go
View File

@ -9,6 +9,7 @@ import (
"io"
"os"
"path/filepath"
"reflect"
"strings"
"github.com/databricks/databricks-sdk-go/logger"
@ -40,6 +41,8 @@ const (
// We also open for possibility of appending other sections of bundle configuration,
// for example, adding new variables. However, this is not supported yet, and CLI rejects
// such changes.
//
// Deprecated, left for backward-compatibility with PyDABs.
PythonMutatorPhaseLoad phase = "load"
// PythonMutatorPhaseInit is the phase after bundle configuration was loaded, and
@ -59,7 +62,46 @@ const (
// PyDABs can output YAML containing references to variables, and CLI should resolve them.
//
// Existing resources can't be removed, and CLI rejects such changes.
//
// Deprecated, left for backward-compatibility with PyDABs.
PythonMutatorPhaseInit phase = "init"
// PythonMutatorPhaseLoadResources is the phase in which YAML configuration was loaded.
//
// At this stage, we execute Python code to load resources defined in Python.
//
// During this process, Python code can access:
// - selected deployment target
// - bundle variable values
// - variables provided through CLI argument or environment variables
//
// The following is not available:
// - variables referencing other variables are in unresolved format
//
// Python code can output YAML referencing variables, and CLI should resolve them.
//
// Existing resources can't be removed or modified, and CLI rejects such changes.
// While it's called 'load_resources', this phase is executed in 'init' phase of mutator pipeline.
PythonMutatorPhaseLoadResources phase = "load_resources"
// PythonMutatorPhaseApplyMutators is the phase in which resources defined in YAML or Python
// are already loaded.
//
// At this stage, we execute Python code to mutate resources defined in YAML or Python.
//
// During this process, Python code can access:
// - selected deployment target
// - bundle variable values
// - variables provided through CLI argument or environment variables
//
// The following is not available:
// - variables referencing other variables are in unresolved format
//
// Python code can output YAML referencing variables, and CLI should resolve them.
//
// Resources can't be added or removed, and CLI rejects such changes. Python code is
// allowed to modify existing resources, but not other parts of bundle configuration.
PythonMutatorPhaseApplyMutators phase = "apply_mutators"
)
type pythonMutator struct {
@ -76,18 +118,64 @@ func (m *pythonMutator) Name() string {
return fmt.Sprintf("PythonMutator(%s)", m.phase)
}
func getExperimental(b *bundle.Bundle) config.Experimental {
if b.Config.Experimental == nil {
return config.Experimental{}
// opts is a common structure for deprecated PyDABs and upcoming Python
// configuration sections
type opts struct {
enabled bool
venvPath string
}
// getOpts adapts deprecated PyDABs and upcoming Python configuration
// into a common structure.
func getOpts(b *bundle.Bundle, phase phase) (opts, error) {
experimental := b.Config.Experimental
if experimental == nil {
return opts{}, nil
}
return *b.Config.Experimental
// using reflect.DeepEquals in case we add more fields
pydabsEnabled := !reflect.DeepEqual(experimental.PyDABs, config.PyDABs{})
pythonEnabled := !reflect.DeepEqual(experimental.Python, config.Python{})
if pydabsEnabled && pythonEnabled {
return opts{}, errors.New("both experimental/pydabs and experimental/python are enabled, only one can be enabled")
} else if pydabsEnabled {
if !experimental.PyDABs.Enabled {
return opts{}, nil
}
// don't execute for phases for 'python' section
if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad {
return opts{
enabled: true,
venvPath: experimental.PyDABs.VEnvPath,
}, nil
} else {
return opts{}, nil
}
} else if pythonEnabled {
// don't execute for phases for 'pydabs' section
if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators {
return opts{
enabled: true,
venvPath: experimental.Python.VEnvPath,
}, nil
} else {
return opts{}, nil
}
} else {
return opts{}, nil
}
}
func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
experimental := getExperimental(b)
opts, err := getOpts(b, m.phase)
if err != nil {
return diag.Errorf("failed to apply python mutator: %s", err)
}
if !experimental.PyDABs.Enabled {
if !opts.enabled {
return nil
}
@ -95,8 +183,8 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
var mutateDiags diag.Diagnostics
mutateDiagsHasError := errors.New("unexpected error")
err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath)
err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) {
pythonPath, err := detectExecutable(ctx, opts.venvPath)
if err != nil {
return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err)
}
@ -137,7 +225,7 @@ func createCacheDir(ctx context.Context) (string, error) {
// support the same env variable as in b.CacheDir
if tempDir, exists := env.TempDir(ctx); exists {
// use 'default' as target name
cacheDir := filepath.Join(tempDir, "default", "pydabs")
cacheDir := filepath.Join(tempDir, "default", "python")
err := os.MkdirAll(cacheDir, 0o700)
if err != nil {
@ -147,7 +235,7 @@ func createCacheDir(ctx context.Context) (string, error) {
return cacheDir, nil
}
return os.MkdirTemp("", "-pydabs")
return os.MkdirTemp("", "-python")
}
func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) {
@ -203,7 +291,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
}
// process can fail without reporting errors in diagnostics file or creating it, for instance,
// venv doesn't have PyDABs library installed
// venv doesn't have 'databricks-bundles' library installed
if processErr != nil {
diagnostic := diag.Diagnostic{
Severity: diag.Error,
@ -226,16 +314,15 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath
return output, pythonDiagnostics
}
const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
and that the wheel is installed in the Python environment:
const pythonInstallExplanation = `Ensure that 'databricks-bundles' is installed in Python environment:
$ .venv/bin/pip install -e .
$ .venv/bin/pip install databricks-bundles
If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
or activate the environment before running CLI commands:
experimental:
pydabs:
python:
venv_path: .venv
`
@ -245,9 +332,9 @@ or activate the environment before running CLI commands:
func explainProcessErr(stderr string) string {
// implemented in cpython/Lib/runpy.py and portable across Python 3.x, including pypy
if strings.Contains(stderr, "Error while finding module specification for 'databricks.bundles.build'") {
summary := color.CyanString("Explanation: ") + "'databricks-pydabs' library is not installed in the Python environment.\n"
summary := color.CyanString("Explanation: ") + "'databricks-bundles' library is not installed in the Python environment.\n"
return stderr + "\n" + summary + "\n" + installExplanation
return stderr + "\n" + summary + "\n" + pythonInstallExplanation
}
return stderr
@ -277,10 +364,10 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) {
//
// virtualPath has to stay in rootPath, because locations outside root path are not allowed:
//
// Error: path /var/folders/.../pydabs/dist/*.whl is not contained in bundle root path
// Error: path /var/folders/.../python/dist/*.whl is not contained in bundle root path
//
// for that, we pass virtualPath instead of outputPath as file location
virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_pydabs__.yml"))
virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_python__.yml"))
if err != nil {
return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err))
}
@ -334,19 +421,23 @@ func loadDiagnosticsFile(path string) (diag.Diagnostics, error) {
func createOverrideVisitor(ctx context.Context, phase phase) (merge.OverrideVisitor, error) {
switch phase {
case PythonMutatorPhaseLoad:
return createLoadOverrideVisitor(ctx), nil
return createLoadResourcesOverrideVisitor(ctx), nil
case PythonMutatorPhaseInit:
return createInitOverrideVisitor(ctx), nil
return createInitOverrideVisitor(ctx, insertResourceModeAllow), nil
case PythonMutatorPhaseLoadResources:
return createLoadResourcesOverrideVisitor(ctx), nil
case PythonMutatorPhaseApplyMutators:
return createInitOverrideVisitor(ctx, insertResourceModeDisallow), nil
default:
return merge.OverrideVisitor{}, fmt.Errorf("unknown phase: %s", phase)
}
}
// createLoadOverrideVisitor creates an override visitor for the load phase.
// createLoadResourcesOverrideVisitor creates an override visitor for the load_resources phase.
//
// During load, it's only possible to create new resources, and not modify or
// During load_resources, it's only possible to create new resources, and not modify or
// delete existing ones.
func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
func createLoadResourcesOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
resourcesPath := dyn.NewPath(dyn.Key("resources"))
jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
@ -385,11 +476,21 @@ func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
}
}
// insertResourceMode controls whether createInitOverrideVisitor allows or disallows inserting new resources.
type insertResourceMode int
const (
insertResourceModeDisallow insertResourceMode = iota
insertResourceModeAllow insertResourceMode = iota
)
// createInitOverrideVisitor creates an override visitor for the init phase.
//
// During the init phase it's possible to create new resources, modify existing
// resources, but not delete existing resources.
func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
//
// If mode is insertResourceModeDisallow, it matching expected behaviour of apply_mutators
func createInitOverrideVisitor(ctx context.Context, mode insertResourceMode) merge.OverrideVisitor {
resourcesPath := dyn.NewPath(dyn.Key("resources"))
jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
@ -424,6 +525,11 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
}
insertResource := len(valuePath) == len(jobsPath)+1
if mode == insertResourceModeDisallow && insertResource {
return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String())
}
log.Debugf(ctx, "Insert value at %q", valuePath.String())
return right, nil
@ -441,9 +547,9 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor {
}
func isOmitemptyDelete(left dyn.Value) bool {
// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
// Python output can omit empty sequences/mappings, because we don't track them as optional,
// there is no semantic difference between empty and missing, so we keep them as they were before
// PyDABs deleted them.
// Python mutator deleted them.
switch left.Kind() {
case dyn.KindMap:

207
bundle/config/mutator/python/python_mutator_test.go
View File

@ -40,13 +40,25 @@ func TestPythonMutator_Name_init(t *testing.T) {
assert.Equal(t, "PythonMutator(init)", mutator.Name())
}
func TestPythonMutator_load(t *testing.T) {
func TestPythonMutator_Name_loadResources(t *testing.T) {
mutator := PythonMutator(PythonMutatorPhaseLoadResources)
assert.Equal(t, "PythonMutator(load_resources)", mutator.Name())
}
func TestPythonMutator_Name_applyMutators(t *testing.T) {
mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
assert.Equal(t, "PythonMutator(apply_mutators)", mutator.Name())
}
func TestPythonMutator_loadResources(t *testing.T) {
withFakeVEnv(t, ".venv")
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true
python:
resources: ["resources:load_resources"]
venv_path: .venv
resources:
jobs:
@ -60,12 +72,12 @@ func TestPythonMutator_load(t *testing.T) {
"-m",
"databricks.bundles.build",
"--phase",
"load",
"load_resources",
},
`{
"experimental": {
"pydabs": {
"enabled": true,
"python": {
"resources": ["resources:load_resources"],
"venv_path": ".venv"
}
},
@ -83,7 +95,7 @@ func TestPythonMutator_load(t *testing.T) {
`{"severity": "warning", "summary": "job doesn't have any tasks", "location": {"file": "src/examples/file.py", "line": 10, "column": 5}}`,
)
mutator := PythonMutator(PythonMutatorPhaseLoad)
mutator := PythonMutator(PythonMutatorPhaseLoadResources)
diags := bundle.Apply(ctx, b, mutator)
assert.NoError(t, diags.Error())
@ -109,13 +121,12 @@ func TestPythonMutator_load(t *testing.T) {
}, diags[0].Locations)
}
func TestPythonMutator_load_disallowed(t *testing.T) {
func TestPythonMutator_loadResources_disallowed(t *testing.T) {
withFakeVEnv(t, ".venv")
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true
python:
resources: ["resources:load_resources"]
venv_path: .venv
resources:
jobs:
@ -129,12 +140,12 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
"-m",
"databricks.bundles.build",
"--phase",
"load",
"load_resources",
},
`{
"experimental": {
"pydabs": {
"enabled": true,
"python": {
"resources": ["resources:load_resources"],
"venv_path": ".venv"
}
},
@ -148,20 +159,20 @@ func TestPythonMutator_load_disallowed(t *testing.T) {
}
}`, "")
mutator := PythonMutator(PythonMutatorPhaseLoad)
mutator := PythonMutator(PythonMutatorPhaseLoadResources)
diag := bundle.Apply(ctx, b, mutator)
assert.EqualError(t, diag.Error(), "unexpected change at \"resources.jobs.job0.description\" (insert)")
}
func TestPythonMutator_init(t *testing.T) {
func TestPythonMutator_applyMutators(t *testing.T) {
withFakeVEnv(t, ".venv")
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true
python:
venv_path: .venv
mutators:
- "mutators:add_description"
resources:
jobs:
job0:
@ -174,13 +185,13 @@ func TestPythonMutator_init(t *testing.T) {
"-m",
"databricks.bundles.build",
"--phase",
"init",
"apply_mutators",
},
`{
"experimental": {
"pydabs": {
"enabled": true,
"venv_path": ".venv"
"python": {
"venv_path": ".venv",
"mutators": ["mutators:add_description"]
}
},
"resources": {
@ -193,7 +204,7 @@ func TestPythonMutator_init(t *testing.T) {
}
}`, "")
mutator := PythonMutator(PythonMutatorPhaseInit)
mutator := PythonMutator(PythonMutatorPhaseApplyMutators)
diag := bundle.Apply(ctx, b, mutator)
assert.NoError(t, diag.Error())
@ -208,12 +219,12 @@ func TestPythonMutator_init(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "databricks.yml", name.Location().File)
// 'description' was updated by PyDABs and has location of generated file until
// 'description' was updated by Python code and has location of generated file until
// we implement source maps
description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description"))
require.NoError(t, err)
expectedVirtualPath, err := filepath.Abs("__generated_by_pydabs__.yml")
expectedVirtualPath, err := filepath.Abs("__generated_by_python__.yml")
require.NoError(t, err)
assert.Equal(t, expectedVirtualPath, description.Location().File)
@ -224,12 +235,12 @@ func TestPythonMutator_init(t *testing.T) {
func TestPythonMutator_badOutput(t *testing.T) {
withFakeVEnv(t, ".venv")
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true
python:
venv_path: .venv
resources:
- "resources:load_resources"
resources:
jobs:
job0:
@ -242,7 +253,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
"-m",
"databricks.bundles.build",
"--phase",
"load",
"load_resources",
},
`{
"resources": {
@ -254,7 +265,7 @@ func TestPythonMutator_badOutput(t *testing.T) {
}
}`, "")
mutator := PythonMutator(PythonMutatorPhaseLoad)
mutator := PythonMutator(PythonMutatorPhaseLoadResources)
diag := bundle.Apply(ctx, b, mutator)
assert.EqualError(t, diag.Error(), "unknown field: unknown_property")
@ -270,34 +281,63 @@ func TestPythonMutator_disabled(t *testing.T) {
assert.NoError(t, diag.Error())
}
func TestPythonMutator_venvRequired(t *testing.T) {
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true`)
ctx := context.Background()
mutator := PythonMutator(PythonMutatorPhaseLoad)
diag := bundle.Apply(ctx, b, mutator)
assert.Error(t, diag.Error(), "\"experimental.enable_pydabs\" is enabled, but \"experimental.venv.path\" is not set")
}
func TestPythonMutator_venvNotFound(t *testing.T) {
expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path"))
b := loadYaml("databricks.yml", `
experimental:
pydabs:
enabled: true
venv_path: bad_path`)
python:
venv_path: bad_path
resources:
- "resources:load_resources"`)
mutator := PythonMutator(PythonMutatorPhaseInit)
mutator := PythonMutator(PythonMutatorPhaseLoadResources)
diag := bundle.Apply(context.Background(), b, mutator)
assert.EqualError(t, diag.Error(), expectedError)
}
func TestGetOps_Python(t *testing.T) {
actual, err := getOpts(&bundle.Bundle{
Config: config.Root{
Experimental: &config.Experimental{
Python: config.Python{
VEnvPath: ".venv",
Resources: []string{
"resources:load_resources",
},
},
},
},
}, PythonMutatorPhaseLoadResources)
assert.NoError(t, err)
assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
}
func TestGetOps_PyDABs(t *testing.T) {
actual, err := getOpts(&bundle.Bundle{
Config: config.Root{
Experimental: &config.Experimental{
PyDABs: config.PyDABs{
VEnvPath: ".venv",
Enabled: true,
},
},
},
}, PythonMutatorPhaseInit)
assert.NoError(t, err)
assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual)
}
func TestGetOps_empty(t *testing.T) {
actual, err := getOpts(&bundle.Bundle{}, PythonMutatorPhaseLoadResources)
assert.NoError(t, err)
assert.Equal(t, opts{enabled: false}, actual)
}
type createOverrideVisitorTestCase struct {
name string
updatePath dyn.Path
@ -315,8 +355,8 @@ func TestCreateOverrideVisitor(t *testing.T) {
testCases := []createOverrideVisitorTestCase{
{
name: "load: can't change an existing job",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can't change an existing job",
phase: PythonMutatorPhaseLoadResources,
updatePath: dyn.MustPathFromString("resources.jobs.job0.name"),
deletePath: dyn.MustPathFromString("resources.jobs.job0.name"),
insertPath: dyn.MustPathFromString("resources.jobs.job0.name"),
@ -325,32 +365,32 @@ func TestCreateOverrideVisitor(t *testing.T) {
updateError: errors.New("unexpected change at \"resources.jobs.job0.name\" (update)"),
},
{
name: "load: can't delete an existing job",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can't delete an existing job",
phase: PythonMutatorPhaseLoadResources,
deletePath: dyn.MustPathFromString("resources.jobs.job0"),
deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
},
{
name: "load: can insert 'resources'",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can insert 'resources'",
phase: PythonMutatorPhaseLoadResources,
insertPath: dyn.MustPathFromString("resources"),
insertError: nil,
},
{
name: "load: can insert 'resources.jobs'",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can insert 'resources.jobs'",
phase: PythonMutatorPhaseLoadResources,
insertPath: dyn.MustPathFromString("resources.jobs"),
insertError: nil,
},
{
name: "load: can insert a job",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can insert a job",
phase: PythonMutatorPhaseLoadResources,
insertPath: dyn.MustPathFromString("resources.jobs.job0"),
insertError: nil,
},
{
name: "load: can't change include",
phase: PythonMutatorPhaseLoad,
name: "load_resources: can't change include",
phase: PythonMutatorPhaseLoadResources,
deletePath: dyn.MustPathFromString("include[0]"),
insertPath: dyn.MustPathFromString("include[0]"),
updatePath: dyn.MustPathFromString("include[0]"),
@ -402,6 +442,40 @@ func TestCreateOverrideVisitor(t *testing.T) {
insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
updateError: errors.New("unexpected change at \"include[0]\" (update)"),
},
{
name: "apply_mutators: can't delete an existing job",
phase: PythonMutatorPhaseInit,
deletePath: dyn.MustPathFromString("resources.jobs.job0"),
deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"),
},
{
name: "apply_mutators: can insert 'resources'",
phase: PythonMutatorPhaseApplyMutators,
insertPath: dyn.MustPathFromString("resources"),
insertError: nil,
},
{
name: "apply_mutators: can insert 'resources.jobs'",
phase: PythonMutatorPhaseApplyMutators,
insertPath: dyn.MustPathFromString("resources.jobs"),
insertError: nil,
},
{
name: "apply_mutators: can't insert a job",
phase: PythonMutatorPhaseApplyMutators,
insertPath: dyn.MustPathFromString("resources.jobs.job0"),
insertError: errors.New("unexpected change at \"resources.jobs.job0\" (insert)"),
},
{
name: "apply_mutators: can't change include",
phase: PythonMutatorPhaseApplyMutators,
deletePath: dyn.MustPathFromString("include[0]"),
insertPath: dyn.MustPathFromString("include[0]"),
updatePath: dyn.MustPathFromString("include[0]"),
deleteError: errors.New("unexpected change at \"include[0]\" (delete)"),
insertError: errors.New("unexpected change at \"include[0]\" (insert)"),
updateError: errors.New("unexpected change at \"include[0]\" (update)"),
},
}
for _, tc := range testCases {
@ -459,9 +533,9 @@ type overrideVisitorOmitemptyTestCase struct {
}
func TestCreateOverrideVisitor_omitempty(t *testing.T) {
// PyDABs can omit empty sequences/mappings in output, because we don't track them as optional,
// Python output can omit empty sequences/mappings in output, because we don't track them as optional,
// there is no semantic difference between empty and missing, so we keep them as they were before
// PyDABs deleted them.
// Python code deleted them.
allPhases := []phase{PythonMutatorPhaseLoad, PythonMutatorPhaseInit}
location := dyn.Location{
@ -568,18 +642,17 @@ func TestExplainProcessErr(t *testing.T) {
stderr := "/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')\n"
expected := `/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')
Explanation: 'databricks-pydabs' library is not installed in the Python environment.
Explanation: 'databricks-bundles' library is not installed in the Python environment.
If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies,
and that the wheel is installed in the Python environment:
Ensure that 'databricks-bundles' is installed in Python environment:
$ .venv/bin/pip install -e .
$ .venv/bin/pip install databricks-bundles
If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml,
or activate the environment before running CLI commands:
experimental:
pydabs:
python:
venv_path: .venv
`

21
bundle/internal/schema/annotations.yml
View File

@ -69,6 +69,9 @@ github.com/databricks/cli/bundle/config.Experimental:
"pydabs":
"description": |-
The PyDABs configuration.
"python":
"description": |-
Configures loading of Python code defined with 'databricks-bundles' package.
"python_wheel_wrapper":
"description": |-
Whether to use a Python wheel wrapper
@ -125,6 +128,24 @@ github.com/databricks/cli/bundle/config.PyDABs:
"venv_path":
"description": |-
The Python virtual environment path
github.com/databricks/cli/bundle/config.Python:
"mutators":
"description": |-
Mutators contains a list of fully qualified function paths to mutator functions.
Example: ["my_project.mutators:add_default_cluster"]
"resources":
"description": |-
Resources contains a list of fully qualified function paths to load resources
defined in Python code.
Example: ["my_project.resources:load_resources"]
"venv_path":
"description": |-
VEnvPath is path to the virtual environment.
If enabled, Python code will execute within this environment. If disabled,
it defaults to using the Python interpreter available in the current shell.
github.com/databricks/cli/bundle/config.Resources:
"clusters":
"description": |-

2
bundle/phases/initialize.go
View File

@ -55,6 +55,8 @@ func Initialize() bundle.Mutator {
// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
// See what is expected in PythonMutatorPhaseInit doc
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoadResources),
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
mutator.ResolveVariableReferencesInLookup(),
mutator.ResolveResourceReferences(),
mutator.ResolveVariableReferencesInComplexVariables(),

34
bundle/schema/jsonschema.json generated
View File

@ -1100,6 +1100,10 @@
"description": "The PyDABs configuration.",
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs"
},
"python": {
"description": "Configures loading of Python code defined with 'databricks-bundles' package.",
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python"
},
"python_wheel_wrapper": {
"description": "Whether to use a Python wheel wrapper",
"$ref": "#/$defs/bool"
@ -1234,6 +1238,36 @@
}
]
},
"config.Python": {
"oneOf": [
{
"type": "object",
"properties": {
"mutators": {
"description": "Mutators contains a list of fully qualified function paths to mutator functions.\n\nExample: [\"my_project.mutators:add_default_cluster\"]",
"$ref": "#/$defs/slice/string"
},
"resources": {
"description": "Resources contains a list of fully qualified function paths to load resources\ndefined in Python code.\n\nExample: [\"my_project.resources:load_resources\"]",
"$ref": "#/$defs/slice/string"
},
"venv_path": {
"description": "VEnvPath is path to the virtual environment.\n\nIf enabled, Python code will execute within this environment. If disabled,\nit defaults to using the Python interpreter available in the current shell.",
"$ref": "#/$defs/string"
}
},
"additionalProperties": false,
"required": [
"resources",
"mutators"
]
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"config.Resources": {
"oneOf": [
{