diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 4c787168f..7ecac5d7d 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -27,9 +27,33 @@ type Experimental struct { // PyDABs determines whether to load the 'databricks-pydabs' package. // // PyDABs allows to define bundle configuration using Python. + // PyDABs is deprecated use Python instead. PyDABs PyDABs `json:"pydabs,omitempty"` + + // Python configures loading of Python code defined with 'databricks-bundles' package. + Python Python `json:"python,omitempty"` } +type Python struct { + // Resources contains a list of fully qualified function paths to load resources + // defined in Python code. + // + // Example: ["my_project.resources:load_resources"] + Resources []string `json:"resources"` + + // Mutators contains a list of fully qualified function paths to mutator functions. + // + // Example: ["my_project.mutators:add_default_cluster"] + Mutators []string `json:"mutators"` + + // VEnvPath is path to the virtual environment. + // + // If enabled, Python code will execute within this environment. If disabled, + // it defaults to using the Python interpreter available in the current shell. + VEnvPath string `json:"venv_path,omitempty"` +} + +// PyDABs is deprecated use Python instead type PyDABs struct { // Enabled is a flag to enable the feature. Enabled bool `json:"enabled,omitempty"` diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 69c1a5dd6..8009ab243 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "reflect" "strings" "github.com/databricks/databricks-sdk-go/logger" @@ -40,6 +41,8 @@ const ( // We also open for possibility of appending other sections of bundle configuration, // for example, adding new variables. However, this is not supported yet, and CLI rejects // such changes. + // + // Deprecated, left for backward-compatibility with PyDABs. PythonMutatorPhaseLoad phase = "load" // PythonMutatorPhaseInit is the phase after bundle configuration was loaded, and @@ -59,7 +62,46 @@ const ( // PyDABs can output YAML containing references to variables, and CLI should resolve them. // // Existing resources can't be removed, and CLI rejects such changes. + // + // Deprecated, left for backward-compatibility with PyDABs. PythonMutatorPhaseInit phase = "init" + + // PythonMutatorPhaseLoadResources is the phase in which YAML configuration was loaded. + // + // At this stage, we execute Python code to load resources defined in Python. + // + // During this process, Python code can access: + // - selected deployment target + // - bundle variable values + // - variables provided through CLI argument or environment variables + // + // The following is not available: + // - variables referencing other variables are in unresolved format + // + // Python code can output YAML referencing variables, and CLI should resolve them. + // + // Existing resources can't be removed or modified, and CLI rejects such changes. + // While it's called 'load_resources', this phase is executed in 'init' phase of mutator pipeline. + PythonMutatorPhaseLoadResources phase = "load_resources" + + // PythonMutatorPhaseApplyMutators is the phase in which resources defined in YAML or Python + // are already loaded. + // + // At this stage, we execute Python code to mutate resources defined in YAML or Python. + // + // During this process, Python code can access: + // - selected deployment target + // - bundle variable values + // - variables provided through CLI argument or environment variables + // + // The following is not available: + // - variables referencing other variables are in unresolved format + // + // Python code can output YAML referencing variables, and CLI should resolve them. + // + // Resources can't be added or removed, and CLI rejects such changes. Python code is + // allowed to modify existing resources, but not other parts of bundle configuration. + PythonMutatorPhaseApplyMutators phase = "apply_mutators" ) type pythonMutator struct { @@ -76,18 +118,64 @@ func (m *pythonMutator) Name() string { return fmt.Sprintf("PythonMutator(%s)", m.phase) } -func getExperimental(b *bundle.Bundle) config.Experimental { - if b.Config.Experimental == nil { - return config.Experimental{} +// opts is a common structure for deprecated PyDABs and upcoming Python +// configuration sections +type opts struct { + enabled bool + + venvPath string +} + +// getOpts adapts deprecated PyDABs and upcoming Python configuration +// into a common structure. +func getOpts(b *bundle.Bundle, phase phase) (opts, error) { + experimental := b.Config.Experimental + if experimental == nil { + return opts{}, nil } - return *b.Config.Experimental + // using reflect.DeepEquals in case we add more fields + pydabsEnabled := !reflect.DeepEqual(experimental.PyDABs, config.PyDABs{}) + pythonEnabled := !reflect.DeepEqual(experimental.Python, config.Python{}) + + if pydabsEnabled && pythonEnabled { + return opts{}, errors.New("both experimental/pydabs and experimental/python are enabled, only one can be enabled") + } else if pydabsEnabled { + if !experimental.PyDABs.Enabled { + return opts{}, nil + } + + // don't execute for phases for 'python' section + if phase == PythonMutatorPhaseInit || phase == PythonMutatorPhaseLoad { + return opts{ + enabled: true, + venvPath: experimental.PyDABs.VEnvPath, + }, nil + } else { + return opts{}, nil + } + } else if pythonEnabled { + // don't execute for phases for 'pydabs' section + if phase == PythonMutatorPhaseLoadResources || phase == PythonMutatorPhaseApplyMutators { + return opts{ + enabled: true, + venvPath: experimental.Python.VEnvPath, + }, nil + } else { + return opts{}, nil + } + } else { + return opts{}, nil + } } func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - experimental := getExperimental(b) + opts, err := getOpts(b, m.phase) + if err != nil { + return diag.Errorf("failed to apply python mutator: %s", err) + } - if !experimental.PyDABs.Enabled { + if !opts.enabled { return nil } @@ -95,8 +183,8 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno var mutateDiags diag.Diagnostics mutateDiagsHasError := errors.New("unexpected error") - err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { - pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath) + err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { + pythonPath, err := detectExecutable(ctx, opts.venvPath) if err != nil { return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err) } @@ -137,7 +225,7 @@ func createCacheDir(ctx context.Context) (string, error) { // support the same env variable as in b.CacheDir if tempDir, exists := env.TempDir(ctx); exists { // use 'default' as target name - cacheDir := filepath.Join(tempDir, "default", "pydabs") + cacheDir := filepath.Join(tempDir, "default", "python") err := os.MkdirAll(cacheDir, 0o700) if err != nil { @@ -147,7 +235,7 @@ func createCacheDir(ctx context.Context) (string, error) { return cacheDir, nil } - return os.MkdirTemp("", "-pydabs") + return os.MkdirTemp("", "-python") } func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath, pythonPath string, root dyn.Value) (dyn.Value, diag.Diagnostics) { @@ -203,7 +291,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath } // process can fail without reporting errors in diagnostics file or creating it, for instance, - // venv doesn't have PyDABs library installed + // venv doesn't have 'databricks-bundles' library installed if processErr != nil { diagnostic := diag.Diagnostic{ Severity: diag.Error, @@ -226,16 +314,15 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir, rootPath return output, pythonDiagnostics } -const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, -and that the wheel is installed in the Python environment: +const pythonInstallExplanation = `Ensure that 'databricks-bundles' is installed in Python environment: - $ .venv/bin/pip install -e . + $ .venv/bin/pip install databricks-bundles If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: - pydabs: + python: venv_path: .venv ` @@ -245,9 +332,9 @@ or activate the environment before running CLI commands: func explainProcessErr(stderr string) string { // implemented in cpython/Lib/runpy.py and portable across Python 3.x, including pypy if strings.Contains(stderr, "Error while finding module specification for 'databricks.bundles.build'") { - summary := color.CyanString("Explanation: ") + "'databricks-pydabs' library is not installed in the Python environment.\n" + summary := color.CyanString("Explanation: ") + "'databricks-bundles' library is not installed in the Python environment.\n" - return stderr + "\n" + summary + "\n" + installExplanation + return stderr + "\n" + summary + "\n" + pythonInstallExplanation } return stderr @@ -277,10 +364,10 @@ func loadOutputFile(rootPath, outputPath string) (dyn.Value, diag.Diagnostics) { // // virtualPath has to stay in rootPath, because locations outside root path are not allowed: // - // Error: path /var/folders/.../pydabs/dist/*.whl is not contained in bundle root path + // Error: path /var/folders/.../python/dist/*.whl is not contained in bundle root path // // for that, we pass virtualPath instead of outputPath as file location - virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_pydabs__.yml")) + virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_python__.yml")) if err != nil { return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) } @@ -334,19 +421,23 @@ func loadDiagnosticsFile(path string) (diag.Diagnostics, error) { func createOverrideVisitor(ctx context.Context, phase phase) (merge.OverrideVisitor, error) { switch phase { case PythonMutatorPhaseLoad: - return createLoadOverrideVisitor(ctx), nil + return createLoadResourcesOverrideVisitor(ctx), nil case PythonMutatorPhaseInit: - return createInitOverrideVisitor(ctx), nil + return createInitOverrideVisitor(ctx, insertResourceModeAllow), nil + case PythonMutatorPhaseLoadResources: + return createLoadResourcesOverrideVisitor(ctx), nil + case PythonMutatorPhaseApplyMutators: + return createInitOverrideVisitor(ctx, insertResourceModeDisallow), nil default: return merge.OverrideVisitor{}, fmt.Errorf("unknown phase: %s", phase) } } -// createLoadOverrideVisitor creates an override visitor for the load phase. +// createLoadResourcesOverrideVisitor creates an override visitor for the load_resources phase. // -// During load, it's only possible to create new resources, and not modify or +// During load_resources, it's only possible to create new resources, and not modify or // delete existing ones. -func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor { +func createLoadResourcesOverrideVisitor(ctx context.Context) merge.OverrideVisitor { resourcesPath := dyn.NewPath(dyn.Key("resources")) jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) @@ -385,11 +476,21 @@ func createLoadOverrideVisitor(ctx context.Context) merge.OverrideVisitor { } } +// insertResourceMode controls whether createInitOverrideVisitor allows or disallows inserting new resources. +type insertResourceMode int + +const ( + insertResourceModeDisallow insertResourceMode = iota + insertResourceModeAllow insertResourceMode = iota +) + // createInitOverrideVisitor creates an override visitor for the init phase. // // During the init phase it's possible to create new resources, modify existing // resources, but not delete existing resources. -func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor { +// +// If mode is insertResourceModeDisallow, it matching expected behaviour of apply_mutators +func createInitOverrideVisitor(ctx context.Context, mode insertResourceMode) merge.OverrideVisitor { resourcesPath := dyn.NewPath(dyn.Key("resources")) jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) @@ -424,6 +525,11 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor { return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String()) } + insertResource := len(valuePath) == len(jobsPath)+1 + if mode == insertResourceModeDisallow && insertResource { + return dyn.InvalidValue, fmt.Errorf("unexpected change at %q (insert)", valuePath.String()) + } + log.Debugf(ctx, "Insert value at %q", valuePath.String()) return right, nil @@ -441,9 +547,9 @@ func createInitOverrideVisitor(ctx context.Context) merge.OverrideVisitor { } func isOmitemptyDelete(left dyn.Value) bool { - // PyDABs can omit empty sequences/mappings in output, because we don't track them as optional, + // Python output can omit empty sequences/mappings, because we don't track them as optional, // there is no semantic difference between empty and missing, so we keep them as they were before - // PyDABs deleted them. + // Python mutator deleted them. switch left.Kind() { case dyn.KindMap: diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index ff21f8ed9..d51572c8a 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -40,13 +40,25 @@ func TestPythonMutator_Name_init(t *testing.T) { assert.Equal(t, "PythonMutator(init)", mutator.Name()) } -func TestPythonMutator_load(t *testing.T) { +func TestPythonMutator_Name_loadResources(t *testing.T) { + mutator := PythonMutator(PythonMutatorPhaseLoadResources) + + assert.Equal(t, "PythonMutator(load_resources)", mutator.Name()) +} + +func TestPythonMutator_Name_applyMutators(t *testing.T) { + mutator := PythonMutator(PythonMutatorPhaseApplyMutators) + + assert.Equal(t, "PythonMutator(apply_mutators)", mutator.Name()) +} + +func TestPythonMutator_loadResources(t *testing.T) { withFakeVEnv(t, ".venv") b := loadYaml("databricks.yml", ` experimental: - pydabs: - enabled: true + python: + resources: ["resources:load_resources"] venv_path: .venv resources: jobs: @@ -60,12 +72,12 @@ func TestPythonMutator_load(t *testing.T) { "-m", "databricks.bundles.build", "--phase", - "load", + "load_resources", }, `{ "experimental": { - "pydabs": { - "enabled": true, + "python": { + "resources": ["resources:load_resources"], "venv_path": ".venv" } }, @@ -83,7 +95,7 @@ func TestPythonMutator_load(t *testing.T) { `{"severity": "warning", "summary": "job doesn't have any tasks", "location": {"file": "src/examples/file.py", "line": 10, "column": 5}}`, ) - mutator := PythonMutator(PythonMutatorPhaseLoad) + mutator := PythonMutator(PythonMutatorPhaseLoadResources) diags := bundle.Apply(ctx, b, mutator) assert.NoError(t, diags.Error()) @@ -109,13 +121,12 @@ func TestPythonMutator_load(t *testing.T) { }, diags[0].Locations) } -func TestPythonMutator_load_disallowed(t *testing.T) { +func TestPythonMutator_loadResources_disallowed(t *testing.T) { withFakeVEnv(t, ".venv") - b := loadYaml("databricks.yml", ` experimental: - pydabs: - enabled: true + python: + resources: ["resources:load_resources"] venv_path: .venv resources: jobs: @@ -129,12 +140,12 @@ func TestPythonMutator_load_disallowed(t *testing.T) { "-m", "databricks.bundles.build", "--phase", - "load", + "load_resources", }, `{ "experimental": { - "pydabs": { - "enabled": true, + "python": { + "resources": ["resources:load_resources"], "venv_path": ".venv" } }, @@ -148,20 +159,20 @@ func TestPythonMutator_load_disallowed(t *testing.T) { } }`, "") - mutator := PythonMutator(PythonMutatorPhaseLoad) + mutator := PythonMutator(PythonMutatorPhaseLoadResources) diag := bundle.Apply(ctx, b, mutator) assert.EqualError(t, diag.Error(), "unexpected change at \"resources.jobs.job0.description\" (insert)") } -func TestPythonMutator_init(t *testing.T) { +func TestPythonMutator_applyMutators(t *testing.T) { withFakeVEnv(t, ".venv") - b := loadYaml("databricks.yml", ` experimental: - pydabs: - enabled: true + python: venv_path: .venv + mutators: + - "mutators:add_description" resources: jobs: job0: @@ -174,13 +185,13 @@ func TestPythonMutator_init(t *testing.T) { "-m", "databricks.bundles.build", "--phase", - "init", + "apply_mutators", }, `{ "experimental": { - "pydabs": { - "enabled": true, - "venv_path": ".venv" + "python": { + "venv_path": ".venv", + "mutators": ["mutators:add_description"] } }, "resources": { @@ -193,7 +204,7 @@ func TestPythonMutator_init(t *testing.T) { } }`, "") - mutator := PythonMutator(PythonMutatorPhaseInit) + mutator := PythonMutator(PythonMutatorPhaseApplyMutators) diag := bundle.Apply(ctx, b, mutator) assert.NoError(t, diag.Error()) @@ -208,12 +219,12 @@ func TestPythonMutator_init(t *testing.T) { require.NoError(t, err) assert.Equal(t, "databricks.yml", name.Location().File) - // 'description' was updated by PyDABs and has location of generated file until + // 'description' was updated by Python code and has location of generated file until // we implement source maps description, err := dyn.GetByPath(v, dyn.MustPathFromString("resources.jobs.job0.description")) require.NoError(t, err) - expectedVirtualPath, err := filepath.Abs("__generated_by_pydabs__.yml") + expectedVirtualPath, err := filepath.Abs("__generated_by_python__.yml") require.NoError(t, err) assert.Equal(t, expectedVirtualPath, description.Location().File) @@ -224,12 +235,12 @@ func TestPythonMutator_init(t *testing.T) { func TestPythonMutator_badOutput(t *testing.T) { withFakeVEnv(t, ".venv") - b := loadYaml("databricks.yml", ` experimental: - pydabs: - enabled: true + python: venv_path: .venv + resources: + - "resources:load_resources" resources: jobs: job0: @@ -242,7 +253,7 @@ func TestPythonMutator_badOutput(t *testing.T) { "-m", "databricks.bundles.build", "--phase", - "load", + "load_resources", }, `{ "resources": { @@ -254,7 +265,7 @@ func TestPythonMutator_badOutput(t *testing.T) { } }`, "") - mutator := PythonMutator(PythonMutatorPhaseLoad) + mutator := PythonMutator(PythonMutatorPhaseLoadResources) diag := bundle.Apply(ctx, b, mutator) assert.EqualError(t, diag.Error(), "unknown field: unknown_property") @@ -270,34 +281,63 @@ func TestPythonMutator_disabled(t *testing.T) { assert.NoError(t, diag.Error()) } -func TestPythonMutator_venvRequired(t *testing.T) { - b := loadYaml("databricks.yml", ` - experimental: - pydabs: - enabled: true`) - - ctx := context.Background() - mutator := PythonMutator(PythonMutatorPhaseLoad) - diag := bundle.Apply(ctx, b, mutator) - - assert.Error(t, diag.Error(), "\"experimental.enable_pydabs\" is enabled, but \"experimental.venv.path\" is not set") -} - func TestPythonMutator_venvNotFound(t *testing.T) { expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path")) b := loadYaml("databricks.yml", ` experimental: - pydabs: - enabled: true - venv_path: bad_path`) + python: + venv_path: bad_path + resources: + - "resources:load_resources"`) - mutator := PythonMutator(PythonMutatorPhaseInit) + mutator := PythonMutator(PythonMutatorPhaseLoadResources) diag := bundle.Apply(context.Background(), b, mutator) assert.EqualError(t, diag.Error(), expectedError) } +func TestGetOps_Python(t *testing.T) { + actual, err := getOpts(&bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{ + Python: config.Python{ + VEnvPath: ".venv", + Resources: []string{ + "resources:load_resources", + }, + }, + }, + }, + }, PythonMutatorPhaseLoadResources) + + assert.NoError(t, err) + assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual) +} + +func TestGetOps_PyDABs(t *testing.T) { + actual, err := getOpts(&bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{ + PyDABs: config.PyDABs{ + VEnvPath: ".venv", + Enabled: true, + }, + }, + }, + }, PythonMutatorPhaseInit) + + assert.NoError(t, err) + assert.Equal(t, opts{venvPath: ".venv", enabled: true}, actual) +} + +func TestGetOps_empty(t *testing.T) { + actual, err := getOpts(&bundle.Bundle{}, PythonMutatorPhaseLoadResources) + + assert.NoError(t, err) + assert.Equal(t, opts{enabled: false}, actual) +} + type createOverrideVisitorTestCase struct { name string updatePath dyn.Path @@ -315,8 +355,8 @@ func TestCreateOverrideVisitor(t *testing.T) { testCases := []createOverrideVisitorTestCase{ { - name: "load: can't change an existing job", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can't change an existing job", + phase: PythonMutatorPhaseLoadResources, updatePath: dyn.MustPathFromString("resources.jobs.job0.name"), deletePath: dyn.MustPathFromString("resources.jobs.job0.name"), insertPath: dyn.MustPathFromString("resources.jobs.job0.name"), @@ -325,32 +365,32 @@ func TestCreateOverrideVisitor(t *testing.T) { updateError: errors.New("unexpected change at \"resources.jobs.job0.name\" (update)"), }, { - name: "load: can't delete an existing job", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can't delete an existing job", + phase: PythonMutatorPhaseLoadResources, deletePath: dyn.MustPathFromString("resources.jobs.job0"), deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"), }, { - name: "load: can insert 'resources'", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can insert 'resources'", + phase: PythonMutatorPhaseLoadResources, insertPath: dyn.MustPathFromString("resources"), insertError: nil, }, { - name: "load: can insert 'resources.jobs'", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can insert 'resources.jobs'", + phase: PythonMutatorPhaseLoadResources, insertPath: dyn.MustPathFromString("resources.jobs"), insertError: nil, }, { - name: "load: can insert a job", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can insert a job", + phase: PythonMutatorPhaseLoadResources, insertPath: dyn.MustPathFromString("resources.jobs.job0"), insertError: nil, }, { - name: "load: can't change include", - phase: PythonMutatorPhaseLoad, + name: "load_resources: can't change include", + phase: PythonMutatorPhaseLoadResources, deletePath: dyn.MustPathFromString("include[0]"), insertPath: dyn.MustPathFromString("include[0]"), updatePath: dyn.MustPathFromString("include[0]"), @@ -402,6 +442,40 @@ func TestCreateOverrideVisitor(t *testing.T) { insertError: errors.New("unexpected change at \"include[0]\" (insert)"), updateError: errors.New("unexpected change at \"include[0]\" (update)"), }, + { + name: "apply_mutators: can't delete an existing job", + phase: PythonMutatorPhaseInit, + deletePath: dyn.MustPathFromString("resources.jobs.job0"), + deleteError: errors.New("unexpected change at \"resources.jobs.job0\" (delete)"), + }, + { + name: "apply_mutators: can insert 'resources'", + phase: PythonMutatorPhaseApplyMutators, + insertPath: dyn.MustPathFromString("resources"), + insertError: nil, + }, + { + name: "apply_mutators: can insert 'resources.jobs'", + phase: PythonMutatorPhaseApplyMutators, + insertPath: dyn.MustPathFromString("resources.jobs"), + insertError: nil, + }, + { + name: "apply_mutators: can't insert a job", + phase: PythonMutatorPhaseApplyMutators, + insertPath: dyn.MustPathFromString("resources.jobs.job0"), + insertError: errors.New("unexpected change at \"resources.jobs.job0\" (insert)"), + }, + { + name: "apply_mutators: can't change include", + phase: PythonMutatorPhaseApplyMutators, + deletePath: dyn.MustPathFromString("include[0]"), + insertPath: dyn.MustPathFromString("include[0]"), + updatePath: dyn.MustPathFromString("include[0]"), + deleteError: errors.New("unexpected change at \"include[0]\" (delete)"), + insertError: errors.New("unexpected change at \"include[0]\" (insert)"), + updateError: errors.New("unexpected change at \"include[0]\" (update)"), + }, } for _, tc := range testCases { @@ -459,9 +533,9 @@ type overrideVisitorOmitemptyTestCase struct { } func TestCreateOverrideVisitor_omitempty(t *testing.T) { - // PyDABs can omit empty sequences/mappings in output, because we don't track them as optional, + // Python output can omit empty sequences/mappings in output, because we don't track them as optional, // there is no semantic difference between empty and missing, so we keep them as they were before - // PyDABs deleted them. + // Python code deleted them. allPhases := []phase{PythonMutatorPhaseLoad, PythonMutatorPhaseInit} location := dyn.Location{ @@ -568,18 +642,17 @@ func TestExplainProcessErr(t *testing.T) { stderr := "/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')\n" expected := `/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks') -Explanation: 'databricks-pydabs' library is not installed in the Python environment. +Explanation: 'databricks-bundles' library is not installed in the Python environment. -If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, -and that the wheel is installed in the Python environment: +Ensure that 'databricks-bundles' is installed in Python environment: - $ .venv/bin/pip install -e . + $ .venv/bin/pip install databricks-bundles If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: - pydabs: + python: venv_path: .venv ` diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 84f6753e3..5283a431b 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -69,6 +69,9 @@ github.com/databricks/cli/bundle/config.Experimental: "pydabs": "description": |- The PyDABs configuration. + "python": + "description": |- + Configures loading of Python code defined with 'databricks-bundles' package. "python_wheel_wrapper": "description": |- Whether to use a Python wheel wrapper @@ -125,6 +128,24 @@ github.com/databricks/cli/bundle/config.PyDABs: "venv_path": "description": |- The Python virtual environment path +github.com/databricks/cli/bundle/config.Python: + "mutators": + "description": |- + Mutators contains a list of fully qualified function paths to mutator functions. + + Example: ["my_project.mutators:add_default_cluster"] + "resources": + "description": |- + Resources contains a list of fully qualified function paths to load resources + defined in Python code. + + Example: ["my_project.resources:load_resources"] + "venv_path": + "description": |- + VEnvPath is path to the virtual environment. + + If enabled, Python code will execute within this environment. If disabled, + it defaults to using the Python interpreter available in the current shell. github.com/databricks/cli/bundle/config.Resources: "clusters": "description": |- diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 6fa0e5fed..f0cbc00c2 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -55,6 +55,8 @@ func Initialize() bundle.Mutator { // ResolveVariableReferencesInComplexVariables and ResolveVariableReferences. // See what is expected in PythonMutatorPhaseInit doc pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit), + pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoadResources), + pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators), mutator.ResolveVariableReferencesInLookup(), mutator.ResolveResourceReferences(), mutator.ResolveVariableReferencesInComplexVariables(), diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9a352ebb2..2f78ffcca 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1100,6 +1100,10 @@ "description": "The PyDABs configuration.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs" }, + "python": { + "description": "Configures loading of Python code defined with 'databricks-bundles' package.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python" + }, "python_wheel_wrapper": { "description": "Whether to use a Python wheel wrapper", "$ref": "#/$defs/bool" @@ -1234,6 +1238,36 @@ } ] }, + "config.Python": { + "oneOf": [ + { + "type": "object", + "properties": { + "mutators": { + "description": "Mutators contains a list of fully qualified function paths to mutator functions.\n\nExample: [\"my_project.mutators:add_default_cluster\"]", + "$ref": "#/$defs/slice/string" + }, + "resources": { + "description": "Resources contains a list of fully qualified function paths to load resources\ndefined in Python code.\n\nExample: [\"my_project.resources:load_resources\"]", + "$ref": "#/$defs/slice/string" + }, + "venv_path": { + "description": "VEnvPath is path to the virtual environment.\n\nIf enabled, Python code will execute within this environment. If disabled,\nit defaults to using the Python interpreter available in the current shell.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "resources", + "mutators" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.Resources": { "oneOf": [ {