mirror of https://github.com/databricks/cli.git
Compare commits
No commits in common. "37b0039c1c041a89e0567678f9f327b897858c3b" and "6dadf667b55644b99e780149e275a6a995dd3af6" have entirely different histories.
37b0039c1c
...
6dadf667b5
|
@ -33,7 +33,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.22.x
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
|
@ -68,7 +68,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.22.x
|
||||
|
||||
# No need to download cached dependencies when running gofmt.
|
||||
cache: false
|
||||
|
@ -100,7 +100,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.22.x
|
||||
|
||||
# Github repo: https://github.com/ajv-validator/ajv-cli
|
||||
- name: Install ajv-cli
|
||||
|
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.22.x
|
||||
|
||||
# The default cache key for this action considers only the `go.sum` file.
|
||||
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
||||
|
|
|
@ -22,7 +22,7 @@ jobs:
|
|||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.22.7
|
||||
go-version: 1.22.x
|
||||
|
||||
# The default cache key for this action considers only the `go.sum` file.
|
||||
# We include .goreleaser.yaml here to differentiate from the cache used by the push action
|
||||
|
|
27
CHANGELOG.md
27
CHANGELOG.md
|
@ -1,32 +1,5 @@
|
|||
# Version changelog
|
||||
|
||||
## [Release] Release v0.228.1
|
||||
|
||||
Bundles:
|
||||
* Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)).
|
||||
* Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)).
|
||||
* Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)).
|
||||
* Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)).
|
||||
* Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)).
|
||||
* Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)).
|
||||
|
||||
Internal:
|
||||
* Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)).
|
||||
* Make bundle JSON schema modular with `` ([#1700](https://github.com/databricks/cli/pull/1700)).
|
||||
* Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)).
|
||||
* Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)).
|
||||
* Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)).
|
||||
|
||||
API Changes:
|
||||
* Added `databricks quality-monitors regenerate-dashboard` command.
|
||||
|
||||
OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04)
|
||||
Dependency updates:
|
||||
* Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)).
|
||||
* Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)).
|
||||
* Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)).
|
||||
* Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)).
|
||||
|
||||
## [Release] Release v0.228.0
|
||||
|
||||
CLI:
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/internal/testutil"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -24,7 +23,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
|
|||
testutil.Touch(t, tmpDir, "bc.txt")
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
RootPath: tmpDir,
|
||||
Config: config.Root{
|
||||
Artifacts: config.Artifacts{
|
||||
"test": {
|
||||
|
@ -37,7 +36,7 @@ func TestExpandGlobs_Nominal(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||
bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||
|
@ -63,7 +62,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
RootPath: tmpDir,
|
||||
Config: config.Root{
|
||||
Artifacts: config.Artifacts{
|
||||
"test": {
|
||||
|
@ -78,7 +77,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||
bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||
|
@ -111,7 +110,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
|
|||
testutil.Touch(t, tmpDir, "b2.txt")
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
RootPath: tmpDir,
|
||||
Config: config.Root{
|
||||
Artifacts: config.Artifacts{
|
||||
"test": {
|
||||
|
@ -126,7 +125,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||
bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml"))
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||
|
|
|
@ -47,7 +47,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
|
|||
|
||||
// If artifact path is not provided, use bundle root dir
|
||||
if artifact.Path == "" {
|
||||
artifact.Path = b.BundleRootPath
|
||||
artifact.Path = b.RootPath
|
||||
}
|
||||
|
||||
if !filepath.IsAbs(artifact.Path) {
|
||||
|
|
|
@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
|
|||
log.Infof(ctx, "Detecting Python wheel project...")
|
||||
|
||||
// checking if there is setup.py in the bundle root
|
||||
setupPy := filepath.Join(b.BundleRootPath, "setup.py")
|
||||
setupPy := filepath.Join(b.RootPath, "setup.py")
|
||||
_, err := os.Stat(setupPy)
|
||||
if err != nil {
|
||||
log.Infof(ctx, "No Python wheel project found at bundle root folder")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath))
|
||||
log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath))
|
||||
module := extractModuleName(setupPy)
|
||||
|
||||
if b.Config.Artifacts == nil {
|
||||
b.Config.Artifacts = make(map[string]*config.Artifact)
|
||||
}
|
||||
|
||||
pkgPath, err := filepath.Abs(b.BundleRootPath)
|
||||
pkgPath, err := filepath.Abs(b.RootPath)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
|
|
@ -31,26 +31,22 @@ import (
|
|||
const internalFolder = ".internal"
|
||||
|
||||
type Bundle struct {
|
||||
// BundleRootPath is the local path to the root directory of the bundle.
|
||||
// RootPath contains the directory path to the root of the bundle.
|
||||
// It is set when we instantiate a new bundle instance.
|
||||
BundleRootPath string
|
||||
RootPath string
|
||||
|
||||
// BundleRoot is a virtual filesystem path to [BundleRootPath].
|
||||
// BundleRoot is a virtual filesystem path to the root of the bundle.
|
||||
// Exclusively use this field for filesystem operations.
|
||||
BundleRoot vfs.Path
|
||||
|
||||
// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
|
||||
// By default, it is the same as [BundleRootPath].
|
||||
// If it is different, it must be an ancestor to [BundleRootPath].
|
||||
// That is, [SyncRootPath] must contain [BundleRootPath].
|
||||
SyncRootPath string
|
||||
|
||||
// SyncRoot is a virtual filesystem path to [SyncRootPath].
|
||||
// Exclusively use this field for filesystem operations.
|
||||
// SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace.
|
||||
// It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot].
|
||||
SyncRoot vfs.Path
|
||||
|
||||
// Config contains the bundle configuration.
|
||||
// It is loaded from the bundle configuration files and mutators may update it.
|
||||
// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
|
||||
// It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access.
|
||||
SyncRootPath string
|
||||
|
||||
Config config.Root
|
||||
|
||||
// Metadata about the bundle deployment. This is the interface Databricks services
|
||||
|
@ -88,14 +84,14 @@ type Bundle struct {
|
|||
|
||||
func Load(ctx context.Context, path string) (*Bundle, error) {
|
||||
b := &Bundle{
|
||||
BundleRootPath: filepath.Clean(path),
|
||||
BundleRoot: vfs.MustNew(path),
|
||||
RootPath: filepath.Clean(path),
|
||||
BundleRoot: vfs.MustNew(path),
|
||||
}
|
||||
configFile, err := config.FileNames.FindInPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile)
|
||||
log.Debugf(ctx, "Found bundle root at %s (file %s)", b.RootPath, configFile)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
|
@ -164,7 +160,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
|
|||
if !exists || cacheDirName == "" {
|
||||
cacheDirName = filepath.Join(
|
||||
// Anchor at bundle root directory.
|
||||
b.BundleRootPath,
|
||||
b.RootPath,
|
||||
// Static cache directory.
|
||||
".databricks",
|
||||
"bundle",
|
||||
|
@ -216,7 +212,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir)
|
||||
internalDirRel, err := filepath.Rel(b.RootPath, internalDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -21,7 +21,7 @@ func (r ReadOnlyBundle) Config() config.Root {
|
|||
}
|
||||
|
||||
func (r ReadOnlyBundle) RootPath() string {
|
||||
return r.b.BundleRootPath
|
||||
return r.b.RootPath
|
||||
}
|
||||
|
||||
func (r ReadOnlyBundle) BundleRoot() vfs.Path {
|
||||
|
|
|
@ -79,7 +79,7 @@ func TestBundleMustLoadSuccess(t *testing.T) {
|
|||
t.Setenv(env.RootVariable, "./tests/basic")
|
||||
b, err := MustLoad(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
|
||||
assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
|
||||
}
|
||||
|
||||
func TestBundleMustLoadFailureWithEnv(t *testing.T) {
|
||||
|
@ -98,7 +98,7 @@ func TestBundleTryLoadSuccess(t *testing.T) {
|
|||
t.Setenv(env.RootVariable, "./tests/basic")
|
||||
b, err := TryLoad(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath))
|
||||
assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath))
|
||||
}
|
||||
|
||||
func TestBundleTryLoadFailureWithEnv(t *testing.T) {
|
||||
|
|
|
@ -38,11 +38,8 @@ type Bundle struct {
|
|||
// Annotated readonly as this should be set at the target level.
|
||||
Mode Mode `json:"mode,omitempty" bundle:"readonly"`
|
||||
|
||||
// DEPRECATED: Overrides the compute used for jobs and other supported assets.
|
||||
ComputeId string `json:"compute_id,omitempty"`
|
||||
|
||||
// Overrides the cluster used for jobs and other supported assets.
|
||||
ClusterId string `json:"cluster_id,omitempty"`
|
||||
// Overrides the compute used for jobs and other supported assets.
|
||||
ComputeID string `json:"compute_id,omitempty"`
|
||||
|
||||
// Deployment section specifies deployment related configuration for bundle
|
||||
Deployment Deployment `json:"deployment,omitempty"`
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
package generate
|
||||
|
||||
import (
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
||||
)
|
||||
|
||||
func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (dyn.Value, error) {
|
||||
// The majority of fields of the dashboard struct are read-only.
|
||||
// We copy the relevant fields manually.
|
||||
dv := map[string]dyn.Value{
|
||||
"display_name": dyn.NewValue(dashboard.DisplayName, []dyn.Location{{Line: 1}}),
|
||||
"parent_path": dyn.NewValue("${workspace.file_path}", []dyn.Location{{Line: 2}}),
|
||||
"warehouse_id": dyn.NewValue(dashboard.WarehouseId, []dyn.Location{{Line: 3}}),
|
||||
"definition_path": dyn.NewValue(filePath, []dyn.Location{{Line: 4}}),
|
||||
}
|
||||
|
||||
return dyn.V(dv), nil
|
||||
}
|
|
@ -20,7 +20,7 @@ func (m *entryPoint) Name() string {
|
|||
}
|
||||
|
||||
func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
path, err := config.FileNames.FindInPath(b.BundleRootPath)
|
||||
path, err := config.FileNames.FindInPath(b.RootPath)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
|
|
@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) {
|
|||
|
||||
func TestEntryPoint(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "testdata",
|
||||
RootPath: "testdata",
|
||||
}
|
||||
diags := bundle.Apply(context.Background(), b, loader.EntryPoint())
|
||||
require.NoError(t, diags.Error())
|
||||
|
|
|
@ -14,7 +14,7 @@ import (
|
|||
|
||||
func TestProcessInclude(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "testdata",
|
||||
RootPath: "testdata",
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
Host: "foo",
|
||||
|
@ -22,7 +22,7 @@ func TestProcessInclude(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, "host.yml"), "host.yml")
|
||||
m := loader.ProcessInclude(filepath.Join(b.RootPath, "host.yml"), "host.yml")
|
||||
assert.Equal(t, "ProcessInclude(host.yml)", m.Name())
|
||||
|
||||
// Assert the host value prior to applying the mutator
|
||||
|
|
|
@ -47,7 +47,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
|||
}
|
||||
|
||||
// Anchor includes to the bundle root path.
|
||||
matches, err := filepath.Glob(filepath.Join(b.BundleRootPath, entry))
|
||||
matches, err := filepath.Glob(filepath.Join(b.RootPath, entry))
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
@ -61,7 +61,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
|||
// Filter matches to ones we haven't seen yet.
|
||||
var includes []string
|
||||
for _, match := range matches {
|
||||
rel, err := filepath.Rel(b.BundleRootPath, match)
|
||||
rel, err := filepath.Rel(b.RootPath, match)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
@ -76,7 +76,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
|||
slices.Sort(includes)
|
||||
files = append(files, includes...)
|
||||
for _, include := range includes {
|
||||
out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include))
|
||||
out = append(out, ProcessInclude(filepath.Join(b.RootPath, include), include))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -15,7 +15,7 @@ import (
|
|||
|
||||
func TestProcessRootIncludesEmpty(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: ".",
|
||||
RootPath: ".",
|
||||
}
|
||||
diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -30,7 +30,7 @@ func TestProcessRootIncludesAbs(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: ".",
|
||||
RootPath: ".",
|
||||
Config: config.Root{
|
||||
Include: []string{
|
||||
"/tmp/*.yml",
|
||||
|
@ -44,7 +44,7 @@ func TestProcessRootIncludesAbs(t *testing.T) {
|
|||
|
||||
func TestProcessRootIncludesSingleGlob(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Include: []string{
|
||||
"*.yml",
|
||||
|
@ -52,9 +52,9 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
testutil.Touch(t, b.BundleRootPath, "databricks.yml")
|
||||
testutil.Touch(t, b.BundleRootPath, "a.yml")
|
||||
testutil.Touch(t, b.BundleRootPath, "b.yml")
|
||||
testutil.Touch(t, b.RootPath, "databricks.yml")
|
||||
testutil.Touch(t, b.RootPath, "a.yml")
|
||||
testutil.Touch(t, b.RootPath, "b.yml")
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -63,7 +63,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) {
|
|||
|
||||
func TestProcessRootIncludesMultiGlob(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Include: []string{
|
||||
"a*.yml",
|
||||
|
@ -72,8 +72,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
testutil.Touch(t, b.BundleRootPath, "a1.yml")
|
||||
testutil.Touch(t, b.BundleRootPath, "b1.yml")
|
||||
testutil.Touch(t, b.RootPath, "a1.yml")
|
||||
testutil.Touch(t, b.RootPath, "b1.yml")
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -82,7 +82,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) {
|
|||
|
||||
func TestProcessRootIncludesRemoveDups(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Include: []string{
|
||||
"*.yml",
|
||||
|
@ -91,7 +91,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
testutil.Touch(t, b.BundleRootPath, "a.yml")
|
||||
testutil.Touch(t, b.RootPath, "a.yml")
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -100,7 +100,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) {
|
|||
|
||||
func TestProcessRootIncludesNotExists(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Include: []string{
|
||||
"notexist.yml",
|
||||
|
|
|
@ -35,10 +35,8 @@ func (m *applyPresets) Name() string {
|
|||
}
|
||||
|
||||
func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
var diags diag.Diagnostics
|
||||
|
||||
if d := validatePauseStatus(b); d != nil {
|
||||
diags = diags.Extend(d)
|
||||
return d
|
||||
}
|
||||
|
||||
r := b.Config.Resources
|
||||
|
@ -47,11 +45,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
|||
tags := toTagArray(t.Tags)
|
||||
|
||||
// Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus
|
||||
for key, j := range r.Jobs {
|
||||
if j.JobSettings == nil {
|
||||
diags = diags.Extend(diag.Errorf("job %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
for _, j := range r.Jobs {
|
||||
j.Name = prefix + j.Name
|
||||
if j.Tags == nil {
|
||||
j.Tags = make(map[string]string)
|
||||
|
@ -83,27 +77,20 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
|||
}
|
||||
|
||||
// Pipelines presets: Prefix, PipelinesDevelopment
|
||||
for key, p := range r.Pipelines {
|
||||
if p.PipelineSpec == nil {
|
||||
diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
p.Name = prefix + p.Name
|
||||
for i := range r.Pipelines {
|
||||
r.Pipelines[i].Name = prefix + r.Pipelines[i].Name
|
||||
if config.IsExplicitlyEnabled(t.PipelinesDevelopment) {
|
||||
p.Development = true
|
||||
r.Pipelines[i].Development = true
|
||||
}
|
||||
if t.TriggerPauseStatus == config.Paused {
|
||||
p.Continuous = false
|
||||
r.Pipelines[i].Continuous = false
|
||||
}
|
||||
|
||||
// As of 2024-06, pipelines don't yet support tags
|
||||
}
|
||||
|
||||
// Models presets: Prefix, Tags
|
||||
for key, m := range r.Models {
|
||||
if m.Model == nil {
|
||||
diags = diags.Extend(diag.Errorf("model %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
for _, m := range r.Models {
|
||||
m.Name = prefix + m.Name
|
||||
for _, t := range tags {
|
||||
exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool {
|
||||
|
@ -117,11 +104,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
|||
}
|
||||
|
||||
// Experiments presets: Prefix, Tags
|
||||
for key, e := range r.Experiments {
|
||||
if e.Experiment == nil {
|
||||
diags = diags.Extend(diag.Errorf("experiment %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
for _, e := range r.Experiments {
|
||||
filepath := e.Name
|
||||
dir := path.Dir(filepath)
|
||||
base := path.Base(filepath)
|
||||
|
@ -145,79 +128,44 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
|||
}
|
||||
|
||||
// Model serving endpoint presets: Prefix
|
||||
for key, e := range r.ModelServingEndpoints {
|
||||
if e.CreateServingEndpoint == nil {
|
||||
diags = diags.Extend(diag.Errorf("model serving endpoint %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
e.Name = normalizePrefix(prefix) + e.Name
|
||||
for i := range r.ModelServingEndpoints {
|
||||
r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name
|
||||
|
||||
// As of 2024-06, model serving endpoints don't yet support tags
|
||||
}
|
||||
|
||||
// Registered models presets: Prefix
|
||||
for key, m := range r.RegisteredModels {
|
||||
if m.CreateRegisteredModelRequest == nil {
|
||||
diags = diags.Extend(diag.Errorf("registered model %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
m.Name = normalizePrefix(prefix) + m.Name
|
||||
for i := range r.RegisteredModels {
|
||||
r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name
|
||||
|
||||
// As of 2024-06, registered models don't yet support tags
|
||||
}
|
||||
|
||||
// Quality monitors presets: Schedule
|
||||
// Quality monitors presets: Prefix
|
||||
if t.TriggerPauseStatus == config.Paused {
|
||||
for key, q := range r.QualityMonitors {
|
||||
if q.CreateMonitor == nil {
|
||||
diags = diags.Extend(diag.Errorf("quality monitor %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
for i := range r.QualityMonitors {
|
||||
// Remove all schedules from monitors, since they don't support pausing/unpausing.
|
||||
// Quality monitors might support the "pause" property in the future, so at the
|
||||
// CLI level we do respect that property if it is set to "unpaused."
|
||||
if q.Schedule != nil && q.Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused {
|
||||
q.Schedule = nil
|
||||
if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused {
|
||||
r.QualityMonitors[i].Schedule = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Schemas: Prefix
|
||||
for key, s := range r.Schemas {
|
||||
if s.CreateSchema == nil {
|
||||
diags = diags.Extend(diag.Errorf("schema %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
s.Name = normalizePrefix(prefix) + s.Name
|
||||
for i := range r.Schemas {
|
||||
r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name
|
||||
// HTTP API for schemas doesn't yet support tags. It's only supported in
|
||||
// the Databricks UI and via the SQL API.
|
||||
}
|
||||
|
||||
// Clusters: Prefix, Tags
|
||||
for key, c := range r.Clusters {
|
||||
if c.ClusterSpec == nil {
|
||||
diags = diags.Extend(diag.Errorf("cluster %s is not defined", key))
|
||||
continue
|
||||
}
|
||||
c.ClusterName = prefix + c.ClusterName
|
||||
if c.CustomTags == nil {
|
||||
c.CustomTags = make(map[string]string)
|
||||
}
|
||||
for _, tag := range tags {
|
||||
normalisedKey := b.Tagging.NormalizeKey(tag.Key)
|
||||
normalisedValue := b.Tagging.NormalizeValue(tag.Value)
|
||||
if _, ok := c.CustomTags[normalisedKey]; !ok {
|
||||
c.CustomTags[normalisedKey] = normalisedValue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Dashboards: Prefix
|
||||
for i := range r.Dashboards {
|
||||
r.Dashboards[i].DisplayName = prefix + r.Dashboards[i].DisplayName
|
||||
}
|
||||
|
||||
return diags
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics {
|
||||
|
|
|
@ -251,116 +251,3 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job1": {}, // no jobsettings inside
|
||||
},
|
||||
},
|
||||
Presets: config.Presets{
|
||||
NamePrefix: "prefix-",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.ApplyPresets())
|
||||
|
||||
require.ErrorContains(t, diags.Error(), "job job1 is not defined")
|
||||
}
|
||||
|
||||
func TestApplyPresetsResourceNotDefined(t *testing.T) {
|
||||
tests := []struct {
|
||||
resources config.Resources
|
||||
error string
|
||||
}{
|
||||
{
|
||||
resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job1": {}, // no jobsettings inside
|
||||
},
|
||||
},
|
||||
error: "job job1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
Pipelines: map[string]*resources.Pipeline{
|
||||
"pipeline1": {}, // no pipelinespec inside
|
||||
},
|
||||
},
|
||||
error: "pipeline pipeline1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
Models: map[string]*resources.MlflowModel{
|
||||
"model1": {}, // no model inside
|
||||
},
|
||||
},
|
||||
error: "model model1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
Experiments: map[string]*resources.MlflowExperiment{
|
||||
"experiment1": {}, // no experiment inside
|
||||
},
|
||||
},
|
||||
error: "experiment experiment1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
|
||||
"endpoint1": {}, // no CreateServingEndpoint inside
|
||||
},
|
||||
RegisteredModels: map[string]*resources.RegisteredModel{
|
||||
"model1": {}, // no CreateRegisteredModelRequest inside
|
||||
},
|
||||
},
|
||||
error: "model serving endpoint endpoint1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
QualityMonitors: map[string]*resources.QualityMonitor{
|
||||
"monitor1": {}, // no CreateMonitor inside
|
||||
},
|
||||
},
|
||||
error: "quality monitor monitor1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
Schemas: map[string]*resources.Schema{
|
||||
"schema1": {}, // no CreateSchema inside
|
||||
},
|
||||
},
|
||||
error: "schema schema1 is not defined",
|
||||
},
|
||||
{
|
||||
resources: config.Resources{
|
||||
Clusters: map[string]*resources.Cluster{
|
||||
"cluster1": {}, // no ClusterSpec inside
|
||||
},
|
||||
},
|
||||
error: "cluster cluster1 is not defined",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.error, func(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Resources: tt.resources,
|
||||
Presets: config.Presets{
|
||||
TriggerPauseStatus: config.Paused,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.ApplyPresets())
|
||||
|
||||
require.ErrorContains(t, diags.Error(), tt.error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,87 +0,0 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type computeIdToClusterId struct{}
|
||||
|
||||
func ComputeIdToClusterId() bundle.Mutator {
|
||||
return &computeIdToClusterId{}
|
||||
}
|
||||
|
||||
func (m *computeIdToClusterId) Name() string {
|
||||
return "ComputeIdToClusterId"
|
||||
}
|
||||
|
||||
func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
var diags diag.Diagnostics
|
||||
|
||||
// The "compute_id" key is set; rewrite it to "cluster_id".
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle")))
|
||||
diags = diags.Extend(d)
|
||||
|
||||
// Check if the "compute_id" key is set in any target overrides.
|
||||
return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
v, d := rewriteComputeIdToClusterId(v, dyn.Path{})
|
||||
diags = diags.Extend(d)
|
||||
return v, nil
|
||||
})
|
||||
})
|
||||
|
||||
diags = diags.Extend(diag.FromErr(err))
|
||||
return diags
|
||||
}
|
||||
|
||||
func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) {
|
||||
var diags diag.Diagnostics
|
||||
computeIdPath := p.Append(dyn.Key("compute_id"))
|
||||
computeId, err := dyn.GetByPath(v, computeIdPath)
|
||||
|
||||
// If the "compute_id" key is not set, we don't need to do anything.
|
||||
if err != nil {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
if computeId.Kind() == dyn.KindInvalid {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Warning,
|
||||
Summary: "compute_id is deprecated, please use cluster_id instead",
|
||||
Locations: computeId.Locations(),
|
||||
Paths: []dyn.Path{computeIdPath},
|
||||
})
|
||||
|
||||
clusterIdPath := p.Append(dyn.Key("cluster_id"))
|
||||
nv, err := dyn.SetByPath(v, clusterIdPath, computeId)
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, diag.FromErr(err)
|
||||
}
|
||||
// Drop the "compute_id" key.
|
||||
vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
switch len(p) {
|
||||
case 0:
|
||||
return v, nil
|
||||
case 1:
|
||||
if p[0] == dyn.Key("compute_id") {
|
||||
return v, dyn.ErrDrop
|
||||
}
|
||||
return v, nil
|
||||
case 2:
|
||||
if p[1] == dyn.Key("compute_id") {
|
||||
return v, dyn.ErrDrop
|
||||
}
|
||||
}
|
||||
return v, dyn.ErrSkip
|
||||
})
|
||||
|
||||
diags = diags.Extend(diag.FromErr(err))
|
||||
return vout, diags
|
||||
}
|
|
@ -1,57 +0,0 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestComputeIdToClusterId(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
ComputeId: "compute-id",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Equal(t, "compute-id", b.Config.Bundle.ClusterId)
|
||||
assert.Empty(t, b.Config.Bundle.ComputeId)
|
||||
|
||||
assert.Len(t, diags, 1)
|
||||
assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary)
|
||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
||||
}
|
||||
|
||||
func TestComputeIdToClusterIdInTargetOverride(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Targets: map[string]*config.Target{
|
||||
"dev": {
|
||||
ComputeId: "compute-id-dev",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId())
|
||||
assert.NoError(t, diags.Error())
|
||||
assert.Empty(t, b.Config.Targets["dev"].ComputeId)
|
||||
|
||||
diags = diags.Extend(bundle.Apply(context.Background(), b, mutator.SelectTarget("dev")))
|
||||
assert.NoError(t, diags.Error())
|
||||
|
||||
assert.Equal(t, "compute-id-dev", b.Config.Bundle.ClusterId)
|
||||
assert.Empty(t, b.Config.Bundle.ComputeId)
|
||||
|
||||
assert.Len(t, diags, 1)
|
||||
assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary)
|
||||
assert.Equal(t, diag.Warning, diags[0].Severity)
|
||||
}
|
|
@ -1,70 +0,0 @@
|
|||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type configureDashboardDefaults struct{}
|
||||
|
||||
func ConfigureDashboardDefaults() bundle.Mutator {
|
||||
return &configureDashboardDefaults{}
|
||||
}
|
||||
|
||||
func (m *configureDashboardDefaults) Name() string {
|
||||
return "ConfigureDashboardDefaults"
|
||||
}
|
||||
|
||||
func (m *configureDashboardDefaults) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
var diags diag.Diagnostics
|
||||
|
||||
pattern := dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("dashboards"),
|
||||
dyn.AnyKey(),
|
||||
)
|
||||
|
||||
// Configure defaults for all dashboards.
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
var err error
|
||||
v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("parent_path")), dyn.V(b.Config.Workspace.ResourcePath))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("embed_credentials")), dyn.V(false))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
return v, nil
|
||||
})
|
||||
})
|
||||
|
||||
diags = diags.Extend(diag.FromErr(err))
|
||||
return diags
|
||||
}
|
||||
|
||||
func setIfNotExists(v dyn.Value, path dyn.Path, defaultValue dyn.Value) (dyn.Value, error) {
|
||||
// Get the field at the specified path (if set).
|
||||
_, err := dyn.GetByPath(v, path)
|
||||
switch {
|
||||
case dyn.IsNoSuchKeyError(err):
|
||||
// OK, we'll set the default value.
|
||||
break
|
||||
case dyn.IsCannotTraverseNilError(err):
|
||||
// Cannot traverse the value, skip it.
|
||||
return v, nil
|
||||
case err == nil:
|
||||
// The field is set, skip it.
|
||||
return v, nil
|
||||
default:
|
||||
// Return the error.
|
||||
return v, err
|
||||
}
|
||||
|
||||
// Set the field at the specified path.
|
||||
return dyn.SetByPath(v, path, defaultValue)
|
||||
}
|
|
@ -1,125 +0,0 @@
|
|||
package mutator_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConfigureDashboardDefaultsParentPath(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
ResourcePath: "/foo/bar",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Dashboards: map[string]*resources.Dashboard{
|
||||
"d1": {
|
||||
// Empty string is skipped.
|
||||
// See below for how it is set.
|
||||
ParentPath: "",
|
||||
},
|
||||
"d2": {
|
||||
// Non-empty string is skipped.
|
||||
ParentPath: "already-set",
|
||||
},
|
||||
"d3": {
|
||||
// No parent path set.
|
||||
},
|
||||
"d4": nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// We can't set an empty string in the typed configuration.
|
||||
// Do it on the dyn.Value directly.
|
||||
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||
return dyn.Set(v, "resources.dashboards.d1.parent_path", dyn.V(""))
|
||||
})
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults())
|
||||
require.NoError(t, diags.Error())
|
||||
|
||||
var v dyn.Value
|
||||
var err error
|
||||
|
||||
// Set to empty string; unchanged.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.parent_path")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, "", v.MustString())
|
||||
}
|
||||
|
||||
// Set to "already-set"; unchanged.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.parent_path")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, "already-set", v.MustString())
|
||||
}
|
||||
|
||||
// Not set; now set to the workspace resource path.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.parent_path")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, "/foo/bar", v.MustString())
|
||||
}
|
||||
|
||||
// No valid dashboard; no change.
|
||||
_, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.parent_path")
|
||||
assert.True(t, dyn.IsCannotTraverseNilError(err))
|
||||
}
|
||||
|
||||
func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Dashboards: map[string]*resources.Dashboard{
|
||||
"d1": {
|
||||
EmbedCredentials: true,
|
||||
},
|
||||
"d2": {
|
||||
EmbedCredentials: false,
|
||||
},
|
||||
"d3": {
|
||||
// No parent path set.
|
||||
},
|
||||
"d4": nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults())
|
||||
require.NoError(t, diags.Error())
|
||||
|
||||
var v dyn.Value
|
||||
var err error
|
||||
|
||||
// Set to true; still true.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, true, v.MustBool())
|
||||
}
|
||||
|
||||
// Set to false; still false.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, false, v.MustBool())
|
||||
}
|
||||
|
||||
// Not set; now false.
|
||||
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials")
|
||||
if assert.NoError(t, err) {
|
||||
assert.Equal(t, false, v.MustBool())
|
||||
}
|
||||
|
||||
// No valid dashboard; no change.
|
||||
_, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.embed_credentials")
|
||||
assert.True(t, dyn.IsCannotTraverseNilError(err))
|
||||
}
|
|
@ -29,10 +29,6 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl
|
|||
b.Config.Workspace.FilePath = path.Join(root, "files")
|
||||
}
|
||||
|
||||
if b.Config.Workspace.ResourcePath == "" {
|
||||
b.Config.Workspace.ResourcePath = path.Join(root, "resources")
|
||||
}
|
||||
|
||||
if b.Config.Workspace.ArtifactPath == "" {
|
||||
b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts")
|
||||
}
|
||||
|
|
|
@ -22,7 +22,6 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) {
|
|||
diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
assert.Equal(t, "/files", b.Config.Workspace.FilePath)
|
||||
assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath)
|
||||
assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath)
|
||||
assert.Equal(t, "/state", b.Config.Workspace.StatePath)
|
||||
}
|
||||
|
@ -33,7 +32,6 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) {
|
|||
Workspace: config.Workspace{
|
||||
RootPath: "/",
|
||||
FilePath: "/foo/bar",
|
||||
ResourcePath: "/foo/bar",
|
||||
ArtifactPath: "/foo/bar",
|
||||
StatePath: "/foo/bar",
|
||||
},
|
||||
|
@ -42,7 +40,6 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) {
|
|||
diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath)
|
||||
assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath)
|
||||
assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath)
|
||||
assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath)
|
||||
}
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -42,7 +41,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "skip/test7.py"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: dir,
|
||||
RootPath: dir,
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Pipelines: map[string]*resources.Pipeline{
|
||||
|
@ -106,8 +105,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []dyn.Location{{File: filepath.Join(dir, "relative", "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", filepath.Join(dir, "relative", "resource.yml"))
|
||||
|
||||
m := ExpandPipelineGlobPaths()
|
||||
diags := bundle.Apply(context.Background(), b, m)
|
||||
|
|
|
@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn
|
|||
}
|
||||
|
||||
// Compute relative path of the bundle root from the Git repo root.
|
||||
absBundlePath, err := filepath.Abs(b.BundleRootPath)
|
||||
absBundlePath, err := filepath.Abs(b.RootPath)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
|
|
@ -23,7 +23,6 @@ func DefaultMutators() []bundle.Mutator {
|
|||
VerifyCliVersion(),
|
||||
|
||||
EnvironmentsToTargets(),
|
||||
ComputeIdToClusterId(),
|
||||
InitializeVariables(),
|
||||
DefineDefaultTarget(),
|
||||
LoadGitDetails(),
|
||||
|
|
|
@ -39,22 +39,22 @@ func overrideJobCompute(j *resources.Job, compute string) {
|
|||
|
||||
func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
if b.Config.Bundle.Mode != config.Development {
|
||||
if b.Config.Bundle.ClusterId != "" {
|
||||
if b.Config.Bundle.ComputeID != "" {
|
||||
return diag.Errorf("cannot override compute for an target that does not use 'mode: development'")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" {
|
||||
b.Config.Bundle.ClusterId = v
|
||||
b.Config.Bundle.ComputeID = v
|
||||
}
|
||||
|
||||
if b.Config.Bundle.ClusterId == "" {
|
||||
if b.Config.Bundle.ComputeID == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
r := b.Config.Resources
|
||||
for i := range r.Jobs {
|
||||
overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId)
|
||||
overrideJobCompute(r.Jobs[i], b.Config.Bundle.ComputeID)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -20,7 +20,7 @@ func TestOverrideDevelopment(t *testing.T) {
|
|||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Mode: config.Development,
|
||||
ClusterId: "newClusterID",
|
||||
ComputeID: "newClusterID",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
|
@ -144,7 +144,7 @@ func TestOverrideProduction(t *testing.T) {
|
|||
b := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
ClusterId: "newClusterID",
|
||||
ComputeID: "newClusterID",
|
||||
},
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
|
|
|
@ -1,115 +0,0 @@
|
|||
package paths
|
||||
|
||||
import (
|
||||
"github.com/databricks/cli/bundle/libraries"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type jobRewritePattern struct {
|
||||
pattern dyn.Pattern
|
||||
kind PathKind
|
||||
skipRewrite func(string) bool
|
||||
}
|
||||
|
||||
func noSkipRewrite(string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern {
|
||||
return []jobRewritePattern{
|
||||
{
|
||||
base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")),
|
||||
PathKindNotebook,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")),
|
||||
PathKindWorkspaceFile,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")),
|
||||
PathKindDirectory,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")),
|
||||
PathKindWorkspaceFile,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")),
|
||||
PathKindLibrary,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")),
|
||||
PathKindLibrary,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
|
||||
PathKindWorkspaceFile,
|
||||
noSkipRewrite,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func jobRewritePatterns() []jobRewritePattern {
|
||||
// Base pattern to match all tasks in all jobs.
|
||||
base := dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("jobs"),
|
||||
dyn.AnyKey(),
|
||||
dyn.Key("tasks"),
|
||||
dyn.AnyIndex(),
|
||||
)
|
||||
|
||||
// Compile list of patterns and their respective rewrite functions.
|
||||
jobEnvironmentsPatterns := []jobRewritePattern{
|
||||
{
|
||||
dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("jobs"),
|
||||
dyn.AnyKey(),
|
||||
dyn.Key("environments"),
|
||||
dyn.AnyIndex(),
|
||||
dyn.Key("spec"),
|
||||
dyn.Key("dependencies"),
|
||||
dyn.AnyIndex(),
|
||||
),
|
||||
PathKindWithPrefix,
|
||||
func(s string) bool {
|
||||
return !libraries.IsLibraryLocal(s)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
taskPatterns := jobTaskRewritePatterns(base)
|
||||
forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
|
||||
allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
|
||||
allPatterns = append(allPatterns, forEachPatterns...)
|
||||
return allPatterns
|
||||
}
|
||||
|
||||
// VisitJobPaths visits all paths in job resources and applies a function to each path.
|
||||
func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
|
||||
var err error
|
||||
var newValue = value
|
||||
|
||||
for _, rewritePattern := range jobRewritePatterns() {
|
||||
newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
if rewritePattern.skipRewrite(v.MustString()) {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
return fn(p, rewritePattern.kind, v)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
}
|
||||
|
||||
return newValue, nil
|
||||
}
|
|
@ -1,168 +0,0 @@
|
|||
package paths
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
assert "github.com/databricks/cli/libs/dyn/dynassert"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVisitJobPaths(t *testing.T) {
|
||||
task0 := jobs.Task{
|
||||
NotebookTask: &jobs.NotebookTask{
|
||||
NotebookPath: "abc",
|
||||
},
|
||||
}
|
||||
task1 := jobs.Task{
|
||||
SparkPythonTask: &jobs.SparkPythonTask{
|
||||
PythonFile: "abc",
|
||||
},
|
||||
}
|
||||
task2 := jobs.Task{
|
||||
DbtTask: &jobs.DbtTask{
|
||||
ProjectDirectory: "abc",
|
||||
},
|
||||
}
|
||||
task3 := jobs.Task{
|
||||
SqlTask: &jobs.SqlTask{
|
||||
File: &jobs.SqlTaskFile{
|
||||
Path: "abc",
|
||||
},
|
||||
},
|
||||
}
|
||||
task4 := jobs.Task{
|
||||
Libraries: []compute.Library{
|
||||
{Whl: "dist/foo.whl"},
|
||||
},
|
||||
}
|
||||
task5 := jobs.Task{
|
||||
Libraries: []compute.Library{
|
||||
{Jar: "dist/foo.jar"},
|
||||
},
|
||||
}
|
||||
task6 := jobs.Task{
|
||||
Libraries: []compute.Library{
|
||||
{Requirements: "requirements.txt"},
|
||||
},
|
||||
}
|
||||
|
||||
job0 := &resources.Job{
|
||||
JobSettings: &jobs.JobSettings{
|
||||
Tasks: []jobs.Task{
|
||||
task0,
|
||||
task1,
|
||||
task2,
|
||||
task3,
|
||||
task4,
|
||||
task5,
|
||||
task6,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
root := config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job0": job0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
actual := visitJobPaths(t, root)
|
||||
expected := []dyn.Path{
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"),
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expected, actual)
|
||||
}
|
||||
|
||||
func TestVisitJobPaths_environments(t *testing.T) {
|
||||
environment0 := jobs.JobEnvironment{
|
||||
Spec: &compute.Environment{
|
||||
Dependencies: []string{
|
||||
"dist_0/*.whl",
|
||||
"dist_1/*.whl",
|
||||
},
|
||||
},
|
||||
}
|
||||
job0 := &resources.Job{
|
||||
JobSettings: &jobs.JobSettings{
|
||||
Environments: []jobs.JobEnvironment{
|
||||
environment0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
root := config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job0": job0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
actual := visitJobPaths(t, root)
|
||||
expected := []dyn.Path{
|
||||
dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"),
|
||||
dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"),
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expected, actual)
|
||||
}
|
||||
|
||||
func TestVisitJobPaths_foreach(t *testing.T) {
|
||||
task0 := jobs.Task{
|
||||
ForEachTask: &jobs.ForEachTask{
|
||||
Task: jobs.Task{
|
||||
NotebookTask: &jobs.NotebookTask{
|
||||
NotebookPath: "abc",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
job0 := &resources.Job{
|
||||
JobSettings: &jobs.JobSettings{
|
||||
Tasks: []jobs.Task{
|
||||
task0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
root := config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: map[string]*resources.Job{
|
||||
"job0": job0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
actual := visitJobPaths(t, root)
|
||||
expected := []dyn.Path{
|
||||
dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"),
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expected, actual)
|
||||
}
|
||||
|
||||
func visitJobPaths(t *testing.T, root config.Root) []dyn.Path {
|
||||
var actual []dyn.Path
|
||||
err := root.Mutate(func(value dyn.Value) (dyn.Value, error) {
|
||||
return VisitJobPaths(value, func(p dyn.Path, kind PathKind, v dyn.Value) (dyn.Value, error) {
|
||||
actual = append(actual, p)
|
||||
return v, nil
|
||||
})
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return actual
|
||||
}
|
|
@ -1,26 +0,0 @@
|
|||
package paths
|
||||
|
||||
import "github.com/databricks/cli/libs/dyn"
|
||||
|
||||
type PathKind int
|
||||
|
||||
const (
|
||||
// PathKindLibrary is a path to a library file
|
||||
PathKindLibrary = iota
|
||||
|
||||
// PathKindNotebook is a path to a notebook file
|
||||
PathKindNotebook
|
||||
|
||||
// PathKindWorkspaceFile is a path to a regular workspace file,
|
||||
// notebooks are not allowed because they are uploaded a special
|
||||
// kind of workspace object.
|
||||
PathKindWorkspaceFile
|
||||
|
||||
// PathKindWithPrefix is a path that starts with './'
|
||||
PathKindWithPrefix
|
||||
|
||||
// PathKindDirectory is a path to directory
|
||||
PathKindDirectory
|
||||
)
|
||||
|
||||
type VisitFunc func(path dyn.Path, kind PathKind, value dyn.Value) (dyn.Value, error)
|
|
@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag.
|
|||
}
|
||||
|
||||
b.Config.Workspace.CurrentUser = &config.User{
|
||||
ShortName: auth.GetShortUserName(me),
|
||||
ShortName: auth.GetShortUserName(me.UserName),
|
||||
User: me,
|
||||
}
|
||||
|
||||
|
|
|
@ -118,18 +118,15 @@ func findNonUserPath(b *bundle.Bundle) string {
|
|||
if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) {
|
||||
return "root_path"
|
||||
}
|
||||
if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) {
|
||||
return "state_path"
|
||||
}
|
||||
if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) {
|
||||
return "file_path"
|
||||
}
|
||||
if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) {
|
||||
return "resource_path"
|
||||
}
|
||||
if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) {
|
||||
return "artifact_path"
|
||||
}
|
||||
if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) {
|
||||
return "state_path"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@ import (
|
|||
"github.com/databricks/cli/libs/tags"
|
||||
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/databricks/databricks-sdk-go/service/iam"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
"github.com/databricks/databricks-sdk-go/service/ml"
|
||||
|
@ -120,9 +119,6 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
|||
Schemas: map[string]*resources.Schema{
|
||||
"schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}},
|
||||
},
|
||||
Clusters: map[string]*resources.Cluster{
|
||||
"cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}},
|
||||
},
|
||||
},
|
||||
},
|
||||
// Use AWS implementation for testing.
|
||||
|
@ -181,9 +177,6 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
|
|||
|
||||
// Schema 1
|
||||
assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name)
|
||||
|
||||
// Clusters
|
||||
assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
||||
}
|
||||
|
||||
func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) {
|
||||
|
@ -288,7 +281,6 @@ func TestProcessTargetModeDefault(t *testing.T) {
|
|||
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
||||
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
||||
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
||||
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
||||
}
|
||||
|
||||
func TestProcessTargetModeProduction(t *testing.T) {
|
||||
|
@ -320,7 +312,6 @@ func TestProcessTargetModeProduction(t *testing.T) {
|
|||
b.Config.Resources.Experiments["experiment2"].Permissions = permissions
|
||||
b.Config.Resources.Models["model1"].Permissions = permissions
|
||||
b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Permissions = permissions
|
||||
b.Config.Resources.Clusters["cluster1"].Permissions = permissions
|
||||
|
||||
diags = validateProductionMode(context.Background(), b, false)
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -331,7 +322,6 @@ func TestProcessTargetModeProduction(t *testing.T) {
|
|||
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
|
||||
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
|
||||
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
|
||||
assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
|
||||
}
|
||||
|
||||
func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
|
||||
|
|
|
@ -108,7 +108,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
|
|||
return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err)
|
||||
}
|
||||
|
||||
rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.BundleRootPath, pythonPath, leftRoot)
|
||||
rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.RootPath, pythonPath, leftRoot)
|
||||
mutateDiags = diags
|
||||
if diags.HasError() {
|
||||
return dyn.InvalidValue, mutateDiagsHasError
|
||||
|
|
|
@ -45,15 +45,15 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc {
|
|||
func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||
return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) {
|
||||
v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
|
||||
v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
|
||||
v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath)))
|
||||
v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.RootPath)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
|
|
|
@ -9,13 +9,12 @@ import (
|
|||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRewriteSyncPathsRelative(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: ".",
|
||||
RootPath: ".",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -34,12 +33,12 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "./databricks.yml"}})
|
||||
bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "./databricks.yml"}})
|
||||
bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "./file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "./a/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "./a/b/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "./a/b/c/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.include[0]", "./file.yml")
|
||||
bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[1]", "./a/b/c/file.yml")
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
|
||||
assert.NoError(t, diags.Error())
|
||||
|
@ -54,7 +53,7 @@ func TestRewriteSyncPathsRelative(t *testing.T) {
|
|||
|
||||
func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/dir",
|
||||
RootPath: "/tmp/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -73,12 +72,12 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}})
|
||||
bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}})
|
||||
bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "/tmp/dir/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "/tmp/dir/a/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "/tmp/dir/a/b/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "/tmp/dir/a/b/c/file.yml"}})
|
||||
bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml")
|
||||
bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml")
|
||||
bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml")
|
||||
bundletest.SetLocation(b, "sync.exclude[1]", "/tmp/dir/a/b/c/file.yml")
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
|
||||
assert.NoError(t, diags.Error())
|
||||
|
@ -94,7 +93,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) {
|
|||
func TestRewriteSyncPathsErrorPaths(t *testing.T) {
|
||||
t.Run("no sync block", func(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: ".",
|
||||
RootPath: ".",
|
||||
}
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths())
|
||||
|
@ -103,7 +102,7 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) {
|
|||
|
||||
t.Run("empty include/exclude blocks", func(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: ".",
|
||||
RootPath: ".",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Include: []string{},
|
||||
|
|
|
@ -32,7 +32,6 @@ func allResourceTypes(t *testing.T) []string {
|
|||
// the dyn library gives us the correct list of all resources supported. Please
|
||||
// also update this check when adding a new resource
|
||||
require.Equal(t, []string{
|
||||
"clusters",
|
||||
"experiments",
|
||||
"jobs",
|
||||
"model_serving_endpoints",
|
||||
|
@ -134,7 +133,6 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
|||
// some point in the future. These resources are (implicitly) on the deny list, since
|
||||
// they are not on the allow list below.
|
||||
allowList := []string{
|
||||
"clusters",
|
||||
"jobs",
|
||||
"models",
|
||||
"registered_models",
|
||||
|
|
|
@ -15,8 +15,8 @@ import (
|
|||
|
||||
func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
@ -51,8 +51,8 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) {
|
|||
for _, tcase := range tcases {
|
||||
t.Run(tcase.name, func(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{},
|
||||
}
|
||||
|
||||
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
|
|
|
@ -57,7 +57,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
|
|||
var diags diag.Diagnostics
|
||||
|
||||
// Use the bundle root path as the starting point for inferring the sync root path.
|
||||
bundleRootPath := filepath.Clean(b.BundleRootPath)
|
||||
bundleRootPath := filepath.Clean(b.RootPath)
|
||||
|
||||
// Infer the sync root path by looking at each one of the sync paths.
|
||||
// Every sync path must be a descendant of the final sync root path.
|
||||
|
|
|
@ -9,14 +9,13 @@ import (
|
|||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/mutator"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir",
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -47,7 +46,7 @@ func TestSyncInferRoot_NominalAbsolute(t *testing.T) {
|
|||
|
||||
func TestSyncInferRoot_NominalRelative(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "./some/dir",
|
||||
RootPath: "./some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -78,7 +77,7 @@ func TestSyncInferRoot_NominalRelative(t *testing.T) {
|
|||
|
||||
func TestSyncInferRoot_ParentDirectory(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir",
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -109,7 +108,7 @@ func TestSyncInferRoot_ParentDirectory(t *testing.T) {
|
|||
|
||||
func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir/that/is/very/deeply/nested",
|
||||
RootPath: "/tmp/some/dir/that/is/very/deeply/nested",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -146,7 +145,7 @@ func TestSyncInferRoot_ManyParentDirectories(t *testing.T) {
|
|||
|
||||
func TestSyncInferRoot_MultiplePaths(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/bundle/root",
|
||||
RootPath: "/tmp/some/bundle/root",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -173,7 +172,7 @@ func TestSyncInferRoot_MultiplePaths(t *testing.T) {
|
|||
|
||||
func TestSyncInferRoot_Error(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: "/tmp/some/dir",
|
||||
RootPath: "/tmp/some/dir",
|
||||
Config: config.Root{
|
||||
Sync: config.Sync{
|
||||
Paths: []string{
|
||||
|
@ -185,7 +184,7 @@ func TestSyncInferRoot_Error(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "sync.paths", []dyn.Location{{File: "databricks.yml"}})
|
||||
bundletest.SetLocation(b, "sync.paths", "databricks.yml")
|
||||
|
||||
ctx := context.Background()
|
||||
diags := bundle.Apply(ctx, b, mutator.SyncInferRoot())
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package trampoline
|
||||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
@ -23,7 +23,6 @@ type TrampolineFunctions interface {
|
|||
GetTasks(b *bundle.Bundle) []TaskWithJobKey
|
||||
CleanUp(task *jobs.Task) error
|
||||
}
|
||||
|
||||
type trampoline struct {
|
||||
name string
|
||||
functions TrampolineFunctions
|
|
@ -1,4 +1,4 @@
|
|||
package trampoline
|
||||
package mutator
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
@ -56,8 +56,8 @@ func TestGenerateTrampoline(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Workspace: config.Workspace{
|
||||
FilePath: "/Workspace/files",
|
|
@ -6,23 +6,45 @@ import (
|
|||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, error) {
|
||||
// Convert the `file_path` field to a local absolute path.
|
||||
// Terraform will load the file at this path and use its contents for the dashboard contents.
|
||||
pattern := dyn.NewPattern(
|
||||
type dashboardRewritePattern struct {
|
||||
pattern dyn.Pattern
|
||||
fn rewriteFunc
|
||||
}
|
||||
|
||||
func (t *translateContext) dashboardRewritePatterns() []dashboardRewritePattern {
|
||||
// Base pattern to match all dashboards.
|
||||
base := dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("dashboards"),
|
||||
dyn.AnyKey(),
|
||||
dyn.Key("file_path"),
|
||||
)
|
||||
|
||||
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
key := p[2].Key()
|
||||
dir, err := v.Location().Directory()
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for dashboard %s: %w", key, err)
|
||||
}
|
||||
|
||||
return t.rewriteRelativeTo(p, v, t.retainLocalAbsoluteFilePath, dir, "")
|
||||
})
|
||||
// Compile list of configuration paths to rewrite.
|
||||
return []dashboardRewritePattern{
|
||||
{
|
||||
base.Append(dyn.Key("definition_path")),
|
||||
t.retainLocalAbsoluteFilePath,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, error) {
|
||||
var err error
|
||||
|
||||
for _, rewritePattern := range t.dashboardRewritePatterns() {
|
||||
v, err = dyn.MapByPattern(v, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
key := p[1].Key()
|
||||
dir, err := v.Location().Directory()
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for dashboard %s: %w", key, err)
|
||||
}
|
||||
|
||||
return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, "")
|
||||
})
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
|
|
@ -4,11 +4,97 @@ import (
|
|||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/databricks/cli/bundle/config/mutator/paths"
|
||||
|
||||
"github.com/databricks/cli/bundle/libraries"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
)
|
||||
|
||||
type jobRewritePattern struct {
|
||||
pattern dyn.Pattern
|
||||
fn rewriteFunc
|
||||
skipRewrite func(string) bool
|
||||
}
|
||||
|
||||
func noSkipRewrite(string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern {
|
||||
return []jobRewritePattern{
|
||||
{
|
||||
base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")),
|
||||
t.translateNotebookPath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")),
|
||||
t.translateFilePath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")),
|
||||
t.translateDirectoryPath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")),
|
||||
t.translateFilePath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")),
|
||||
t.translateNoOp,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")),
|
||||
t.translateNoOp,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
|
||||
t.translateFilePath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *translateContext) jobRewritePatterns() []jobRewritePattern {
|
||||
// Base pattern to match all tasks in all jobs.
|
||||
base := dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("jobs"),
|
||||
dyn.AnyKey(),
|
||||
dyn.Key("tasks"),
|
||||
dyn.AnyIndex(),
|
||||
)
|
||||
|
||||
// Compile list of patterns and their respective rewrite functions.
|
||||
jobEnvironmentsPatterns := []jobRewritePattern{
|
||||
{
|
||||
dyn.NewPattern(
|
||||
dyn.Key("resources"),
|
||||
dyn.Key("jobs"),
|
||||
dyn.AnyKey(),
|
||||
dyn.Key("environments"),
|
||||
dyn.AnyIndex(),
|
||||
dyn.Key("spec"),
|
||||
dyn.Key("dependencies"),
|
||||
dyn.AnyIndex(),
|
||||
),
|
||||
t.translateNoOpWithPrefix,
|
||||
func(s string) bool {
|
||||
return !libraries.IsLibraryLocal(s)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
taskPatterns := rewritePatterns(t, base)
|
||||
forEachPatterns := rewritePatterns(t, base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
|
||||
allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
|
||||
allPatterns = append(allPatterns, forEachPatterns...)
|
||||
return allPatterns
|
||||
}
|
||||
|
||||
func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) {
|
||||
var err error
|
||||
|
||||
|
@ -25,41 +111,30 @@ func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error)
|
|||
}
|
||||
}
|
||||
|
||||
return paths.VisitJobPaths(v, func(p dyn.Path, kind paths.PathKind, v dyn.Value) (dyn.Value, error) {
|
||||
key := p[2].Key()
|
||||
for _, rewritePattern := range t.jobRewritePatterns() {
|
||||
v, err = dyn.MapByPattern(v, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
key := p[2].Key()
|
||||
|
||||
// Skip path translation if the job is using git source.
|
||||
if slices.Contains(ignore, key) {
|
||||
return v, nil
|
||||
}
|
||||
// Skip path translation if the job is using git source.
|
||||
if slices.Contains(ignore, key) {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
dir, err := v.Location().Directory()
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
|
||||
}
|
||||
dir, err := v.Location().Directory()
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
|
||||
}
|
||||
|
||||
rewritePatternFn, err := t.getRewritePatternFn(kind)
|
||||
sv := v.MustString()
|
||||
if rewritePattern.skipRewrite(sv) {
|
||||
return v, nil
|
||||
}
|
||||
return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, fallback[key])
|
||||
})
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
|
||||
return t.rewriteRelativeTo(p, v, rewritePatternFn, dir, fallback[key])
|
||||
})
|
||||
}
|
||||
|
||||
func (t *translateContext) getRewritePatternFn(kind paths.PathKind) (rewriteFunc, error) {
|
||||
switch kind {
|
||||
case paths.PathKindLibrary:
|
||||
return t.translateNoOp, nil
|
||||
case paths.PathKindNotebook:
|
||||
return t.translateNotebookPath, nil
|
||||
case paths.PathKindWorkspaceFile:
|
||||
return t.translateFilePath, nil
|
||||
case paths.PathKindDirectory:
|
||||
return t.translateDirectoryPath, nil
|
||||
case paths.PathKindWithPrefix:
|
||||
return t.translateNoOpWithPrefix, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported path kind: %d", kind)
|
||||
return v, nil
|
||||
}
|
||||
|
|
|
@ -82,7 +82,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -210,7 +210,7 @@ func TestTranslatePaths(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -346,8 +346,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
|
||||
bundletest.SetLocation(b, "resources.pipelines", []dyn.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}})
|
||||
bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
|
||||
bundletest.SetLocation(b, "resources.pipelines", filepath.Join(dir, "pipeline/resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -408,7 +408,7 @@ func TestTranslatePathsOutsideSyncRoot(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "../resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), "is not contained in sync root path")
|
||||
|
@ -439,7 +439,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
|
||||
|
@ -470,7 +470,7 @@ func TestJobFileDoesNotExistError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found")
|
||||
|
@ -501,7 +501,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found")
|
||||
|
@ -532,7 +532,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found")
|
||||
|
@ -567,7 +567,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), `expected a file for "resources.jobs.job.tasks[0].spark_python_task.python_file" but got a notebook`)
|
||||
|
@ -602,7 +602,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.jobs.job.tasks[0].notebook_task.notebook_path" but got a file`)
|
||||
|
@ -637,7 +637,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.pipelines.pipeline.libraries[0].notebook.path" but got a file`)
|
||||
|
@ -672,7 +672,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
assert.ErrorContains(t, diags.Error(), `expected a file for "resources.pipelines.pipeline.libraries[0].file.path" but got a notebook`)
|
||||
|
@ -710,7 +710,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
|
||||
bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||
require.NoError(t, diags.Error())
|
||||
|
@ -753,8 +753,8 @@ func TestTranslatePathWithComplexVariables(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "variables", []dyn.Location{{File: filepath.Join(dir, "variables/variables.yml")}})
|
||||
bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}})
|
||||
bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml"))
|
||||
bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml"))
|
||||
|
||||
ctx := context.Background()
|
||||
// Assign the variables to the dynamic configuration.
|
||||
|
|
|
@ -19,7 +19,6 @@ type Resources struct {
|
|||
RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"`
|
||||
QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"`
|
||||
Schemas map[string]*resources.Schema `json:"schemas,omitempty"`
|
||||
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
|
||||
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
@ -1,39 +0,0 @@
|
|||
package resources
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/databricks/cli/libs/log"
|
||||
"github.com/databricks/databricks-sdk-go"
|
||||
"github.com/databricks/databricks-sdk-go/marshal"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
)
|
||||
|
||||
type Cluster struct {
|
||||
ID string `json:"id,omitempty" bundle:"readonly"`
|
||||
Permissions []Permission `json:"permissions,omitempty"`
|
||||
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
|
||||
|
||||
*compute.ClusterSpec
|
||||
}
|
||||
|
||||
func (s *Cluster) UnmarshalJSON(b []byte) error {
|
||||
return marshal.Unmarshal(b, s)
|
||||
}
|
||||
|
||||
func (s Cluster) MarshalJSON() ([]byte, error) {
|
||||
return marshal.Marshal(s)
|
||||
}
|
||||
|
||||
func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
|
||||
_, err := w.Clusters.GetByClusterId(ctx, id)
|
||||
if err != nil {
|
||||
log.Debugf(ctx, "cluster %s does not exist", id)
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Cluster) TerraformResourceName() string {
|
||||
return "databricks_cluster"
|
||||
}
|
|
@ -18,36 +18,21 @@ type Dashboard struct {
|
|||
// === BEGIN OF API FIELDS ===
|
||||
// ===========================
|
||||
|
||||
// DisplayName is the display name of the dashboard (both as title and as basename in the workspace).
|
||||
DisplayName string `json:"display_name"`
|
||||
// DisplayName is the name of the dashboard (both as title and as basename in the workspace).
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
|
||||
// WarehouseID is the ID of the SQL Warehouse used to run the dashboard's queries.
|
||||
WarehouseID string `json:"warehouse_id"`
|
||||
|
||||
// SerializedDashboard holds the contents of the dashboard in serialized JSON form.
|
||||
// Note: its type is any and not string such that it can be inlined as YAML.
|
||||
// If it is not a string, its contents will be marshalled as JSON.
|
||||
SerializedDashboard any `json:"serialized_dashboard,omitempty"`
|
||||
|
||||
// ParentPath is the workspace path of the folder containing the dashboard.
|
||||
// Includes leading slash and no trailing slash.
|
||||
//
|
||||
// Defaults to ${workspace.resource_path} if not set.
|
||||
// ParentPath is the path to the parent directory of the dashboard.
|
||||
ParentPath string `json:"parent_path,omitempty"`
|
||||
|
||||
// EmbedCredentials is a flag to indicate if the publisher's credentials should
|
||||
// be embedded in the published dashboard. These embedded credentials will be used
|
||||
// to execute the published dashboard's queries.
|
||||
//
|
||||
// Defaults to false if not set.
|
||||
EmbedCredentials bool `json:"embed_credentials,omitempty"`
|
||||
// WarehouseID is the ID of the warehouse to use for the dashboard.
|
||||
WarehouseID string `json:"warehouse_id,omitempty"`
|
||||
|
||||
// ===========================
|
||||
// ==== END OF API FIELDS ====
|
||||
// ===========================
|
||||
|
||||
// FilePath points to the local `.lvdash.json` file containing the dashboard definition.
|
||||
FilePath string `json:"file_path,omitempty"`
|
||||
// DefinitionPath points to the local `.lvdash.json` file containing the dashboard definition.
|
||||
DefinitionPath string `json:"definition_path,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Dashboard) UnmarshalJSON(b []byte) error {
|
||||
|
@ -58,7 +43,7 @@ func (s Dashboard) MarshalJSON() ([]byte, error) {
|
|||
return marshal.Marshal(s)
|
||||
}
|
||||
|
||||
func (*Dashboard) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
|
||||
func (_ *Dashboard) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
|
||||
_, err := w.Lakeview.Get(ctx, dashboards.GetDashboardRequest{
|
||||
DashboardId: id,
|
||||
})
|
||||
|
@ -69,6 +54,6 @@ func (*Dashboard) Exists(ctx context.Context, w *databricks.WorkspaceClient, id
|
|||
return true, nil
|
||||
}
|
||||
|
||||
func (*Dashboard) TerraformResourceName() string {
|
||||
func (_ *Dashboard) TerraformResourceName() string {
|
||||
return "databricks_dashboard"
|
||||
}
|
||||
|
|
|
@ -366,9 +366,9 @@ func (r *Root) MergeTargetOverrides(name string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// Merge `cluster_id`. This field must be overwritten if set, not merged.
|
||||
if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid {
|
||||
root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v)
|
||||
// Merge `compute_id`. This field must be overwritten if set, not merged.
|
||||
if v := target.Get("compute_id"); v.Kind() != dyn.KindInvalid {
|
||||
root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("compute_id")), v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -406,45 +406,23 @@ func (r *Root) MergeTargetOverrides(name string) error {
|
|||
return r.updateWithDynamicValue(root)
|
||||
}
|
||||
|
||||
var allowedVariableDefinitions = []([]string){
|
||||
{"default", "type", "description"},
|
||||
{"default", "type"},
|
||||
{"default", "description"},
|
||||
{"lookup", "description"},
|
||||
{"default"},
|
||||
{"lookup"},
|
||||
}
|
||||
var variableKeywords = []string{"default", "lookup"}
|
||||
|
||||
// isFullVariableOverrideDef checks if the given value is a full syntax varaible override.
|
||||
// A full syntax variable override is a map with either 1 of 2 keys.
|
||||
// If it's 2 keys, the keys should be "default" and "type".
|
||||
// If it's 1 key, the key should be one of the following keys: "default", "lookup".
|
||||
// A full syntax variable override is a map with only one of the following
|
||||
// keys: "default", "lookup".
|
||||
func isFullVariableOverrideDef(v dyn.Value) bool {
|
||||
mv, ok := v.AsMap()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// If the map has more than 3 keys, it is not a full variable override.
|
||||
if mv.Len() > 3 {
|
||||
if mv.Len() != 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, keys := range allowedVariableDefinitions {
|
||||
if len(keys) != mv.Len() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the keys are the same.
|
||||
match := true
|
||||
for _, key := range keys {
|
||||
if _, ok := mv.GetByString(key); !ok {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if match {
|
||||
for _, keyword := range variableKeywords {
|
||||
if _, ok := mv.GetByString(keyword); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,7 +6,6 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle/config/variable"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -170,87 +169,3 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) {
|
|||
assert.Equal(t, "complex var", root.Variables["complex"].Description)
|
||||
|
||||
}
|
||||
|
||||
func TestIsFullVariableOverrideDef(t *testing.T) {
|
||||
testCases := []struct {
|
||||
value dyn.Value
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
"default": dyn.V("foo"),
|
||||
"description": dyn.V("foo var"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
"lookup": dyn.V("foo"),
|
||||
"description": dyn.V("foo var"),
|
||||
}),
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
"default": dyn.V("foo"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
"lookup": dyn.V("foo"),
|
||||
}),
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"description": dyn.V("string"),
|
||||
"default": dyn.V("foo"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"description": dyn.V("string"),
|
||||
"lookup": dyn.V("foo"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"default": dyn.V("foo"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"lookup": dyn.V("foo"),
|
||||
}),
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
}),
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
value: dyn.V(map[string]dyn.Value{
|
||||
"type": dyn.V("string"),
|
||||
"default": dyn.V("foo"),
|
||||
"description": dyn.V("foo var"),
|
||||
"lookup": dyn.V("foo"),
|
||||
}),
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -24,11 +24,8 @@ type Target struct {
|
|||
// name prefix of deployed resources.
|
||||
Presets Presets `json:"presets,omitempty"`
|
||||
|
||||
// DEPRECATED: Overrides the compute used for jobs and other supported assets.
|
||||
ComputeId string `json:"compute_id,omitempty"`
|
||||
|
||||
// Overrides the cluster used for jobs and other supported assets.
|
||||
ClusterId string `json:"cluster_id,omitempty"`
|
||||
// Overrides the compute used for jobs and other supported assets.
|
||||
ComputeID string `json:"compute_id,omitempty"`
|
||||
|
||||
Bundle *Bundle `json:"bundle,omitempty"`
|
||||
|
||||
|
|
|
@ -1,161 +0,0 @@
|
|||
package validate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
)
|
||||
|
||||
// JobTaskClusterSpec validates that job tasks have cluster spec defined
|
||||
// if task requires a cluster
|
||||
func JobTaskClusterSpec() bundle.ReadOnlyMutator {
|
||||
return &jobTaskClusterSpec{}
|
||||
}
|
||||
|
||||
type jobTaskClusterSpec struct {
|
||||
}
|
||||
|
||||
func (v *jobTaskClusterSpec) Name() string {
|
||||
return "validate:job_task_cluster_spec"
|
||||
}
|
||||
|
||||
func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))
|
||||
|
||||
for resourceName, job := range rb.Config().Resources.Jobs {
|
||||
resourcePath := jobsPath.Append(dyn.Key(resourceName))
|
||||
|
||||
for taskIndex, task := range job.Tasks {
|
||||
taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex))
|
||||
|
||||
diags = diags.Extend(validateJobTask(rb, task, taskPath))
|
||||
}
|
||||
}
|
||||
|
||||
return diags
|
||||
}
|
||||
|
||||
func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics {
|
||||
diags := diag.Diagnostics{}
|
||||
|
||||
var specified []string
|
||||
var unspecified []string
|
||||
|
||||
if task.JobClusterKey != "" {
|
||||
specified = append(specified, "job_cluster_key")
|
||||
} else {
|
||||
unspecified = append(unspecified, "job_cluster_key")
|
||||
}
|
||||
|
||||
if task.EnvironmentKey != "" {
|
||||
specified = append(specified, "environment_key")
|
||||
} else {
|
||||
unspecified = append(unspecified, "environment_key")
|
||||
}
|
||||
|
||||
if task.ExistingClusterId != "" {
|
||||
specified = append(specified, "existing_cluster_id")
|
||||
} else {
|
||||
unspecified = append(unspecified, "existing_cluster_id")
|
||||
}
|
||||
|
||||
if task.NewCluster != nil {
|
||||
specified = append(specified, "new_cluster")
|
||||
} else {
|
||||
unspecified = append(unspecified, "new_cluster")
|
||||
}
|
||||
|
||||
if task.ForEachTask != nil {
|
||||
forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task"))
|
||||
|
||||
diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath))
|
||||
}
|
||||
|
||||
if isComputeTask(task) && len(specified) == 0 {
|
||||
if task.NotebookTask != nil {
|
||||
// notebook tasks without cluster spec will use notebook environment
|
||||
} else {
|
||||
// path might be not very helpful, adding user-specified task key clarifies the context
|
||||
detail := fmt.Sprintf(
|
||||
"Task %q requires a cluster or an environment to run.\nSpecify one of the following fields: %s.",
|
||||
task.TaskKey,
|
||||
strings.Join(unspecified, ", "),
|
||||
)
|
||||
|
||||
diags = diags.Append(diag.Diagnostic{
|
||||
Severity: diag.Error,
|
||||
Summary: "Missing required cluster or environment settings",
|
||||
Detail: detail,
|
||||
Locations: rb.Config().GetLocations(taskPath.String()),
|
||||
Paths: []dyn.Path{taskPath},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return diags
|
||||
}
|
||||
|
||||
// isComputeTask returns true if the task runs on a cluster or serverless GC
|
||||
func isComputeTask(task jobs.Task) bool {
|
||||
if task.NotebookTask != nil {
|
||||
// if warehouse_id is set, it's SQL notebook that doesn't need cluster or serverless GC
|
||||
if task.NotebookTask.WarehouseId != "" {
|
||||
return false
|
||||
} else {
|
||||
// task settings don't require specifying a cluster/serverless GC, but task itself can run on one
|
||||
// we handle that case separately in validateJobTask
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if task.PythonWheelTask != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if task.DbtTask != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if task.SparkJarTask != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if task.SparkSubmitTask != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if task.SparkPythonTask != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if task.SqlTask != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if task.PipelineTask != nil {
|
||||
// while pipelines use clusters, pipeline tasks don't, they only trigger pipelines
|
||||
return false
|
||||
}
|
||||
|
||||
if task.RunJobTask != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if task.ConditionTask != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// for each task doesn't use clusters, underlying task(s) can though
|
||||
if task.ForEachTask != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
|
@ -1,203 +0,0 @@
|
|||
package validate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestJobTaskClusterSpec(t *testing.T) {
|
||||
expectedSummary := "Missing required cluster or environment settings"
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
task jobs.Task
|
||||
errorPath string
|
||||
errorDetail string
|
||||
errorSummary string
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
{
|
||||
name: "valid notebook task",
|
||||
task: jobs.Task{
|
||||
// while a cluster is needed, it will use notebook environment to create one
|
||||
NotebookTask: &jobs.NotebookTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid notebook task (job_cluster_key)",
|
||||
task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
NotebookTask: &jobs.NotebookTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid notebook task (new_cluster)",
|
||||
task: jobs.Task{
|
||||
NewCluster: &compute.ClusterSpec{},
|
||||
NotebookTask: &jobs.NotebookTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid notebook task (existing_cluster_id)",
|
||||
task: jobs.Task{
|
||||
ExistingClusterId: "cluster1",
|
||||
NotebookTask: &jobs.NotebookTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid SQL notebook task",
|
||||
task: jobs.Task{
|
||||
NotebookTask: &jobs.NotebookTask{
|
||||
WarehouseId: "warehouse1",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid python wheel task",
|
||||
task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
PythonWheelTask: &jobs.PythonWheelTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid python wheel task (environment_key)",
|
||||
task: jobs.Task{
|
||||
EnvironmentKey: "environment1",
|
||||
PythonWheelTask: &jobs.PythonWheelTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid dbt task",
|
||||
task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
DbtTask: &jobs.DbtTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid spark jar task",
|
||||
task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
SparkJarTask: &jobs.SparkJarTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid spark submit",
|
||||
task: jobs.Task{
|
||||
NewCluster: &compute.ClusterSpec{},
|
||||
SparkSubmitTask: &jobs.SparkSubmitTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid spark python task",
|
||||
task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
SparkPythonTask: &jobs.SparkPythonTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid SQL task",
|
||||
task: jobs.Task{
|
||||
SqlTask: &jobs.SqlTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid pipeline task",
|
||||
task: jobs.Task{
|
||||
PipelineTask: &jobs.PipelineTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid run job task",
|
||||
task: jobs.Task{
|
||||
RunJobTask: &jobs.RunJobTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid condition task",
|
||||
task: jobs.Task{
|
||||
ConditionTask: &jobs.ConditionTask{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid for each task",
|
||||
task: jobs.Task{
|
||||
ForEachTask: &jobs.ForEachTask{
|
||||
Task: jobs.Task{
|
||||
JobClusterKey: "cluster1",
|
||||
NotebookTask: &jobs.NotebookTask{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid python wheel task",
|
||||
task: jobs.Task{
|
||||
PythonWheelTask: &jobs.PythonWheelTask{},
|
||||
TaskKey: "my_task",
|
||||
},
|
||||
errorPath: "resources.jobs.job1.tasks[0]",
|
||||
errorDetail: `Task "my_task" requires a cluster or an environment to run.
|
||||
Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`,
|
||||
errorSummary: expectedSummary,
|
||||
},
|
||||
{
|
||||
name: "invalid for each task",
|
||||
task: jobs.Task{
|
||||
ForEachTask: &jobs.ForEachTask{
|
||||
Task: jobs.Task{
|
||||
PythonWheelTask: &jobs.PythonWheelTask{},
|
||||
TaskKey: "my_task",
|
||||
},
|
||||
},
|
||||
},
|
||||
errorPath: "resources.jobs.job1.tasks[0].for_each_task.task",
|
||||
errorDetail: `Task "my_task" requires a cluster or an environment to run.
|
||||
Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`,
|
||||
errorSummary: expectedSummary,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
job := &resources.Job{
|
||||
JobSettings: &jobs.JobSettings{
|
||||
Tasks: []jobs.Task{tc.task},
|
||||
},
|
||||
}
|
||||
|
||||
b := createBundle(map[string]*resources.Job{"job1": job})
|
||||
diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec())
|
||||
|
||||
if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" {
|
||||
assert.Len(t, diags, 1)
|
||||
assert.Len(t, diags[0].Paths, 1)
|
||||
|
||||
diag := diags[0]
|
||||
|
||||
assert.Equal(t, tc.errorPath, diag.Paths[0].String())
|
||||
assert.Equal(t, tc.errorSummary, diag.Summary)
|
||||
assert.Equal(t, tc.errorDetail, diag.Detail)
|
||||
} else {
|
||||
assert.ElementsMatch(t, []string{}, diags)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func createBundle(jobs map[string]*resources.Job) *bundle.Bundle {
|
||||
return &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
Resources: config.Resources{
|
||||
Jobs: jobs,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
|
@ -34,7 +34,6 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
|
|||
JobClusterKeyDefined(),
|
||||
FilesToSync(),
|
||||
ValidateSyncPatterns(),
|
||||
JobTaskClusterSpec(),
|
||||
))
|
||||
}
|
||||
|
||||
|
|
|
@ -54,11 +54,6 @@ type Workspace struct {
|
|||
// This defaults to "${workspace.root}/files".
|
||||
FilePath string `json:"file_path,omitempty"`
|
||||
|
||||
// Remote workspace path for resources with a presence in the workspace.
|
||||
// These are kept outside [FilePath] to avoid potential naming collisions.
|
||||
// This defaults to "${workspace.root}/resources".
|
||||
ResourcePath string `json:"resource_path,omitempty"`
|
||||
|
||||
// Remote workspace path for build artifacts.
|
||||
// This defaults to "${workspace.root}/artifacts".
|
||||
ArtifactPath string `json:"artifact_path,omitempty"`
|
||||
|
|
|
@ -8,12 +8,9 @@ import (
|
|||
"github.com/databricks/cli/libs/cmdio"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/log"
|
||||
"github.com/databricks/cli/libs/sync"
|
||||
)
|
||||
|
||||
type upload struct {
|
||||
outputHandler sync.OutputHandler
|
||||
}
|
||||
type upload struct{}
|
||||
|
||||
func (m *upload) Name() string {
|
||||
return "files.Upload"
|
||||
|
@ -21,18 +18,11 @@ func (m *upload) Name() string {
|
|||
|
||||
func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath))
|
||||
opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b))
|
||||
sync, err := GetSync(ctx, bundle.ReadOnly(b))
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
|
||||
opts.OutputHandler = m.outputHandler
|
||||
sync, err := sync.New(ctx, *opts)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
}
|
||||
defer sync.Close()
|
||||
|
||||
b.Files, err = sync.RunOnce(ctx)
|
||||
if err != nil {
|
||||
return diag.FromErr(err)
|
||||
|
@ -42,6 +32,6 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
|||
return nil
|
||||
}
|
||||
|
||||
func Upload(outputHandler sync.OutputHandler) bundle.Mutator {
|
||||
return &upload{outputHandler}
|
||||
func Upload() bundle.Mutator {
|
||||
return &upload{}
|
||||
}
|
||||
|
|
|
@ -40,7 +40,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
|
|||
// Compute config file path the job is defined in, relative to the bundle
|
||||
// root
|
||||
l := b.Config.GetLocation("resources.jobs." + name)
|
||||
relativePath, err := filepath.Rel(b.BundleRootPath, l.File)
|
||||
relativePath, err := filepath.Rel(b.RootPath, l.File)
|
||||
if err != nil {
|
||||
return diag.Errorf("failed to compute relative path for job %s: %v", name, err)
|
||||
}
|
||||
|
|
|
@ -9,7 +9,6 @@ import (
|
|||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/bundle/metadata"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -56,9 +55,9 @@ func TestComputeMetadataMutator(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}})
|
||||
bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}})
|
||||
bundletest.SetLocation(b, "resources.pipelines.my-pipeline", []dyn.Location{{File: "abc"}})
|
||||
bundletest.SetLocation(b, "resources.jobs.my-job-1", "a/b/c")
|
||||
bundletest.SetLocation(b, "resources.jobs.my-job-2", "d/e/f")
|
||||
bundletest.SetLocation(b, "resources.pipelines.my-pipeline", "abc")
|
||||
|
||||
expectedMetadata := metadata.Metadata{
|
||||
Version: metadata.Version,
|
||||
|
|
|
@ -62,8 +62,8 @@ func testStatePull(t *testing.T, opts statePullOpts) {
|
|||
|
||||
tmpDir := t.TempDir()
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
BundleRoot: vfs.MustNew(tmpDir),
|
||||
RootPath: tmpDir,
|
||||
BundleRoot: vfs.MustNew(tmpDir),
|
||||
|
||||
SyncRootPath: tmpDir,
|
||||
SyncRoot: vfs.MustNew(tmpDir),
|
||||
|
@ -259,7 +259,7 @@ func TestStatePullNoState(t *testing.T) {
|
|||
}}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
@ -447,7 +447,7 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) {
|
|||
}}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
|
@ -45,7 +45,7 @@ func TestStatePush(t *testing.T) {
|
|||
}}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
|
@ -27,7 +27,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle {
|
|||
require.NoError(t, err)
|
||||
|
||||
return &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
RootPath: tmpDir,
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
|
@ -83,10 +82,6 @@ func BundleToTerraform(config *config.Root) *schema.Root {
|
|||
conv(src, &dst)
|
||||
|
||||
if src.JobSettings != nil {
|
||||
sort.Slice(src.JobSettings.Tasks, func(i, j int) bool {
|
||||
return src.JobSettings.Tasks[i].TaskKey < src.JobSettings.Tasks[j].TaskKey
|
||||
})
|
||||
|
||||
for _, v := range src.Tasks {
|
||||
var t schema.ResourceJobTask
|
||||
conv(v, &t)
|
||||
|
@ -236,13 +231,6 @@ func BundleToTerraform(config *config.Root) *schema.Root {
|
|||
tfroot.Resource.QualityMonitor[k] = &dst
|
||||
}
|
||||
|
||||
for k, src := range config.Resources.Clusters {
|
||||
noResources = false
|
||||
var dst schema.ResourceCluster
|
||||
conv(src, &dst)
|
||||
tfroot.Resource.Cluster[k] = &dst
|
||||
}
|
||||
|
||||
// We explicitly set "resource" to nil to omit it from a JSON encoding.
|
||||
// This is required because the terraform CLI requires >= 1 resources defined
|
||||
// if the "resource" property is used in a .tf.json file.
|
||||
|
@ -406,16 +394,6 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
|
|||
}
|
||||
cur.ID = instance.Attributes.ID
|
||||
config.Resources.Schemas[resource.Name] = cur
|
||||
case "databricks_cluster":
|
||||
if config.Resources.Clusters == nil {
|
||||
config.Resources.Clusters = make(map[string]*resources.Cluster)
|
||||
}
|
||||
cur := config.Resources.Clusters[resource.Name]
|
||||
if cur == nil {
|
||||
cur = &resources.Cluster{ModifiedStatus: resources.ModifiedStatusDeleted}
|
||||
}
|
||||
cur.ID = instance.Attributes.ID
|
||||
config.Resources.Clusters[resource.Name] = cur
|
||||
case "databricks_dashboard":
|
||||
if config.Resources.Dashboards == nil {
|
||||
config.Resources.Dashboards = make(map[string]*resources.Dashboard)
|
||||
|
@ -475,11 +453,6 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
|
|||
src.ModifiedStatus = resources.ModifiedStatusCreated
|
||||
}
|
||||
}
|
||||
for _, src := range config.Resources.Clusters {
|
||||
if src.ModifiedStatus == "" && src.ID == "" {
|
||||
src.ModifiedStatus = resources.ModifiedStatusCreated
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -58,12 +58,9 @@ func TestBundleToTerraformJob(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
vin, err := convert.FromTyped(config, dyn.NilValue)
|
||||
require.NoError(t, err)
|
||||
out, err := BundleToTerraformWithDynValue(context.Background(), vin)
|
||||
require.NoError(t, err)
|
||||
|
||||
out := BundleToTerraform(&config)
|
||||
resource := out.Resource.Job["my_job"].(*schema.ResourceJob)
|
||||
|
||||
assert.Equal(t, "my job", resource.Name)
|
||||
assert.Len(t, resource.JobCluster, 1)
|
||||
assert.Equal(t, "https://github.com/foo/bar", resource.GitSource.Url)
|
||||
|
@ -666,14 +663,6 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
|
|||
{Attributes: stateInstanceAttributes{ID: "1"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "databricks_cluster",
|
||||
Mode: "managed",
|
||||
Name: "test_cluster",
|
||||
Instances: []stateResourceInstance{
|
||||
{Attributes: stateInstanceAttributes{ID: "1"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := TerraformToBundle(&tfState, &config)
|
||||
|
@ -703,9 +692,6 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
|
|||
assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus)
|
||||
|
||||
assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus)
|
||||
|
||||
AssertFullResourceCoverage(t, &config)
|
||||
}
|
||||
|
||||
|
@ -768,13 +754,6 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
Clusters: map[string]*resources.Cluster{
|
||||
"test_cluster": {
|
||||
ClusterSpec: &compute.ClusterSpec{
|
||||
ClusterName: "test_cluster",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
var tfState = resourcesState{
|
||||
|
@ -807,9 +786,6 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
|
|||
assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus)
|
||||
|
||||
assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus)
|
||||
|
||||
AssertFullResourceCoverage(t, &config)
|
||||
}
|
||||
|
||||
|
@ -912,18 +888,6 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
Clusters: map[string]*resources.Cluster{
|
||||
"test_cluster": {
|
||||
ClusterSpec: &compute.ClusterSpec{
|
||||
ClusterName: "test_cluster",
|
||||
},
|
||||
},
|
||||
"test_cluster_new": {
|
||||
ClusterSpec: &compute.ClusterSpec{
|
||||
ClusterName: "test_cluster_new",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
var tfState = resourcesState{
|
||||
|
@ -1056,22 +1020,6 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
|||
{Attributes: stateInstanceAttributes{ID: "2"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "databricks_cluster",
|
||||
Mode: "managed",
|
||||
Name: "test_cluster",
|
||||
Instances: []stateResourceInstance{
|
||||
{Attributes: stateInstanceAttributes{ID: "1"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "databricks_cluster",
|
||||
Mode: "managed",
|
||||
Name: "test_cluster_old",
|
||||
Instances: []stateResourceInstance{
|
||||
{Attributes: stateInstanceAttributes{ID: "2"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
err := TerraformToBundle(&tfState, &config)
|
||||
|
@ -1133,13 +1081,6 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
|||
assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus)
|
||||
|
||||
assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID)
|
||||
assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ModifiedStatus)
|
||||
assert.Equal(t, "2", config.Resources.Clusters["test_cluster_old"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster_old"].ModifiedStatus)
|
||||
assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID)
|
||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus)
|
||||
|
||||
AssertFullResourceCoverage(t, &config)
|
||||
}
|
||||
|
||||
|
|
|
@ -33,7 +33,7 @@ func TestInitEnvironmentVariables(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -60,7 +60,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -88,7 +88,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -114,7 +114,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -144,7 +144,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -174,7 +174,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) {
|
|||
|
||||
func TestSetProxyEnvVars(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -250,7 +250,7 @@ func TestSetProxyEnvVars(t *testing.T) {
|
|||
|
||||
func TestSetUserAgentExtraEnvVar(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Experimental: &config.Experimental{
|
||||
PyDABs: config.PyDABs{
|
||||
|
@ -333,7 +333,7 @@ func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
m := &initialize{}
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -357,7 +357,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T)
|
|||
ctx := context.Background()
|
||||
m := &initialize{}
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -380,7 +380,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
m := &initialize{}
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
|
|
@ -58,8 +58,6 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
|
|||
path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...)
|
||||
case dyn.Key("schemas"):
|
||||
path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...)
|
||||
case dyn.Key("clusters"):
|
||||
path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
|
||||
default:
|
||||
// Trigger "key not found" for unknown resource types.
|
||||
return dyn.GetByPath(root, path)
|
||||
|
|
|
@ -31,7 +31,6 @@ func TestInterpolate(t *testing.T) {
|
|||
"other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}",
|
||||
"other_registered_model": "${resources.registered_models.other_registered_model.id}",
|
||||
"other_schema": "${resources.schemas.other_schema.id}",
|
||||
"other_cluster": "${resources.clusters.other_cluster.id}",
|
||||
},
|
||||
Tasks: []jobs.Task{
|
||||
{
|
||||
|
@ -68,7 +67,6 @@ func TestInterpolate(t *testing.T) {
|
|||
assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"])
|
||||
assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"])
|
||||
assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"])
|
||||
assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
|
||||
|
||||
m := b.Config.Resources.Models["my_model"]
|
||||
assert.Equal(t, "my_model", m.Model.Name)
|
||||
|
|
|
@ -17,7 +17,7 @@ func TestLoadWithNoState(t *testing.T) {
|
|||
}
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
|
|
@ -32,7 +32,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) fi
|
|||
|
||||
func statePullTestBundle(t *testing.T) *bundle.Bundle {
|
||||
return &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
|
@ -29,7 +29,7 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer {
|
|||
|
||||
func statePushTestBundle(t *testing.T) *bundle.Bundle {
|
||||
return &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "default",
|
||||
|
|
|
@ -1,52 +0,0 @@
|
|||
package tfdyn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/cli/libs/dyn/convert"
|
||||
"github.com/databricks/cli/libs/log"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
)
|
||||
|
||||
func convertClusterResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
|
||||
// Normalize the output value to the target schema.
|
||||
vout, diags := convert.Normalize(compute.ClusterSpec{}, vin)
|
||||
for _, diag := range diags {
|
||||
log.Debugf(ctx, "cluster normalization diagnostic: %s", diag.Summary)
|
||||
}
|
||||
|
||||
return vout, nil
|
||||
}
|
||||
|
||||
type clusterConverter struct{}
|
||||
|
||||
func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
|
||||
vout, err := convertClusterResource(ctx, vin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// We always set no_wait as it allows DABs not to wait for cluster to be started.
|
||||
vout, err = dyn.Set(vout, "no_wait", dyn.V(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add the converted resource to the output.
|
||||
out.Cluster[key] = vout.AsAny()
|
||||
|
||||
// Configure permissions for this resource.
|
||||
if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
|
||||
permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key)
|
||||
out.Permissions["cluster_"+key] = permissions
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
registerConverter("clusters", clusterConverter{})
|
||||
}
|
|
@ -1,97 +0,0 @@
|
|||
package tfdyn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/cli/libs/dyn/convert"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConvertCluster(t *testing.T) {
|
||||
var src = resources.Cluster{
|
||||
ClusterSpec: &compute.ClusterSpec{
|
||||
NumWorkers: 3,
|
||||
SparkVersion: "13.3.x-scala2.12",
|
||||
ClusterName: "cluster",
|
||||
SparkConf: map[string]string{
|
||||
"spark.executor.memory": "2g",
|
||||
},
|
||||
AwsAttributes: &compute.AwsAttributes{
|
||||
Availability: "ON_DEMAND",
|
||||
},
|
||||
AzureAttributes: &compute.AzureAttributes{
|
||||
Availability: "SPOT",
|
||||
},
|
||||
DataSecurityMode: "USER_ISOLATION",
|
||||
NodeTypeId: "m5.xlarge",
|
||||
Autoscale: &compute.AutoScale{
|
||||
MinWorkers: 1,
|
||||
MaxWorkers: 10,
|
||||
},
|
||||
},
|
||||
|
||||
Permissions: []resources.Permission{
|
||||
{
|
||||
Level: "CAN_RUN",
|
||||
UserName: "jack@gmail.com",
|
||||
},
|
||||
{
|
||||
Level: "CAN_MANAGE",
|
||||
ServicePrincipalName: "sp",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
vin, err := convert.FromTyped(src, dyn.NilValue)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
out := schema.NewResources()
|
||||
err = clusterConverter{}.Convert(ctx, "my_cluster", vin, out)
|
||||
require.NoError(t, err)
|
||||
|
||||
cluster := out.Cluster["my_cluster"]
|
||||
assert.Equal(t, map[string]any{
|
||||
"num_workers": int64(3),
|
||||
"spark_version": "13.3.x-scala2.12",
|
||||
"cluster_name": "cluster",
|
||||
"spark_conf": map[string]any{
|
||||
"spark.executor.memory": "2g",
|
||||
},
|
||||
"aws_attributes": map[string]any{
|
||||
"availability": "ON_DEMAND",
|
||||
},
|
||||
"azure_attributes": map[string]any{
|
||||
"availability": "SPOT",
|
||||
},
|
||||
"data_security_mode": "USER_ISOLATION",
|
||||
"no_wait": true,
|
||||
"node_type_id": "m5.xlarge",
|
||||
"autoscale": map[string]any{
|
||||
"min_workers": int64(1),
|
||||
"max_workers": int64(10),
|
||||
},
|
||||
}, cluster)
|
||||
|
||||
// Assert equality on the permissions
|
||||
assert.Equal(t, &schema.ResourcePermissions{
|
||||
JobId: "${databricks_cluster.my_cluster.id}",
|
||||
AccessControl: []schema.ResourcePermissionsAccessControl{
|
||||
{
|
||||
PermissionLevel: "CAN_RUN",
|
||||
UserName: "jack@gmail.com",
|
||||
},
|
||||
{
|
||||
PermissionLevel: "CAN_MANAGE",
|
||||
ServicePrincipalName: "sp",
|
||||
},
|
||||
},
|
||||
}, out.Permissions["cluster_my_cluster"])
|
||||
|
||||
}
|
|
@ -19,39 +19,20 @@ func convertDashboardResource(ctx context.Context, vin dyn.Value) (dyn.Value, er
|
|||
log.Debugf(ctx, "dashboard normalization diagnostic: %s", diag.Summary)
|
||||
}
|
||||
|
||||
// Include "serialized_dashboard" field if "file_path" is set.
|
||||
// Note: the Terraform resource supports "file_path" natively, but its
|
||||
// change detection mechanism doesn't work as expected at the time of writing (Sep 30).
|
||||
if path, ok := vout.Get("file_path").AsString(); ok {
|
||||
// Include "serialized_dashboard" field if "definition_path" is set.
|
||||
if path, ok := vin.Get("definition_path").AsString(); ok {
|
||||
vout, err = dyn.Set(vout, "serialized_dashboard", dyn.V(fmt.Sprintf("${file(\"%s\")}", path)))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("failed to set serialized_dashboard: %w", err)
|
||||
}
|
||||
// Drop the "file_path" field. It is mutually exclusive with "serialized_dashboard".
|
||||
vout, err = dyn.Walk(vout, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
switch len(p) {
|
||||
case 0:
|
||||
return v, nil
|
||||
case 1:
|
||||
if p[0] == dyn.Key("file_path") {
|
||||
return v, dyn.ErrDrop
|
||||
}
|
||||
}
|
||||
|
||||
// Skip everything else.
|
||||
return v, dyn.ErrSkip
|
||||
})
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, fmt.Errorf("failed to drop file_path: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return vout, nil
|
||||
}
|
||||
|
||||
type dashboardConverter struct{}
|
||||
type DashboardConverter struct{}
|
||||
|
||||
func (dashboardConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
|
||||
func (DashboardConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
|
||||
vout, err := convertDashboardResource(ctx, vin)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -70,5 +51,5 @@ func (dashboardConverter) Convert(ctx context.Context, key string, vin dyn.Value
|
|||
}
|
||||
|
||||
func init() {
|
||||
registerConverter("dashboards", dashboardConverter{})
|
||||
registerConverter("dashboards", DashboardConverter{})
|
||||
}
|
||||
|
|
|
@ -1,80 +1,7 @@
|
|||
package tfdyn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/cli/libs/dyn/convert"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
func TestConvertDashboard(t *testing.T) {
|
||||
var src = resources.Dashboard{
|
||||
DisplayName: "my dashboard",
|
||||
WarehouseID: "f00dcafe",
|
||||
ParentPath: "/some/path",
|
||||
EmbedCredentials: true,
|
||||
|
||||
Permissions: []resources.Permission{
|
||||
{
|
||||
Level: "CAN_VIEW",
|
||||
UserName: "jane@doe.com",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
vin, err := convert.FromTyped(src, dyn.NilValue)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
out := schema.NewResources()
|
||||
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert equality on the job
|
||||
assert.Equal(t, map[string]any{
|
||||
"display_name": "my dashboard",
|
||||
"warehouse_id": "f00dcafe",
|
||||
"parent_path": "/some/path",
|
||||
"embed_credentials": true,
|
||||
}, out.Dashboard["my_dashboard"])
|
||||
|
||||
// Assert equality on the permissions
|
||||
assert.Equal(t, &schema.ResourcePermissions{
|
||||
DashboardId: "${databricks_dashboard.my_dashboard.id}",
|
||||
AccessControl: []schema.ResourcePermissionsAccessControl{
|
||||
{
|
||||
PermissionLevel: "CAN_VIEW",
|
||||
UserName: "jane@doe.com",
|
||||
},
|
||||
},
|
||||
}, out.Permissions["dashboard_my_dashboard"])
|
||||
}
|
||||
|
||||
func TestConvertDashboardFilePath(t *testing.T) {
|
||||
var src = resources.Dashboard{
|
||||
FilePath: "some/path",
|
||||
}
|
||||
|
||||
vin, err := convert.FromTyped(src, dyn.NilValue)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
out := schema.NewResources()
|
||||
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that the "serialized_dashboard" is included.
|
||||
assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
|
||||
"serialized_dashboard": "${file(\"some/path\")}",
|
||||
})
|
||||
|
||||
// Assert that the "file_path" doesn't carry over.
|
||||
assert.NotSubset(t, out.Dashboard["my_dashboard"], map[string]any{
|
||||
"file_path": "some/path",
|
||||
})
|
||||
}
|
||||
|
|
|
@ -3,7 +3,6 @@ package tfdyn
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
|
@ -20,38 +19,8 @@ func convertJobResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
|
|||
log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary)
|
||||
}
|
||||
|
||||
// Sort the tasks of each job in the bundle by task key. Sorting
|
||||
// the task keys ensures that the diff computed by terraform is correct and avoids
|
||||
// recreates. For more details see the NOTE at
|
||||
// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage
|
||||
// and https://github.com/databricks/terraform-provider-databricks/issues/4011
|
||||
// and https://github.com/databricks/cli/pull/1776
|
||||
vout := vin
|
||||
var err error
|
||||
tasks, ok := vin.Get("tasks").AsSequence()
|
||||
if ok {
|
||||
sort.Slice(tasks, func(i, j int) bool {
|
||||
// We sort the tasks by their task key. Tasks without task keys are ordered
|
||||
// before tasks with task keys. We do not error for those tasks
|
||||
// since presence of a task_key is validated for in the Jobs backend.
|
||||
tk1, ok := tasks[i].Get("task_key").AsString()
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
tk2, ok := tasks[j].Get("task_key").AsString()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return tk1 < tk2
|
||||
})
|
||||
vout, err = dyn.Set(vin, "tasks", dyn.V(tasks))
|
||||
if err != nil {
|
||||
return dyn.InvalidValue, err
|
||||
}
|
||||
}
|
||||
|
||||
// Modify top-level keys.
|
||||
vout, err = renameKeys(vout, map[string]string{
|
||||
vout, err := renameKeys(vin, map[string]string{
|
||||
"tasks": "task",
|
||||
"job_clusters": "job_cluster",
|
||||
"parameters": "parameter",
|
||||
|
|
|
@ -42,8 +42,8 @@ func TestConvertJob(t *testing.T) {
|
|||
},
|
||||
Tasks: []jobs.Task{
|
||||
{
|
||||
TaskKey: "task_key_b",
|
||||
JobClusterKey: "job_cluster_key_b",
|
||||
TaskKey: "task_key",
|
||||
JobClusterKey: "job_cluster_key",
|
||||
Libraries: []compute.Library{
|
||||
{
|
||||
Pypi: &compute.PythonPyPiLibrary{
|
||||
|
@ -55,17 +55,6 @@ func TestConvertJob(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
TaskKey: "task_key_a",
|
||||
JobClusterKey: "job_cluster_key_a",
|
||||
},
|
||||
{
|
||||
TaskKey: "task_key_c",
|
||||
JobClusterKey: "job_cluster_key_c",
|
||||
},
|
||||
{
|
||||
Description: "missing task key 😱",
|
||||
},
|
||||
},
|
||||
},
|
||||
Permissions: []resources.Permission{
|
||||
|
@ -111,15 +100,8 @@ func TestConvertJob(t *testing.T) {
|
|||
},
|
||||
"task": []any{
|
||||
map[string]any{
|
||||
"description": "missing task key 😱",
|
||||
},
|
||||
map[string]any{
|
||||
"task_key": "task_key_a",
|
||||
"job_cluster_key": "job_cluster_key_a",
|
||||
},
|
||||
map[string]any{
|
||||
"task_key": "task_key_b",
|
||||
"job_cluster_key": "job_cluster_key_b",
|
||||
"task_key": "task_key",
|
||||
"job_cluster_key": "job_cluster_key",
|
||||
"library": []any{
|
||||
map[string]any{
|
||||
"pypi": map[string]any{
|
||||
|
@ -131,10 +113,6 @@ func TestConvertJob(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
map[string]any{
|
||||
"task_key": "task_key_c",
|
||||
"job_cluster_key": "job_cluster_key_c",
|
||||
},
|
||||
},
|
||||
}, out.Job["my_job"])
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
|
||||
func TestParseResourcesStateWithNoFile(t *testing.T) {
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
@ -31,7 +31,7 @@ func TestParseResourcesStateWithNoFile(t *testing.T) {
|
|||
func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: t.TempDir(),
|
||||
RootPath: t.TempDir(),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "whatever",
|
||||
|
|
|
@ -8,13 +8,15 @@ import (
|
|||
// SetLocation sets the location of all values in the bundle to the given path.
|
||||
// This is useful for testing where we need to associate configuration
|
||||
// with the path it is loaded from.
|
||||
func SetLocation(b *bundle.Bundle, prefix string, locations []dyn.Location) {
|
||||
func SetLocation(b *bundle.Bundle, prefix string, filePath string) {
|
||||
start := dyn.MustPathFromString(prefix)
|
||||
b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
|
||||
return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||
// If the path has the given prefix, set the location.
|
||||
if p.HasPrefix(start) {
|
||||
return v.WithLocations(locations), nil
|
||||
return v.WithLocations([]dyn.Location{{
|
||||
File: filePath,
|
||||
}}), nil
|
||||
}
|
||||
|
||||
// The path is not nested under the given prefix.
|
||||
|
|
|
@ -1,20 +0,0 @@
|
|||
package bundletest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/libs/diag"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) {
|
||||
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||
err := b.Config.Mutate(f)
|
||||
require.NoError(t, err)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, diags.Error())
|
||||
}
|
|
@ -51,15 +51,9 @@ func (r *root) Generate(path string) error {
|
|||
}
|
||||
|
||||
func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error {
|
||||
// Generate types for resources
|
||||
// Generate types for resources.
|
||||
var resources []*namedBlock
|
||||
for _, k := range sortKeys(schema.ResourceSchemas) {
|
||||
// Skipping all plugin framework struct generation.
|
||||
// TODO: This is a temporary fix, generation should be fixed in the future.
|
||||
if strings.HasSuffix(k, "_pluginframework") {
|
||||
continue
|
||||
}
|
||||
|
||||
v := schema.ResourceSchemas[k]
|
||||
b := &namedBlock{
|
||||
filePattern: "resource_%s.go",
|
||||
|
@ -77,12 +71,6 @@ func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error
|
|||
// Generate types for data sources.
|
||||
var dataSources []*namedBlock
|
||||
for _, k := range sortKeys(schema.DataSourceSchemas) {
|
||||
// Skipping all plugin framework struct generation.
|
||||
// TODO: This is a temporary fix, generation should be fixed in the future.
|
||||
if strings.HasSuffix(k, "_pluginframework") {
|
||||
continue
|
||||
}
|
||||
|
||||
v := schema.DataSourceSchemas[k]
|
||||
b := &namedBlock{
|
||||
filePattern: "data_source_%s.go",
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
package schema
|
||||
|
||||
const ProviderVersion = "1.52.0"
|
||||
const ProviderVersion = "1.50.0"
|
||||
|
|
|
@ -2,16 +2,8 @@
|
|||
|
||||
package schema
|
||||
|
||||
type DataSourceClustersFilterBy struct {
|
||||
ClusterSources []string `json:"cluster_sources,omitempty"`
|
||||
ClusterStates []string `json:"cluster_states,omitempty"`
|
||||
IsPinned bool `json:"is_pinned,omitempty"`
|
||||
PolicyId string `json:"policy_id,omitempty"`
|
||||
}
|
||||
|
||||
type DataSourceClusters struct {
|
||||
ClusterNameContains string `json:"cluster_name_contains,omitempty"`
|
||||
Id string `json:"id,omitempty"`
|
||||
Ids []string `json:"ids,omitempty"`
|
||||
FilterBy *DataSourceClustersFilterBy `json:"filter_by,omitempty"`
|
||||
ClusterNameContains string `json:"cluster_name_contains,omitempty"`
|
||||
Id string `json:"id,omitempty"`
|
||||
Ids []string `json:"ids,omitempty"`
|
||||
}
|
||||
|
|
|
@ -19,7 +19,6 @@ type DataSourceExternalLocationExternalLocationInfo struct {
|
|||
CreatedBy string `json:"created_by,omitempty"`
|
||||
CredentialId string `json:"credential_id,omitempty"`
|
||||
CredentialName string `json:"credential_name,omitempty"`
|
||||
Fallback bool `json:"fallback,omitempty"`
|
||||
IsolationMode string `json:"isolation_mode,omitempty"`
|
||||
MetastoreId string `json:"metastore_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
|
|
|
@ -18,14 +18,12 @@ type DataSourceShareObject struct {
|
|||
AddedBy string `json:"added_by,omitempty"`
|
||||
CdfEnabled bool `json:"cdf_enabled,omitempty"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
DataObjectType string `json:"data_object_type"`
|
||||
HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"`
|
||||
Name string `json:"name"`
|
||||
SharedAs string `json:"shared_as,omitempty"`
|
||||
StartVersion int `json:"start_version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StringSharedAs string `json:"string_shared_as,omitempty"`
|
||||
Partition []DataSourceShareObjectPartition `json:"partition,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
@ -2,14 +2,20 @@
|
|||
|
||||
package schema
|
||||
|
||||
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct {
|
||||
ForcedForComplianceMode bool `json:"forced_for_compliance_mode,omitempty"`
|
||||
UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"`
|
||||
UnavailableForNonEnterpriseTier bool `json:"unavailable_for_non_enterprise_tier,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct {
|
||||
Hours int `json:"hours"`
|
||||
Minutes int `json:"minutes"`
|
||||
Hours int `json:"hours,omitempty"`
|
||||
Minutes int `json:"minutes,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct {
|
||||
DayOfWeek string `json:"day_of_week"`
|
||||
Frequency string `json:"frequency"`
|
||||
DayOfWeek string `json:"day_of_week,omitempty"`
|
||||
Frequency string `json:"frequency,omitempty"`
|
||||
WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"`
|
||||
}
|
||||
|
||||
|
@ -19,9 +25,9 @@ type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspa
|
|||
|
||||
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct {
|
||||
CanToggle bool `json:"can_toggle,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
EnablementDetails []any `json:"enablement_details,omitempty"`
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
RestartEvenIfNoUpdatesAvailable bool `json:"restart_even_if_no_updates_available,omitempty"`
|
||||
EnablementDetails *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"`
|
||||
MaintenanceWindow *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
@ -176,7 +176,6 @@ type ResourceCluster struct {
|
|||
IdempotencyToken string `json:"idempotency_token,omitempty"`
|
||||
InstancePoolId string `json:"instance_pool_id,omitempty"`
|
||||
IsPinned bool `json:"is_pinned,omitempty"`
|
||||
NoWait bool `json:"no_wait,omitempty"`
|
||||
NodeTypeId string `json:"node_type_id,omitempty"`
|
||||
NumWorkers int `json:"num_workers,omitempty"`
|
||||
PolicyId string `json:"policy_id,omitempty"`
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
package schema
|
||||
|
||||
type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct {
|
||||
ComplianceStandards []string `json:"compliance_standards"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
ComplianceStandards []string `json:"compliance_standards,omitempty"`
|
||||
IsEnabled bool `json:"is_enabled,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceComplianceSecurityProfileWorkspaceSetting struct {
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
package schema
|
||||
|
||||
type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct {
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
IsEnabled bool `json:"is_enabled,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct {
|
||||
|
|
|
@ -95,16 +95,14 @@ type ResourceModelServingConfigServedEntities struct {
|
|||
}
|
||||
|
||||
type ResourceModelServingConfigServedModels struct {
|
||||
EnvironmentVars map[string]string `json:"environment_vars,omitempty"`
|
||||
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
|
||||
MaxProvisionedThroughput int `json:"max_provisioned_throughput,omitempty"`
|
||||
MinProvisionedThroughput int `json:"min_provisioned_throughput,omitempty"`
|
||||
ModelName string `json:"model_name"`
|
||||
ModelVersion string `json:"model_version"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"`
|
||||
WorkloadSize string `json:"workload_size,omitempty"`
|
||||
WorkloadType string `json:"workload_type,omitempty"`
|
||||
EnvironmentVars map[string]string `json:"environment_vars,omitempty"`
|
||||
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
|
||||
ModelName string `json:"model_name"`
|
||||
ModelVersion string `json:"model_version"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"`
|
||||
WorkloadSize string `json:"workload_size"`
|
||||
WorkloadType string `json:"workload_type,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceModelServingConfigTrafficConfigRoutes struct {
|
||||
|
|
|
@ -18,27 +18,20 @@ type ResourceShareObject struct {
|
|||
AddedBy string `json:"added_by,omitempty"`
|
||||
CdfEnabled bool `json:"cdf_enabled,omitempty"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
DataObjectType string `json:"data_object_type"`
|
||||
HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"`
|
||||
Name string `json:"name"`
|
||||
SharedAs string `json:"shared_as,omitempty"`
|
||||
StartVersion int `json:"start_version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StringSharedAs string `json:"string_shared_as,omitempty"`
|
||||
Partition []ResourceShareObjectPartition `json:"partition,omitempty"`
|
||||
}
|
||||
|
||||
type ResourceShare struct {
|
||||
Comment string `json:"comment,omitempty"`
|
||||
CreatedAt int `json:"created_at,omitempty"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
Id string `json:"id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
StorageLocation string `json:"storage_location,omitempty"`
|
||||
StorageRoot string `json:"storage_root,omitempty"`
|
||||
UpdatedAt int `json:"updated_at,omitempty"`
|
||||
UpdatedBy string `json:"updated_by,omitempty"`
|
||||
Object []ResourceShareObject `json:"object,omitempty"`
|
||||
CreatedAt int `json:"created_at,omitempty"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
Id string `json:"id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Object []ResourceShareObject `json:"object,omitempty"`
|
||||
}
|
||||
|
|
|
@ -15,7 +15,6 @@ type ResourceSqlTable struct {
|
|||
ClusterKeys []string `json:"cluster_keys,omitempty"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
DataSourceFormat string `json:"data_source_format,omitempty"`
|
||||
EffectiveProperties map[string]string `json:"effective_properties,omitempty"`
|
||||
Id string `json:"id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Options map[string]string `json:"options,omitempty"`
|
||||
|
|
|
@ -21,7 +21,7 @@ type Root struct {
|
|||
|
||||
const ProviderHost = "registry.terraform.io"
|
||||
const ProviderSource = "databricks/databricks"
|
||||
const ProviderVersion = "1.52.0"
|
||||
const ProviderVersion = "1.50.0"
|
||||
|
||||
func NewRoot() *Root {
|
||||
return &Root{
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"github.com/databricks/cli/bundle/config/resources"
|
||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||
"github.com/databricks/cli/internal/testutil"
|
||||
"github.com/databricks/cli/libs/dyn"
|
||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -62,7 +61,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, ExpandGlobReferences())
|
||||
require.Empty(t, diags)
|
||||
|
@ -147,7 +146,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, ExpandGlobReferences())
|
||||
require.Empty(t, diags)
|
||||
|
@ -222,7 +221,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
|
||||
bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml"))
|
||||
|
||||
diags := bundle.Apply(context.Background(), b, ExpandGlobReferences())
|
||||
require.Empty(t, diags)
|
||||
|
|
|
@ -15,10 +15,9 @@ import (
|
|||
"github.com/databricks/cli/bundle/deploy/terraform"
|
||||
"github.com/databricks/cli/bundle/libraries"
|
||||
"github.com/databricks/cli/bundle/permissions"
|
||||
"github.com/databricks/cli/bundle/python"
|
||||
"github.com/databricks/cli/bundle/scripts"
|
||||
"github.com/databricks/cli/bundle/trampoline"
|
||||
"github.com/databricks/cli/libs/cmdio"
|
||||
"github.com/databricks/cli/libs/sync"
|
||||
terraformlib "github.com/databricks/cli/libs/terraform"
|
||||
tfjson "github.com/hashicorp/terraform-json"
|
||||
)
|
||||
|
@ -129,7 +128,7 @@ properties such as the 'catalog' or 'storage' are changed:`
|
|||
}
|
||||
|
||||
// The deploy phase deploys artifacts and resources.
|
||||
func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
|
||||
func Deploy() bundle.Mutator {
|
||||
// Core mutators that CRUD resources and modify deployment state. These
|
||||
// mutators need informed consent if they are potentially destructive.
|
||||
deployCore := bundle.Defer(
|
||||
|
@ -157,8 +156,8 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
|
|||
artifacts.CleanUp(),
|
||||
libraries.ExpandGlobReferences(),
|
||||
libraries.Upload(),
|
||||
trampoline.TransformWheelTask(),
|
||||
files.Upload(outputHandler),
|
||||
python.TransformWheelTask(),
|
||||
files.Upload(),
|
||||
deploy.StateUpdate(),
|
||||
deploy.StatePush(),
|
||||
permissions.ApplyWorkspaceRootPermissions(),
|
||||
|
|
|
@ -9,8 +9,8 @@ import (
|
|||
"github.com/databricks/cli/bundle/deploy/metadata"
|
||||
"github.com/databricks/cli/bundle/deploy/terraform"
|
||||
"github.com/databricks/cli/bundle/permissions"
|
||||
"github.com/databricks/cli/bundle/python"
|
||||
"github.com/databricks/cli/bundle/scripts"
|
||||
"github.com/databricks/cli/bundle/trampoline"
|
||||
)
|
||||
|
||||
// The initialize phase fills in defaults and connects to the workspace.
|
||||
|
@ -57,7 +57,6 @@ func Initialize() bundle.Mutator {
|
|||
),
|
||||
mutator.SetRunAs(),
|
||||
mutator.OverrideCompute(),
|
||||
mutator.ConfigureDashboardDefaults(),
|
||||
mutator.ProcessTargetMode(),
|
||||
mutator.ApplyPresets(),
|
||||
mutator.DefaultQueueing(),
|
||||
|
@ -67,7 +66,7 @@ func Initialize() bundle.Mutator {
|
|||
mutator.ConfigureWSFS(),
|
||||
|
||||
mutator.TranslatePaths(),
|
||||
trampoline.WrapperWarning(),
|
||||
python.WrapperWarning(),
|
||||
permissions.ApplyBundlePermissions(),
|
||||
permissions.FilterCurrentUser(),
|
||||
metadata.AnnotateJobs(),
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package trampoline
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
@ -17,8 +17,8 @@ func TestNoTransformByDefault(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
||||
|
@ -66,8 +66,8 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
RootPath: filepath.Join(tmpDir, "parent", "my_bundle"),
|
||||
SyncRootPath: filepath.Join(tmpDir, "parent"),
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
|
@ -1,4 +1,4 @@
|
|||
package trampoline
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
@ -69,7 +69,7 @@ func TransformWheelTask() bundle.Mutator {
|
|||
res := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper
|
||||
return res, nil
|
||||
},
|
||||
NewTrampoline(
|
||||
mutator.NewTrampoline(
|
||||
"python_wheel",
|
||||
&pythonTrampoline{},
|
||||
NOTEBOOK_TEMPLATE,
|
||||
|
@ -94,9 +94,9 @@ func (t *pythonTrampoline) CleanUp(task *jobs.Task) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []TaskWithJobKey {
|
||||
func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
|
||||
r := b.Config.Resources
|
||||
result := make([]TaskWithJobKey, 0)
|
||||
result := make([]mutator.TaskWithJobKey, 0)
|
||||
for k := range b.Config.Resources.Jobs {
|
||||
tasks := r.Jobs[k].JobSettings.Tasks
|
||||
for i := range tasks {
|
||||
|
@ -110,7 +110,7 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []TaskWithJobKey {
|
|||
continue
|
||||
}
|
||||
|
||||
result = append(result, TaskWithJobKey{
|
||||
result = append(result, mutator.TaskWithJobKey{
|
||||
JobKey: k,
|
||||
Task: task,
|
||||
})
|
|
@ -1,4 +1,4 @@
|
|||
package trampoline
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
@ -115,7 +115,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) {
|
|||
func TestNoPanicWithNoPythonWheelTasks(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
b := &bundle.Bundle{
|
||||
BundleRootPath: tmpDir,
|
||||
RootPath: tmpDir,
|
||||
Config: config.Root{
|
||||
Bundle: config.Bundle{
|
||||
Target: "development",
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue