mirror of https://github.com/databricks/cli.git
Added support for Databricks Apps in DABs (#1928)
## Changes Now it's possible to configure new `app` resource in bundle and point it to the custom `source_code_path` location where Databricks App code is defined. On `databricks bundle deploy` DABs will create an app. All consecutive `databricks bundle deploy` execution will update an existing app if there are any updated On `databricks bundle run <my_app>` DABs will execute app deployment. If the app is not started yet, it will start the app first. ### Bundle configuration ``` bundle: name: apps variables: my_job_id: description: "ID of job to run app" lookup: job: "My Job" databricks_name: description: "Name for app user" additional_flags: description: "Additional flags to run command app" default: "" my_app_config: type: complex description: "Configuration for my Databricks App" default: command: - flask - --app - hello - run - ${var.additional_flags} env: - name: DATABRICKS_NAME value: ${var.databricks_name} resources: apps: my_app: name: "anester-app" # required and has to be unique description: "My App" source_code_path: ./app # required and points to location of app code config: ${var.my_app_config} resources: - name: "my-job" description: "A job for app to be able to run" job: id: ${var.my_job_id} permission: "CAN_MANAGE_RUN" permissions: - user_name: "foo@bar.com" level: "CAN_VIEW" - service_principal_name: "my_sp" level: "CAN_MANAGE" targets: dev: variables: databricks_name: "Andrew (from dev)" additional_flags: --debug prod: variables: databricks_name: "Andrew (from prod)" ``` ### Execution 1. `databricks bundle deploy -t dev` 2. `databricks bundle run my_app -t dev` **If app is started** ``` ✓ Getting the status of the app my-app ✓ App is in RUNNING state ✓ Preparing source code for new app deployment. ✓ Deployment is pending ✓ Starting app with command: flask --app hello run --debug ✓ App started successfully You can access the app at <app-url> ``` **If app is not started** ``` ✓ Getting the status of the app my-app ✓ App is in UNAVAILABLE state ✓ Starting the app my-app ✓ App is starting... .... ✓ App is starting... ✓ App is started! ✓ Preparing source code for new app deployment. ✓ Downloading source code from /Workspace/Users/... ✓ Starting app with command: flask --app hello run --debug ✓ App started successfully You can access the app at <app-url> ``` ## Tests Added unit and config tests + manual test. ``` --- PASS: TestAccDeployBundleWithApp (404.59s) PASS coverage: 36.8% of statements in ./... ok github.com/databricks/cli/internal/bundle 405.035s coverage: 36.8% of statements in ./... ```
This commit is contained in:
parent
a6412e4334
commit
913e10a037
|
@ -0,0 +1,50 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/dynvar"
|
||||||
|
)
|
||||||
|
|
||||||
|
type interpolateVariables struct{}
|
||||||
|
|
||||||
|
func (i *interpolateVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
pattern := dyn.NewPattern(
|
||||||
|
dyn.Key("resources"),
|
||||||
|
dyn.Key("apps"),
|
||||||
|
dyn.AnyKey(),
|
||||||
|
dyn.Key("config"),
|
||||||
|
)
|
||||||
|
|
||||||
|
tfToConfigMap := map[string]string{}
|
||||||
|
for k, r := range config.SupportedResources() {
|
||||||
|
tfToConfigMap[r.TerraformResourceName] = k
|
||||||
|
}
|
||||||
|
|
||||||
|
err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
|
||||||
|
key, ok := tfToConfigMap[path[0].Key()]
|
||||||
|
if ok {
|
||||||
|
path = dyn.NewPath(dyn.Key("resources"), dyn.Key(key)).Append(path[1:]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dyn.GetByPath(root, path)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *interpolateVariables) Name() string {
|
||||||
|
return "apps.InterpolateVariables"
|
||||||
|
}
|
||||||
|
|
||||||
|
func InterpolateVariables() bundle.Mutator {
|
||||||
|
return &interpolateVariables{}
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAppInterpolateVariables(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"my_app_1": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "my_app_1",
|
||||||
|
},
|
||||||
|
Config: map[string]any{
|
||||||
|
"command": []string{"echo", "hello"},
|
||||||
|
"env": []map[string]string{
|
||||||
|
{"name": "JOB_ID", "value": "${databricks_job.my_job.id}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"my_app_2": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "my_app_2",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Jobs: map[string]*resources.Job{
|
||||||
|
"my_job": {
|
||||||
|
ID: "123",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, InterpolateVariables())
|
||||||
|
require.Empty(t, diags)
|
||||||
|
require.Equal(t, []any([]any{map[string]any{"name": "JOB_ID", "value": "123"}}), b.Config.Resources.Apps["my_app_1"].Config["env"])
|
||||||
|
require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config)
|
||||||
|
}
|
|
@ -0,0 +1,97 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/deploy"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type uploadConfig struct {
|
||||||
|
filerFactory deploy.FilerFactory
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *uploadConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
var diags diag.Diagnostics
|
||||||
|
errGroup, ctx := errgroup.WithContext(ctx)
|
||||||
|
|
||||||
|
mu := sync.Mutex{}
|
||||||
|
for key, app := range b.Config.Resources.Apps {
|
||||||
|
// If the app has a config, we need to deploy it first.
|
||||||
|
// It means we need to write app.yml file with the content of the config field
|
||||||
|
// to the remote source code path of the app.
|
||||||
|
if app.Config != nil {
|
||||||
|
appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
|
||||||
|
|
||||||
|
buf, err := configToYaml(app)
|
||||||
|
if err != nil {
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := u.filerFactory(b)
|
||||||
|
if err != nil {
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
errGroup.Go(func() error {
|
||||||
|
err := f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists)
|
||||||
|
if err != nil {
|
||||||
|
mu.Lock()
|
||||||
|
diags = append(diags, diag.Diagnostic{
|
||||||
|
Severity: diag.Error,
|
||||||
|
Summary: "Failed to save config",
|
||||||
|
Detail: fmt.Sprintf("Failed to write %s file: %s", path.Join(app.SourceCodePath, "app.yml"), err),
|
||||||
|
Locations: b.Config.GetLocations("resources.apps." + key),
|
||||||
|
})
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := errGroup.Wait(); err != nil {
|
||||||
|
return diags.Extend(diag.FromErr(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name implements bundle.Mutator.
|
||||||
|
func (u *uploadConfig) Name() string {
|
||||||
|
return "apps:UploadConfig"
|
||||||
|
}
|
||||||
|
|
||||||
|
func UploadConfig() bundle.Mutator {
|
||||||
|
return &uploadConfig{
|
||||||
|
filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
|
||||||
|
return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func configToYaml(app *resources.App) (*bytes.Buffer, error) {
|
||||||
|
buf := bytes.NewBuffer(nil)
|
||||||
|
enc := yaml.NewEncoder(buf)
|
||||||
|
enc.SetIndent(2)
|
||||||
|
|
||||||
|
err := enc.Encode(app.Config)
|
||||||
|
defer enc.Close()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to encode app config to yaml: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return buf, nil
|
||||||
|
}
|
|
@ -0,0 +1,75 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/mock"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAppUploadConfig(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
BundleRootPath: root,
|
||||||
|
SyncRootPath: root,
|
||||||
|
SyncRoot: vfs.MustNew(root),
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
RootPath: "/Workspace/Users/foo@bar.com/",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"my_app": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./my_app",
|
||||||
|
Config: map[string]any{
|
||||||
|
"command": []string{"echo", "hello"},
|
||||||
|
"env": []map[string]string{
|
||||||
|
{"name": "MY_APP", "value": "my value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mockFiler := mockfiler.NewMockFiler(t)
|
||||||
|
mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command:
|
||||||
|
- echo
|
||||||
|
- hello
|
||||||
|
env:
|
||||||
|
- name: MY_APP
|
||||||
|
value: my value
|
||||||
|
`), filer.OverwriteIfExists).Return(nil)
|
||||||
|
|
||||||
|
u := uploadConfig{
|
||||||
|
filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
|
||||||
|
return mockFiler, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(root, "databricks.yml")}})
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), &u))
|
||||||
|
require.NoError(t, diags.Error())
|
||||||
|
}
|
|
@ -0,0 +1,53 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
)
|
||||||
|
|
||||||
|
type validate struct{}
|
||||||
|
|
||||||
|
func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
var diags diag.Diagnostics
|
||||||
|
possibleConfigFiles := []string{"app.yml", "app.yaml"}
|
||||||
|
usedSourceCodePaths := make(map[string]string)
|
||||||
|
|
||||||
|
for key, app := range b.Config.Resources.Apps {
|
||||||
|
if _, ok := usedSourceCodePaths[app.SourceCodePath]; ok {
|
||||||
|
diags = append(diags, diag.Diagnostic{
|
||||||
|
Severity: diag.Error,
|
||||||
|
Summary: "Duplicate app source code path",
|
||||||
|
Detail: fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overriden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
|
||||||
|
Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
usedSourceCodePaths[app.SourceCodePath] = key
|
||||||
|
|
||||||
|
for _, configFile := range possibleConfigFiles {
|
||||||
|
appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
|
||||||
|
cf := path.Join(appPath, configFile)
|
||||||
|
if _, err := b.SyncRoot.Stat(cf); err == nil {
|
||||||
|
diags = append(diags, diag.Diagnostic{
|
||||||
|
Severity: diag.Error,
|
||||||
|
Summary: configFile + " detected",
|
||||||
|
Detail: fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *validate) Name() string {
|
||||||
|
return "apps.Validate"
|
||||||
|
}
|
||||||
|
|
||||||
|
func Validate() bundle.Mutator {
|
||||||
|
return &validate{}
|
||||||
|
}
|
|
@ -0,0 +1,97 @@
|
||||||
|
package apps
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/internal/testutil"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAppsValidate(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testutil.Touch(t, tmpDir, "app1", "app.yml")
|
||||||
|
testutil.Touch(t, tmpDir, "app2", "app.py")
|
||||||
|
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
BundleRootPath: tmpDir,
|
||||||
|
SyncRootPath: tmpDir,
|
||||||
|
SyncRoot: vfs.MustNew(tmpDir),
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
FilePath: "/foo/bar/",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"app1": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app1",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./app1",
|
||||||
|
},
|
||||||
|
"app2": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app2",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./app2",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
|
||||||
|
require.Len(t, diags, 1)
|
||||||
|
require.Equal(t, "app.yml detected", diags[0].Summary)
|
||||||
|
require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppsValidateSameSourcePath(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
testutil.Touch(t, tmpDir, "app1", "app.py")
|
||||||
|
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
BundleRootPath: tmpDir,
|
||||||
|
SyncRootPath: tmpDir,
|
||||||
|
SyncRoot: vfs.MustNew(tmpDir),
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
FilePath: "/foo/bar/",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"app1": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app1",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./app1",
|
||||||
|
},
|
||||||
|
"app2": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app2",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./app1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
|
||||||
|
require.Len(t, diags, 1)
|
||||||
|
require.Equal(t, "Duplicate app source code path", diags[0].Summary)
|
||||||
|
require.Contains(t, diags[0].Detail, "has the same source code path as app resource")
|
||||||
|
}
|
|
@ -0,0 +1,37 @@
|
||||||
|
package generate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/convert"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]any) (dyn.Value, error) {
|
||||||
|
ac, err := convert.FromTyped(appConfig, dyn.NilValue)
|
||||||
|
if err != nil {
|
||||||
|
return dyn.NilValue, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ar, err := convert.FromTyped(app.Resources, dyn.NilValue)
|
||||||
|
if err != nil {
|
||||||
|
return dyn.NilValue, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The majority of fields of the app struct are read-only.
|
||||||
|
// We copy the relevant fields manually.
|
||||||
|
dv := map[string]dyn.Value{
|
||||||
|
"name": dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}),
|
||||||
|
"description": dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}),
|
||||||
|
"source_code_path": dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 3}}),
|
||||||
|
}
|
||||||
|
|
||||||
|
if ac.Kind() != dyn.KindNil {
|
||||||
|
dv["config"] = ac.WithLocations([]dyn.Location{{Line: 4}})
|
||||||
|
}
|
||||||
|
|
||||||
|
if ar.Kind() != dyn.KindNil {
|
||||||
|
dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}})
|
||||||
|
}
|
||||||
|
|
||||||
|
return dyn.V(dv), nil
|
||||||
|
}
|
|
@ -221,6 +221,8 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
dashboard.DisplayName = prefix + dashboard.DisplayName
|
dashboard.DisplayName = prefix + dashboard.DisplayName
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Apps: No presets
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -56,6 +56,22 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle
|
||||||
b.Config.Presets.SourceLinkedDeployment = &enabled
|
b.Config.Presets.SourceLinkedDeployment = &enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(b.Config.Resources.Apps) > 0 && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
|
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
|
||||||
|
diags = diags.Append(
|
||||||
|
diag.Diagnostic{
|
||||||
|
Severity: diag.Error,
|
||||||
|
Summary: "source-linked deployment is not supported for apps",
|
||||||
|
Paths: []dyn.Path{
|
||||||
|
path,
|
||||||
|
},
|
||||||
|
Locations: b.Config.GetLocations(path[2:].String()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
|
||||||
if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
|
||||||
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
|
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/bundle/internal/bundletest"
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
"github.com/databricks/cli/libs/dbr"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
@ -31,6 +32,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
initialValue *bool
|
initialValue *bool
|
||||||
expectedValue *bool
|
expectedValue *bool
|
||||||
expectedWarning string
|
expectedWarning string
|
||||||
|
expectedError string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "preset enabled, bundle in Workspace, databricks runtime",
|
name: "preset enabled, bundle in Workspace, databricks runtime",
|
||||||
|
@ -86,6 +88,18 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
expectedValue: &enabled,
|
expectedValue: &enabled,
|
||||||
expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
|
expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "preset enabled, apps is defined by user",
|
||||||
|
ctx: dbr.MockRuntime(testContext, true),
|
||||||
|
mutateBundle: func(b *bundle.Bundle) {
|
||||||
|
b.Config.Resources.Apps = map[string]*resources.App{
|
||||||
|
"app": {},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
initialValue: &enabled,
|
||||||
|
expectedValue: &enabled,
|
||||||
|
expectedError: "source-linked deployment is not supported for apps",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
@ -107,7 +121,7 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
|
bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
|
||||||
|
|
||||||
diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
|
diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
|
||||||
if diags.HasError() {
|
if diags.HasError() && tt.expectedError == "" {
|
||||||
t.Fatalf("unexpected error: %v", diags)
|
t.Fatalf("unexpected error: %v", diags)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,6 +130,11 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
|
||||||
require.NotEmpty(t, diags[0].Locations)
|
require.NotEmpty(t, diags[0].Locations)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if tt.expectedError != "" {
|
||||||
|
require.Equal(t, tt.expectedError, diags[0].Summary)
|
||||||
|
require.NotEmpty(t, diags[0].Locations)
|
||||||
|
}
|
||||||
|
|
||||||
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/merge"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mergeApps struct{}
|
||||||
|
|
||||||
|
func MergeApps() bundle.Mutator {
|
||||||
|
return &mergeApps{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mergeApps) Name() string {
|
||||||
|
return "MergeApps"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mergeApps) resourceName(v dyn.Value) string {
|
||||||
|
switch v.Kind() {
|
||||||
|
case dyn.KindInvalid, dyn.KindNil:
|
||||||
|
return ""
|
||||||
|
case dyn.KindString:
|
||||||
|
return v.MustString()
|
||||||
|
default:
|
||||||
|
panic("app name must be a string")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
|
||||||
|
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
if v.Kind() == dyn.KindNil {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return dyn.Map(v, "resources.apps", dyn.Foreach(func(_ dyn.Path, app dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Map(app, "resources", merge.ElementsByKeyWithOverride("name", m.resourceName))
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
|
||||||
|
return diag.FromErr(err)
|
||||||
|
}
|
|
@ -0,0 +1,73 @@
|
||||||
|
package mutator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeApps(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"foo": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "foo",
|
||||||
|
Resources: []apps.AppResource{
|
||||||
|
{
|
||||||
|
Name: "job1",
|
||||||
|
Job: &apps.AppResourceJob{
|
||||||
|
Id: "1234",
|
||||||
|
Permission: "CAN_MANAGE_RUN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "sql1",
|
||||||
|
SqlWarehouse: &apps.AppResourceSqlWarehouse{
|
||||||
|
Id: "5678",
|
||||||
|
Permission: "CAN_USE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "job1",
|
||||||
|
Job: &apps.AppResourceJob{
|
||||||
|
Id: "1234",
|
||||||
|
Permission: "CAN_MANAGE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "sql1",
|
||||||
|
Job: &apps.AppResourceJob{
|
||||||
|
Id: "9876",
|
||||||
|
Permission: "CAN_MANAGE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, mutator.MergeApps())
|
||||||
|
assert.NoError(t, diags.Error())
|
||||||
|
|
||||||
|
j := b.Config.Resources.Apps["foo"]
|
||||||
|
|
||||||
|
assert.Len(t, j.Resources, 2)
|
||||||
|
assert.Equal(t, "job1", j.Resources[0].Name)
|
||||||
|
assert.Equal(t, "sql1", j.Resources[1].Name)
|
||||||
|
|
||||||
|
assert.Equal(t, "CAN_MANAGE", string(j.Resources[0].Job.Permission))
|
||||||
|
|
||||||
|
assert.Nil(t, j.Resources[1].SqlWarehouse)
|
||||||
|
assert.Equal(t, "CAN_MANAGE", string(j.Resources[1].Job.Permission))
|
||||||
|
}
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"github.com/databricks/cli/libs/tags"
|
"github.com/databricks/cli/libs/tags"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
sdkconfig "github.com/databricks/databricks-sdk-go/config"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
||||||
|
@ -142,6 +143,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"app1": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
|
SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
|
||||||
|
@ -433,6 +441,13 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
|
||||||
for _, key := range field.MapKeys() {
|
for _, key := range field.MapKeys() {
|
||||||
resource := field.MapIndex(key)
|
resource := field.MapIndex(key)
|
||||||
nameField := resource.Elem().FieldByName("Name")
|
nameField := resource.Elem().FieldByName("Name")
|
||||||
|
resourceType := resources.Type().Field(i).Name
|
||||||
|
|
||||||
|
// Skip apps, as they are not renamed
|
||||||
|
if resourceType == "Apps" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if !nameField.IsValid() || nameField.Kind() != reflect.String {
|
if !nameField.IsValid() || nameField.Kind() != reflect.String {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
@ -119,6 +119,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Apps do not support run_as in the API.
|
||||||
|
if len(b.Config.Resources.Apps) > 0 {
|
||||||
|
diags = diags.Extend(reportRunAsNotSupported(
|
||||||
|
"apps",
|
||||||
|
b.Config.GetLocation("resources.apps"),
|
||||||
|
b.Config.Workspace.CurrentUser.UserName,
|
||||||
|
identity,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
return diags
|
return diags
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string {
|
||||||
// the dyn library gives us the correct list of all resources supported. Please
|
// the dyn library gives us the correct list of all resources supported. Please
|
||||||
// also update this check when adding a new resource
|
// also update this check when adding a new resource
|
||||||
require.Equal(t, []string{
|
require.Equal(t, []string{
|
||||||
|
"apps",
|
||||||
"clusters",
|
"clusters",
|
||||||
"dashboards",
|
"dashboards",
|
||||||
"experiments",
|
"experiments",
|
||||||
|
@ -104,38 +105,37 @@ func TestRunAsWorksForAllowedResources(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
// Bundle "run_as" has two modes of operation, each with a different set of
|
||||||
// Bundle "run_as" has two modes of operation, each with a different set of
|
// resources that are supported.
|
||||||
// resources that are supported.
|
// Cases:
|
||||||
// Cases:
|
// 1. When the bundle "run_as" identity is same as the current deployment
|
||||||
// 1. When the bundle "run_as" identity is same as the current deployment
|
// identity. In this case all resources are supported.
|
||||||
// identity. In this case all resources are supported.
|
// 2. When the bundle "run_as" identity is different from the current
|
||||||
// 2. When the bundle "run_as" identity is different from the current
|
// deployment identity. In this case only a subset of resources are
|
||||||
// deployment identity. In this case only a subset of resources are
|
// supported. This subset of resources are defined in the allow list below.
|
||||||
// supported. This subset of resources are defined in the allow list below.
|
//
|
||||||
//
|
// To be a part of the allow list, the resource must satisfy one of the following
|
||||||
// To be a part of the allow list, the resource must satisfy one of the following
|
// two conditions:
|
||||||
// two conditions:
|
// 1. The resource supports setting a run_as identity to a different user
|
||||||
// 1. The resource supports setting a run_as identity to a different user
|
// from the owner/creator of the resource. For example, jobs.
|
||||||
// from the owner/creator of the resource. For example, jobs.
|
// 2. Run as semantics do not apply to the resource. We do not plan to add
|
||||||
// 2. Run as semantics do not apply to the resource. We do not plan to add
|
// platform side support for `run_as` for these resources. For example,
|
||||||
// platform side support for `run_as` for these resources. For example,
|
// experiments or registered models.
|
||||||
// experiments or registered models.
|
//
|
||||||
//
|
// Any resource that is not on the allow list cannot be used when the bundle
|
||||||
// Any resource that is not on the allow list cannot be used when the bundle
|
// run_as is different from the current deployment user. "bundle validate" must
|
||||||
// run_as is different from the current deployment user. "bundle validate" must
|
// return an error if such a resource has been defined, and the run_as identity
|
||||||
// return an error if such a resource has been defined, and the run_as identity
|
// is different from the current deployment identity.
|
||||||
// is different from the current deployment identity.
|
//
|
||||||
//
|
// Action Item: If you are adding a new resource to DABs, please check in with
|
||||||
// Action Item: If you are adding a new resource to DABs, please check in with
|
// the relevant owning team whether the resource should be on the allow list or (implicitly) on
|
||||||
// the relevant owning team whether the resource should be on the allow list or (implicitly) on
|
// the deny list. Any resources that could have run_as semantics in the future
|
||||||
// the deny list. Any resources that could have run_as semantics in the future
|
// should be on the deny list.
|
||||||
// should be on the deny list.
|
// For example: Teams for pipelines, model serving endpoints or Lakeview dashboards
|
||||||
// For example: Teams for pipelines, model serving endpoints or Lakeview dashboards
|
// are planning to add platform side support for `run_as` for these resources at
|
||||||
// are planning to add platform side support for `run_as` for these resources at
|
// some point in the future. These resources are (implicitly) on the deny list, since
|
||||||
// some point in the future. These resources are (implicitly) on the deny list, since
|
// they are not on the allow list below.
|
||||||
// they are not on the allow list below.
|
var allowList = []string{
|
||||||
allowList := []string{
|
|
||||||
"clusters",
|
"clusters",
|
||||||
"jobs",
|
"jobs",
|
||||||
"models",
|
"models",
|
||||||
|
@ -143,8 +143,9 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
||||||
"experiments",
|
"experiments",
|
||||||
"schemas",
|
"schemas",
|
||||||
"volumes",
|
"volumes",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
||||||
base := config.Root{
|
base := config.Root{
|
||||||
Workspace: config.Workspace{
|
Workspace: config.Workspace{
|
||||||
CurrentUser: &config.User{
|
CurrentUser: &config.User{
|
||||||
|
@ -197,3 +198,54 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
|
||||||
"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)
|
"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunAsNoErrorForSupportedResources(t *testing.T) {
|
||||||
|
base := config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
CurrentUser: &config.User{
|
||||||
|
User: &iam.User{
|
||||||
|
UserName: "alice",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
RunAs: &jobs.JobRunAs{
|
||||||
|
UserName: "bob",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
v, err := convert.FromTyped(base, dyn.NilValue)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Define top level resources key in the bundle configuration.
|
||||||
|
// This is not part of the typed configuration, so we need to add it manually.
|
||||||
|
v, err = dyn.Set(v, "resources", dyn.V(map[string]dyn.Value{}))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
for _, rt := range allResourceTypes(t) {
|
||||||
|
// Skip unsupported resources
|
||||||
|
if !slices.Contains(allowList, rt) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add an instance of the resource type that is not on the allow list to
|
||||||
|
// the bundle configuration.
|
||||||
|
nv, err := dyn.SetByPath(v, dyn.NewPath(dyn.Key("resources"), dyn.Key(rt)), dyn.V(map[string]dyn.Value{
|
||||||
|
"foo": dyn.V(map[string]dyn.Value{
|
||||||
|
"name": dyn.V("bar"),
|
||||||
|
}),
|
||||||
|
}))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Get back typed configuration from the newly created invalid bundle configuration.
|
||||||
|
r := &config.Root{}
|
||||||
|
err = convert.ToTyped(r, nv)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Assert this configuration passes validation.
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: *r,
|
||||||
|
}
|
||||||
|
diags := bundle.Apply(context.Background(), b, SetRunAs())
|
||||||
|
require.NoError(t, diags.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -262,6 +262,7 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
|
||||||
t.applyPipelineTranslations,
|
t.applyPipelineTranslations,
|
||||||
t.applyArtifactTranslations,
|
t.applyArtifactTranslations,
|
||||||
t.applyDashboardTranslations,
|
t.applyDashboardTranslations,
|
||||||
|
t.applyAppsTranslations,
|
||||||
} {
|
} {
|
||||||
v, err = fn(v)
|
v, err = fn(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
package mutator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error) {
|
||||||
|
// Convert the `source_code_path` field to a remote absolute path.
|
||||||
|
// We use this path for app deployment to point to the source code.
|
||||||
|
pattern := dyn.NewPattern(
|
||||||
|
dyn.Key("resources"),
|
||||||
|
dyn.Key("apps"),
|
||||||
|
dyn.AnyKey(),
|
||||||
|
dyn.Key("source_code_path"),
|
||||||
|
)
|
||||||
|
|
||||||
|
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
key := p[2].Key()
|
||||||
|
dir, err := v.Location().Directory()
|
||||||
|
if err != nil {
|
||||||
|
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for app %s: %w", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return t.rewriteRelativeTo(p, v, t.translateDirectoryPath, dir, "")
|
||||||
|
})
|
||||||
|
}
|
|
@ -0,0 +1,57 @@
|
||||||
|
package mutator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
touchEmptyFile(t, filepath.Join(dir, "src", "app", "app.py"))
|
||||||
|
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
SyncRootPath: dir,
|
||||||
|
SyncRoot: vfs.MustNew(dir),
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
FilePath: "/bundle/files",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"app": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "My App",
|
||||||
|
},
|
||||||
|
SourceCodePath: "../src/app",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "resources.apps", []dyn.Location{{
|
||||||
|
File: filepath.Join(dir, "resources/app.yml"),
|
||||||
|
}})
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
|
||||||
|
require.NoError(t, diags.Error())
|
||||||
|
|
||||||
|
// Assert that the file path for the app has been converted to its local absolute path.
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
"/bundle/files/src/app",
|
||||||
|
b.Config.Resources.Apps["app"].SourceCodePath,
|
||||||
|
)
|
||||||
|
}
|
|
@ -23,6 +23,7 @@ type Resources struct {
|
||||||
Volumes map[string]*resources.Volume `json:"volumes,omitempty"`
|
Volumes map[string]*resources.Volume `json:"volumes,omitempty"`
|
||||||
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
|
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
|
||||||
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
|
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
|
||||||
|
Apps map[string]*resources.App `json:"apps,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ConfigResource interface {
|
type ConfigResource interface {
|
||||||
|
@ -87,6 +88,7 @@ func (r *Resources) AllResources() []ResourceGroup {
|
||||||
collectResourceMap(descriptions["clusters"], r.Clusters),
|
collectResourceMap(descriptions["clusters"], r.Clusters),
|
||||||
collectResourceMap(descriptions["dashboards"], r.Dashboards),
|
collectResourceMap(descriptions["dashboards"], r.Dashboards),
|
||||||
collectResourceMap(descriptions["volumes"], r.Volumes),
|
collectResourceMap(descriptions["volumes"], r.Volumes),
|
||||||
|
collectResourceMap(descriptions["apps"], r.Apps),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,12 +99,19 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
|
||||||
found = append(found, r.Jobs[k])
|
found = append(found, r.Jobs[k])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for k := range r.Pipelines {
|
for k := range r.Pipelines {
|
||||||
if k == key {
|
if k == key {
|
||||||
found = append(found, r.Pipelines[k])
|
found = append(found, r.Pipelines[k])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for k := range r.Apps {
|
||||||
|
if k == key {
|
||||||
|
found = append(found, r.Apps[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if len(found) == 0 {
|
if len(found) == 0 {
|
||||||
return nil, fmt.Errorf("no such resource: %s", key)
|
return nil, fmt.Errorf("no such resource: %s", key)
|
||||||
}
|
}
|
||||||
|
@ -126,6 +135,8 @@ type ResourceDescription struct {
|
||||||
// Singular and plural title when used in summaries / terminal UI.
|
// Singular and plural title when used in summaries / terminal UI.
|
||||||
SingularTitle string
|
SingularTitle string
|
||||||
PluralTitle string
|
PluralTitle string
|
||||||
|
|
||||||
|
TerraformResourceName string
|
||||||
}
|
}
|
||||||
|
|
||||||
// The keys of the map corresponds to the resource key in the bundle configuration.
|
// The keys of the map corresponds to the resource key in the bundle configuration.
|
||||||
|
@ -136,66 +147,84 @@ func SupportedResources() map[string]ResourceDescription {
|
||||||
PluralName: "jobs",
|
PluralName: "jobs",
|
||||||
SingularTitle: "Job",
|
SingularTitle: "Job",
|
||||||
PluralTitle: "Jobs",
|
PluralTitle: "Jobs",
|
||||||
|
TerraformResourceName: "databricks_job",
|
||||||
},
|
},
|
||||||
"pipelines": {
|
"pipelines": {
|
||||||
SingularName: "pipeline",
|
SingularName: "pipeline",
|
||||||
PluralName: "pipelines",
|
PluralName: "pipelines",
|
||||||
SingularTitle: "Pipeline",
|
SingularTitle: "Pipeline",
|
||||||
PluralTitle: "Pipelines",
|
PluralTitle: "Pipelines",
|
||||||
|
TerraformResourceName: "databricks_pipeline",
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
SingularName: "model",
|
SingularName: "model",
|
||||||
PluralName: "models",
|
PluralName: "models",
|
||||||
SingularTitle: "Model",
|
SingularTitle: "Model",
|
||||||
PluralTitle: "Models",
|
PluralTitle: "Models",
|
||||||
|
TerraformResourceName: "databricks_mlflow_model",
|
||||||
},
|
},
|
||||||
"experiments": {
|
"experiments": {
|
||||||
SingularName: "experiment",
|
SingularName: "experiment",
|
||||||
PluralName: "experiments",
|
PluralName: "experiments",
|
||||||
SingularTitle: "Experiment",
|
SingularTitle: "Experiment",
|
||||||
PluralTitle: "Experiments",
|
PluralTitle: "Experiments",
|
||||||
|
TerraformResourceName: "databricks_mlflow_experiment",
|
||||||
},
|
},
|
||||||
"model_serving_endpoints": {
|
"model_serving_endpoints": {
|
||||||
SingularName: "model_serving_endpoint",
|
SingularName: "model_serving_endpoint",
|
||||||
PluralName: "model_serving_endpoints",
|
PluralName: "model_serving_endpoints",
|
||||||
SingularTitle: "Model Serving Endpoint",
|
SingularTitle: "Model Serving Endpoint",
|
||||||
PluralTitle: "Model Serving Endpoints",
|
PluralTitle: "Model Serving Endpoints",
|
||||||
|
TerraformResourceName: "databricks_model_serving_endpoint",
|
||||||
},
|
},
|
||||||
"registered_models": {
|
"registered_models": {
|
||||||
SingularName: "registered_model",
|
SingularName: "registered_model",
|
||||||
PluralName: "registered_models",
|
PluralName: "registered_models",
|
||||||
SingularTitle: "Registered Model",
|
SingularTitle: "Registered Model",
|
||||||
PluralTitle: "Registered Models",
|
PluralTitle: "Registered Models",
|
||||||
|
TerraformResourceName: "databricks_registered_model",
|
||||||
},
|
},
|
||||||
"quality_monitors": {
|
"quality_monitors": {
|
||||||
SingularName: "quality_monitor",
|
SingularName: "quality_monitor",
|
||||||
PluralName: "quality_monitors",
|
PluralName: "quality_monitors",
|
||||||
SingularTitle: "Quality Monitor",
|
SingularTitle: "Quality Monitor",
|
||||||
PluralTitle: "Quality Monitors",
|
PluralTitle: "Quality Monitors",
|
||||||
|
TerraformResourceName: "databricks_quality_monitor",
|
||||||
},
|
},
|
||||||
"schemas": {
|
"schemas": {
|
||||||
SingularName: "schema",
|
SingularName: "schema",
|
||||||
PluralName: "schemas",
|
PluralName: "schemas",
|
||||||
SingularTitle: "Schema",
|
SingularTitle: "Schema",
|
||||||
PluralTitle: "Schemas",
|
PluralTitle: "Schemas",
|
||||||
|
TerraformResourceName: "databricks_schema",
|
||||||
},
|
},
|
||||||
"clusters": {
|
"clusters": {
|
||||||
SingularName: "cluster",
|
SingularName: "cluster",
|
||||||
PluralName: "clusters",
|
PluralName: "clusters",
|
||||||
SingularTitle: "Cluster",
|
SingularTitle: "Cluster",
|
||||||
PluralTitle: "Clusters",
|
PluralTitle: "Clusters",
|
||||||
|
TerraformResourceName: "databricks_cluster",
|
||||||
},
|
},
|
||||||
"dashboards": {
|
"dashboards": {
|
||||||
SingularName: "dashboard",
|
SingularName: "dashboard",
|
||||||
PluralName: "dashboards",
|
PluralName: "dashboards",
|
||||||
SingularTitle: "Dashboard",
|
SingularTitle: "Dashboard",
|
||||||
PluralTitle: "Dashboards",
|
PluralTitle: "Dashboards",
|
||||||
|
TerraformResourceName: "databricks_dashboard",
|
||||||
},
|
},
|
||||||
"volumes": {
|
"volumes": {
|
||||||
SingularName: "volume",
|
SingularName: "volume",
|
||||||
PluralName: "volumes",
|
PluralName: "volumes",
|
||||||
SingularTitle: "Volume",
|
SingularTitle: "Volume",
|
||||||
PluralTitle: "Volumes",
|
PluralTitle: "Volumes",
|
||||||
|
TerraformResourceName: "databricks_volume",
|
||||||
|
},
|
||||||
|
"apps": {
|
||||||
|
SingularName: "app",
|
||||||
|
PluralName: "apps",
|
||||||
|
SingularTitle: "App",
|
||||||
|
PluralTitle: "Apps",
|
||||||
|
TerraformResourceName: "databricks_app",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,70 @@
|
||||||
|
package resources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/libs/log"
|
||||||
|
"github.com/databricks/databricks-sdk-go"
|
||||||
|
"github.com/databricks/databricks-sdk-go/marshal"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
)
|
||||||
|
|
||||||
|
type App struct {
|
||||||
|
// SourceCodePath is a required field used by DABs to point to Databricks app source code
|
||||||
|
// on local disk and to the corresponding workspace path during app deployment.
|
||||||
|
SourceCodePath string `json:"source_code_path"`
|
||||||
|
|
||||||
|
// Config is an optional field which allows configuring the app following Databricks app configuration format like in app.yml.
|
||||||
|
// When this field is set, DABs read the configuration set in this field and write
|
||||||
|
// it to app.yml in the root of the source code folder in Databricks workspace.
|
||||||
|
// If there’s app.yml defined locally, DABs will raise an error.
|
||||||
|
Config map[string]any `json:"config,omitempty"`
|
||||||
|
|
||||||
|
Permissions []Permission `json:"permissions,omitempty"`
|
||||||
|
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
|
||||||
|
URL string `json:"url,omitempty" bundle:"internal"`
|
||||||
|
|
||||||
|
*apps.App
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) UnmarshalJSON(b []byte) error {
|
||||||
|
return marshal.Unmarshal(b, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a App) MarshalJSON() ([]byte, error) {
|
||||||
|
return marshal.Marshal(a)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) {
|
||||||
|
_, err := w.Apps.GetByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
log.Debugf(ctx, "app %s does not exist", name)
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) TerraformResourceName() string {
|
||||||
|
return "databricks_app"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) InitializeURL(baseURL url.URL) {
|
||||||
|
if a.ModifiedStatus == "" || a.ModifiedStatus == ModifiedStatusCreated {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
baseURL.Path = "apps/" + a.Name
|
||||||
|
a.URL = baseURL.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) GetName() string {
|
||||||
|
return a.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) GetURL() string {
|
||||||
|
return a.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *App) IsNil() bool {
|
||||||
|
return a.App == nil
|
||||||
|
}
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle/deploy/terraform/tfdyn"
|
"github.com/databricks/cli/bundle/deploy/terraform/tfdyn"
|
||||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
tfjson "github.com/hashicorp/terraform-json"
|
tfjson "github.com/hashicorp/terraform-json"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -196,6 +197,20 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
|
||||||
}
|
}
|
||||||
cur.ID = instance.Attributes.ID
|
cur.ID = instance.Attributes.ID
|
||||||
config.Resources.Dashboards[resource.Name] = cur
|
config.Resources.Dashboards[resource.Name] = cur
|
||||||
|
case "databricks_app":
|
||||||
|
if config.Resources.Apps == nil {
|
||||||
|
config.Resources.Apps = make(map[string]*resources.App)
|
||||||
|
}
|
||||||
|
cur := config.Resources.Apps[resource.Name]
|
||||||
|
if cur == nil {
|
||||||
|
cur = &resources.App{ModifiedStatus: resources.ModifiedStatusDeleted, App: &apps.App{}}
|
||||||
|
} else {
|
||||||
|
// If the app exists in terraform and bundle, we always set modified status to updated
|
||||||
|
// because we don't really know if the app source code was updated or not.
|
||||||
|
cur.ModifiedStatus = resources.ModifiedStatusUpdated
|
||||||
|
}
|
||||||
|
cur.Name = instance.Attributes.Name
|
||||||
|
config.Resources.Apps[resource.Name] = cur
|
||||||
case "databricks_permissions":
|
case "databricks_permissions":
|
||||||
case "databricks_grants":
|
case "databricks_grants":
|
||||||
// Ignore; no need to pull these back into the configuration.
|
// Ignore; no need to pull these back into the configuration.
|
||||||
|
@ -260,6 +275,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
|
||||||
src.ModifiedStatus = resources.ModifiedStatusCreated
|
src.ModifiedStatus = resources.ModifiedStatusCreated
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, src := range config.Resources.Apps {
|
||||||
|
if src.ModifiedStatus == "" {
|
||||||
|
src.ModifiedStatus = resources.ModifiedStatusCreated
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
"github.com/databricks/cli/bundle/internal/tf/schema"
|
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/dyn/convert"
|
"github.com/databricks/cli/libs/dyn/convert"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
||||||
|
@ -694,6 +695,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
|
||||||
{Attributes: stateInstanceAttributes{ID: "1"}},
|
{Attributes: stateInstanceAttributes{ID: "1"}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Type: "databricks_app",
|
||||||
|
Mode: "managed",
|
||||||
|
Name: "test_app",
|
||||||
|
Instances: []stateResourceInstance{
|
||||||
|
{Attributes: stateInstanceAttributes{Name: "app1"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
err := TerraformToBundle(&tfState, &config)
|
err := TerraformToBundle(&tfState, &config)
|
||||||
|
@ -732,6 +741,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
|
||||||
assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
|
assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
|
||||||
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
|
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
|
||||||
|
|
||||||
|
assert.Equal(t, "app1", config.Resources.Apps["test_app"].Name)
|
||||||
|
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus)
|
||||||
|
|
||||||
AssertFullResourceCoverage(t, &config)
|
AssertFullResourceCoverage(t, &config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -815,6 +827,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"test_app": {
|
||||||
|
App: &apps.App{
|
||||||
|
Description: "test_app",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
tfState := resourcesState{
|
tfState := resourcesState{
|
||||||
|
@ -856,6 +875,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
|
||||||
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID)
|
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID)
|
||||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
|
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
|
||||||
|
|
||||||
|
assert.Equal(t, "", config.Resources.Apps["test_app"].Name)
|
||||||
|
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus)
|
||||||
|
|
||||||
AssertFullResourceCoverage(t, &config)
|
AssertFullResourceCoverage(t, &config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -994,6 +1016,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"test_app": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "test_app",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"test_app_new": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "test_app_new",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
tfState := resourcesState{
|
tfState := resourcesState{
|
||||||
|
@ -1174,6 +1208,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
||||||
{Attributes: stateInstanceAttributes{ID: "2"}},
|
{Attributes: stateInstanceAttributes{ID: "2"}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Type: "databricks_app",
|
||||||
|
Mode: "managed",
|
||||||
|
Name: "test_app",
|
||||||
|
Instances: []stateResourceInstance{
|
||||||
|
{Attributes: stateInstanceAttributes{Name: "test_app"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: "databricks_app",
|
||||||
|
Mode: "managed",
|
||||||
|
Name: "test_app_old",
|
||||||
|
Instances: []stateResourceInstance{
|
||||||
|
{Attributes: stateInstanceAttributes{Name: "test_app_old"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
err := TerraformToBundle(&tfState, &config)
|
err := TerraformToBundle(&tfState, &config)
|
||||||
|
@ -1256,6 +1306,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
|
||||||
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID)
|
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID)
|
||||||
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus)
|
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus)
|
||||||
|
|
||||||
|
assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name)
|
||||||
|
assert.Equal(t, resources.ModifiedStatusUpdated, config.Resources.Apps["test_app"].ModifiedStatus)
|
||||||
|
assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].Name)
|
||||||
|
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app_old"].ModifiedStatus)
|
||||||
|
assert.Equal(t, "test_app_new", config.Resources.Apps["test_app_new"].Name)
|
||||||
|
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app_new"].ModifiedStatus)
|
||||||
|
|
||||||
AssertFullResourceCoverage(t, &config)
|
AssertFullResourceCoverage(t, &config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -63,6 +63,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
|
||||||
path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
|
path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
|
||||||
case dyn.Key("dashboards"):
|
case dyn.Key("dashboards"):
|
||||||
path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...)
|
path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...)
|
||||||
|
case dyn.Key("apps"):
|
||||||
|
path = dyn.NewPath(dyn.Key("databricks_app")).Append(path[2:]...)
|
||||||
default:
|
default:
|
||||||
// Trigger "key not found" for unknown resource types.
|
// Trigger "key not found" for unknown resource types.
|
||||||
return dyn.GetByPath(root, path)
|
return dyn.GetByPath(root, path)
|
||||||
|
|
|
@ -34,6 +34,7 @@ func TestInterpolate(t *testing.T) {
|
||||||
"other_volume": "${resources.volumes.other_volume.id}",
|
"other_volume": "${resources.volumes.other_volume.id}",
|
||||||
"other_cluster": "${resources.clusters.other_cluster.id}",
|
"other_cluster": "${resources.clusters.other_cluster.id}",
|
||||||
"other_dashboard": "${resources.dashboards.other_dashboard.id}",
|
"other_dashboard": "${resources.dashboards.other_dashboard.id}",
|
||||||
|
"other_app": "${resources.apps.other_app.id}",
|
||||||
},
|
},
|
||||||
Tasks: []jobs.Task{
|
Tasks: []jobs.Task{
|
||||||
{
|
{
|
||||||
|
@ -73,6 +74,7 @@ func TestInterpolate(t *testing.T) {
|
||||||
assert.Equal(t, "${databricks_volume.other_volume.id}", j.Tags["other_volume"])
|
assert.Equal(t, "${databricks_volume.other_volume.id}", j.Tags["other_volume"])
|
||||||
assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
|
assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
|
||||||
assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"])
|
assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"])
|
||||||
|
assert.Equal(t, "${databricks_app.other_app.id}", j.Tags["other_app"])
|
||||||
|
|
||||||
m := b.Config.Resources.Models["my_model"]
|
m := b.Config.Resources.Models["my_model"]
|
||||||
assert.Equal(t, "my_model", m.Model.Name)
|
assert.Equal(t, "my_model", m.Model.Name)
|
||||||
|
|
|
@ -0,0 +1,55 @@
|
||||||
|
package tfdyn
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/convert"
|
||||||
|
"github.com/databricks/cli/libs/log"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
)
|
||||||
|
|
||||||
|
func convertAppResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
|
||||||
|
// Check if the description is not set and if it's not, set it to an empty string.
|
||||||
|
// This is done to avoid TF drift because Apps API return empty string for description when if it's not set.
|
||||||
|
if _, err := dyn.Get(vin, "description"); err != nil {
|
||||||
|
vin, err = dyn.Set(vin, "description", dyn.V(""))
|
||||||
|
if err != nil {
|
||||||
|
return vin, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize the output value to the target schema.
|
||||||
|
vout, diags := convert.Normalize(apps.App{}, vin)
|
||||||
|
for _, diag := range diags {
|
||||||
|
log.Debugf(ctx, "app normalization diagnostic: %s", diag.Summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
return vout, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type appConverter struct{}
|
||||||
|
|
||||||
|
func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
|
||||||
|
vout, err := convertAppResource(ctx, vin)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the converted resource to the output.
|
||||||
|
out.App[key] = vout.AsAny()
|
||||||
|
|
||||||
|
// Configure permissions for this resource.
|
||||||
|
if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
|
||||||
|
permissions.AppName = fmt.Sprintf("${databricks_app.%s.name}", key)
|
||||||
|
out.Permissions["app_"+key] = permissions
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
registerConverter("apps", appConverter{})
|
||||||
|
}
|
|
@ -0,0 +1,156 @@
|
||||||
|
package tfdyn
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/tf/schema"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/convert"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConvertApp(t *testing.T) {
|
||||||
|
src := resources.App{
|
||||||
|
SourceCodePath: "./app",
|
||||||
|
Config: map[string]any{
|
||||||
|
"command": []string{"python", "app.py"},
|
||||||
|
},
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app_id",
|
||||||
|
Description: "app description",
|
||||||
|
Resources: []apps.AppResource{
|
||||||
|
{
|
||||||
|
Name: "job1",
|
||||||
|
Job: &apps.AppResourceJob{
|
||||||
|
Id: "1234",
|
||||||
|
Permission: "CAN_MANAGE_RUN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "sql1",
|
||||||
|
SqlWarehouse: &apps.AppResourceSqlWarehouse{
|
||||||
|
Id: "5678",
|
||||||
|
Permission: "CAN_USE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Permissions: []resources.Permission{
|
||||||
|
{
|
||||||
|
Level: "CAN_RUN",
|
||||||
|
UserName: "jack@gmail.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Level: "CAN_MANAGE",
|
||||||
|
ServicePrincipalName: "sp",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
vin, err := convert.FromTyped(src, dyn.NilValue)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
out := schema.NewResources()
|
||||||
|
err = appConverter{}.Convert(ctx, "my_app", vin, out)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
app := out.App["my_app"]
|
||||||
|
assert.Equal(t, map[string]any{
|
||||||
|
"description": "app description",
|
||||||
|
"name": "app_id",
|
||||||
|
"resources": []any{
|
||||||
|
map[string]any{
|
||||||
|
"name": "job1",
|
||||||
|
"job": map[string]any{
|
||||||
|
"id": "1234",
|
||||||
|
"permission": "CAN_MANAGE_RUN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
map[string]any{
|
||||||
|
"name": "sql1",
|
||||||
|
"sql_warehouse": map[string]any{
|
||||||
|
"id": "5678",
|
||||||
|
"permission": "CAN_USE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, app)
|
||||||
|
|
||||||
|
// Assert equality on the permissions
|
||||||
|
assert.Equal(t, &schema.ResourcePermissions{
|
||||||
|
AppName: "${databricks_app.my_app.name}",
|
||||||
|
AccessControl: []schema.ResourcePermissionsAccessControl{
|
||||||
|
{
|
||||||
|
PermissionLevel: "CAN_RUN",
|
||||||
|
UserName: "jack@gmail.com",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PermissionLevel: "CAN_MANAGE",
|
||||||
|
ServicePrincipalName: "sp",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, out.Permissions["app_my_app"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertAppWithNoDescription(t *testing.T) {
|
||||||
|
src := resources.App{
|
||||||
|
SourceCodePath: "./app",
|
||||||
|
Config: map[string]any{
|
||||||
|
"command": []string{"python", "app.py"},
|
||||||
|
},
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "app_id",
|
||||||
|
Resources: []apps.AppResource{
|
||||||
|
{
|
||||||
|
Name: "job1",
|
||||||
|
Job: &apps.AppResourceJob{
|
||||||
|
Id: "1234",
|
||||||
|
Permission: "CAN_MANAGE_RUN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "sql1",
|
||||||
|
SqlWarehouse: &apps.AppResourceSqlWarehouse{
|
||||||
|
Id: "5678",
|
||||||
|
Permission: "CAN_USE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
vin, err := convert.FromTyped(src, dyn.NilValue)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
out := schema.NewResources()
|
||||||
|
err = appConverter{}.Convert(ctx, "my_app", vin, out)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
app := out.App["my_app"]
|
||||||
|
assert.Equal(t, map[string]any{
|
||||||
|
"name": "app_id",
|
||||||
|
"description": "", // Due to Apps API always returning a description field, we set it in the output as well to avoid permanent TF drift
|
||||||
|
"resources": []any{
|
||||||
|
map[string]any{
|
||||||
|
"name": "job1",
|
||||||
|
"job": map[string]any{
|
||||||
|
"id": "1234",
|
||||||
|
"permission": "CAN_MANAGE_RUN",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
map[string]any{
|
||||||
|
"name": "sql1",
|
||||||
|
"sql_warehouse": map[string]any{
|
||||||
|
"id": "5678",
|
||||||
|
"permission": "CAN_USE",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, app)
|
||||||
|
}
|
|
@ -34,6 +34,11 @@ type stateResourceInstance struct {
|
||||||
|
|
||||||
type stateInstanceAttributes struct {
|
type stateInstanceAttributes struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
|
|
||||||
|
// Some resources such as Apps do not have an ID, so we use the name instead.
|
||||||
|
// We need this for cases when such resource is removed from bundle config but
|
||||||
|
// exists in the workspace still so we can correctly display its summary.
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
ETag string `json:"etag,omitempty"`
|
ETag string `json:"etag,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -97,7 +97,7 @@ func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
|
||||||
Type: "databricks_pipeline",
|
Type: "databricks_pipeline",
|
||||||
Name: "test_pipeline",
|
Name: "test_pipeline",
|
||||||
Instances: []stateResourceInstance{
|
Instances: []stateResourceInstance{
|
||||||
{Attributes: stateInstanceAttributes{ID: "123"}},
|
{Attributes: stateInstanceAttributes{ID: "123", Name: "test_pipeline"}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
@ -147,6 +147,9 @@ github.com/databricks/cli/bundle/config.Python:
|
||||||
If enabled, Python code will execute within this environment. If disabled,
|
If enabled, Python code will execute within this environment. If disabled,
|
||||||
it defaults to using the Python interpreter available in the current shell.
|
it defaults to using the Python interpreter available in the current shell.
|
||||||
github.com/databricks/cli/bundle/config.Resources:
|
github.com/databricks/cli/bundle/config.Resources:
|
||||||
|
"apps":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
"clusters":
|
"clusters":
|
||||||
"description": |-
|
"description": |-
|
||||||
The cluster definitions for the bundle.
|
The cluster definitions for the bundle.
|
||||||
|
@ -371,6 +374,64 @@ github.com/databricks/cli/bundle/config.Workspace:
|
||||||
"state_path":
|
"state_path":
|
||||||
"description": |-
|
"description": |-
|
||||||
The workspace state path
|
The workspace state path
|
||||||
|
github.com/databricks/cli/bundle/config/resources.App:
|
||||||
|
"active_deployment":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"app_status":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"compute_status":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"config":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"create_time":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"creator":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"default_source_code_path":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"description":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"name":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"pending_deployment":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"permissions":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"resources":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"service_principal_client_id":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"service_principal_id":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"service_principal_name":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"source_code_path":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"update_time":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"updater":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"url":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
github.com/databricks/cli/bundle/config/resources.Grant:
|
github.com/databricks/cli/bundle/config/resources.Grant:
|
||||||
"principal":
|
"principal":
|
||||||
"description": |-
|
"description": |-
|
||||||
|
@ -459,3 +520,103 @@ github.com/databricks/cli/bundle/config/variable.Variable:
|
||||||
"type":
|
"type":
|
||||||
"description": |-
|
"description": |-
|
||||||
The type of the variable.
|
The type of the variable.
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
|
||||||
|
"create_time":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"creator":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"deployment_artifacts":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"deployment_id":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"mode":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"source_code_path":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"status":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"update_time":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
|
||||||
|
"source_code_path":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
|
||||||
|
"message":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"state":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppResource:
|
||||||
|
"description":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"job":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"name":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"secret":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"serving_endpoint":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"sql_warehouse":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
|
||||||
|
"id":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"permission":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
|
||||||
|
"key":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"permission":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"scope":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
|
||||||
|
"name":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"permission":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
|
||||||
|
"id":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"permission":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
|
||||||
|
"message":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"state":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
|
||||||
|
"message":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
"state":
|
||||||
|
"description": |-
|
||||||
|
PLACEHOLDER
|
||||||
|
|
|
@ -51,6 +51,10 @@ var (
|
||||||
CAN_MANAGE: "CAN_MANAGE",
|
CAN_MANAGE: "CAN_MANAGE",
|
||||||
CAN_VIEW: "CAN_READ",
|
CAN_VIEW: "CAN_READ",
|
||||||
},
|
},
|
||||||
|
"apps": {
|
||||||
|
CAN_MANAGE: "CAN_MANAGE",
|
||||||
|
CAN_VIEW: "CAN_USE",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -58,6 +58,10 @@ func TestApplyBundlePermissions(t *testing.T) {
|
||||||
"dashboard_1": {},
|
"dashboard_1": {},
|
||||||
"dashboard_2": {},
|
"dashboard_2": {},
|
||||||
},
|
},
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"app_1": {},
|
||||||
|
"app_2": {},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -114,6 +118,10 @@ func TestApplyBundlePermissions(t *testing.T) {
|
||||||
require.Len(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, 2)
|
require.Len(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, 2)
|
||||||
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
|
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
|
||||||
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"})
|
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"})
|
||||||
|
|
||||||
|
require.Len(t, b.Config.Resources.Apps["app_1"].Permissions, 2)
|
||||||
|
require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
|
||||||
|
require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWarningOnOverlapPermission(t *testing.T) {
|
func TestWarningOnOverlapPermission(t *testing.T) {
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/apps"
|
||||||
"github.com/databricks/cli/bundle/artifacts"
|
"github.com/databricks/cli/bundle/artifacts"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
@ -135,6 +136,8 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
|
||||||
bundle.Seq(
|
bundle.Seq(
|
||||||
terraform.StatePush(),
|
terraform.StatePush(),
|
||||||
terraform.Load(),
|
terraform.Load(),
|
||||||
|
apps.InterpolateVariables(),
|
||||||
|
apps.UploadConfig(),
|
||||||
metadata.Compute(),
|
metadata.Compute(),
|
||||||
metadata.Upload(),
|
metadata.Upload(),
|
||||||
bundle.LogString("Deployment complete!"),
|
bundle.LogString("Deployment complete!"),
|
||||||
|
|
|
@ -2,6 +2,7 @@ package phases
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/apps"
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
pythonmutator "github.com/databricks/cli/bundle/config/mutator/python"
|
pythonmutator "github.com/databricks/cli/bundle/config/mutator/python"
|
||||||
|
@ -71,6 +72,7 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.MergeJobParameters(),
|
mutator.MergeJobParameters(),
|
||||||
mutator.MergeJobTasks(),
|
mutator.MergeJobTasks(),
|
||||||
mutator.MergePipelineClusters(),
|
mutator.MergePipelineClusters(),
|
||||||
|
mutator.MergeApps(),
|
||||||
|
|
||||||
// Provide permission config errors & warnings after initializing all variables
|
// Provide permission config errors & warnings after initializing all variables
|
||||||
permissions.PermissionDiagnostics(),
|
permissions.PermissionDiagnostics(),
|
||||||
|
@ -89,6 +91,8 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.TranslatePaths(),
|
mutator.TranslatePaths(),
|
||||||
trampoline.WrapperWarning(),
|
trampoline.WrapperWarning(),
|
||||||
|
|
||||||
|
apps.Validate(),
|
||||||
|
|
||||||
permissions.ValidateSharedRootPermissions(),
|
permissions.ValidateSharedRootPermissions(),
|
||||||
permissions.ApplyBundlePermissions(),
|
permissions.ApplyBundlePermissions(),
|
||||||
permissions.FilterCurrentUser(),
|
permissions.FilterCurrentUser(),
|
||||||
|
|
|
@ -0,0 +1,212 @@
|
||||||
|
package run
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/run/output"
|
||||||
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
func logProgress(ctx context.Context, msg string) {
|
||||||
|
if msg == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cmdio.LogString(ctx, "✓ "+msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
type appRunner struct {
|
||||||
|
key
|
||||||
|
|
||||||
|
bundle *bundle.Bundle
|
||||||
|
app *resources.App
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) Name() string {
|
||||||
|
if a.app == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return a.app.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
func isAppStopped(app *apps.App) bool {
|
||||||
|
return app.ComputeStatus == nil ||
|
||||||
|
(app.ComputeStatus.State == apps.ComputeStateStopped || app.ComputeStatus.State == apps.ComputeStateError)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) {
|
||||||
|
app := a.app
|
||||||
|
b := a.bundle
|
||||||
|
if app == nil {
|
||||||
|
return nil, errors.New("app is not defined")
|
||||||
|
}
|
||||||
|
|
||||||
|
logProgress(ctx, "Getting the status of the app "+app.Name)
|
||||||
|
w := b.WorkspaceClient()
|
||||||
|
|
||||||
|
// Check the status of the app first.
|
||||||
|
createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if createdApp.AppStatus != nil {
|
||||||
|
logProgress(ctx, fmt.Sprintf("App is in %s state", createdApp.AppStatus.State))
|
||||||
|
}
|
||||||
|
|
||||||
|
if createdApp.ComputeStatus != nil {
|
||||||
|
logProgress(ctx, fmt.Sprintf("App compute is in %s state", createdApp.ComputeStatus.State))
|
||||||
|
}
|
||||||
|
|
||||||
|
// There could be 2 reasons why the app is not running:
|
||||||
|
// 1. The app is new and was never deployed yet.
|
||||||
|
// 2. The app was stopped (compute not running).
|
||||||
|
// We need to start the app only if the compute is not running.
|
||||||
|
if isAppStopped(createdApp) {
|
||||||
|
err := a.start(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deploy the app.
|
||||||
|
err = a.deploy(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
cmdio.LogString(ctx, "You can access the app at "+createdApp.Url)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) start(ctx context.Context) error {
|
||||||
|
app := a.app
|
||||||
|
b := a.bundle
|
||||||
|
w := b.WorkspaceClient()
|
||||||
|
|
||||||
|
logProgress(ctx, "Starting the app "+app.Name)
|
||||||
|
wait, err := w.Apps.Start(ctx, apps.StartAppRequest{Name: app.Name})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
startedApp, err := wait.OnProgress(func(p *apps.App) {
|
||||||
|
if p.AppStatus == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logProgress(ctx, "App is starting...")
|
||||||
|
}).Get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// After the app is started (meaning the compute is running), the API will return the app object with the
|
||||||
|
// active and pending deployments fields (if any). If there are active or pending deployments,
|
||||||
|
// we need to wait for them to complete before we can do the new deployment.
|
||||||
|
// Otherwise, the new deployment will fail.
|
||||||
|
// Thus, we first wait for the active deployment to complete.
|
||||||
|
if startedApp.ActiveDeployment != nil &&
|
||||||
|
startedApp.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
|
||||||
|
logProgress(ctx, "Waiting for the active deployment to complete...")
|
||||||
|
_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logProgress(ctx, "Active deployment is completed!")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then, we wait for the pending deployment to complete.
|
||||||
|
if startedApp.PendingDeployment != nil &&
|
||||||
|
startedApp.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
|
||||||
|
logProgress(ctx, "Waiting for the pending deployment to complete...")
|
||||||
|
_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.PendingDeployment.DeploymentId, 20*time.Minute, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logProgress(ctx, "Pending deployment is completed!")
|
||||||
|
}
|
||||||
|
|
||||||
|
logProgress(ctx, "App is started!")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) deploy(ctx context.Context) error {
|
||||||
|
app := a.app
|
||||||
|
b := a.bundle
|
||||||
|
w := b.WorkspaceClient()
|
||||||
|
|
||||||
|
wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
|
||||||
|
AppName: app.Name,
|
||||||
|
AppDeployment: &apps.AppDeployment{
|
||||||
|
Mode: apps.AppDeploymentModeSnapshot,
|
||||||
|
SourceCodePath: app.SourceCodePath,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
// If deploy returns an error, then there's an active deployment in progress, wait for it to complete.
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = wait.OnProgress(func(ad *apps.AppDeployment) {
|
||||||
|
if ad.Status == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logProgress(ctx, ad.Status.Message)
|
||||||
|
}).Get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) Cancel(ctx context.Context) error {
|
||||||
|
// We should cancel the app by stopping it.
|
||||||
|
app := a.app
|
||||||
|
b := a.bundle
|
||||||
|
if app == nil {
|
||||||
|
return errors.New("app is not defined")
|
||||||
|
}
|
||||||
|
|
||||||
|
w := b.WorkspaceClient()
|
||||||
|
|
||||||
|
logProgress(ctx, "Stopping app "+app.Name)
|
||||||
|
wait, err := w.Apps.Stop(ctx, apps.StopAppRequest{Name: app.Name})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = wait.OnProgress(func(p *apps.App) {
|
||||||
|
if p.AppStatus == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logProgress(ctx, p.AppStatus.Message)
|
||||||
|
}).Get()
|
||||||
|
|
||||||
|
logProgress(ctx, "App is stopped!")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) {
|
||||||
|
// We should restart the app by just running it again meaning a new app deployment will be done.
|
||||||
|
return a.Run(ctx, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) ParseArgs(args []string, opts *Options) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("received %d unexpected positional arguments", len(args))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *appRunner) CompleteArgs(args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
|
||||||
|
return nil, cobra.ShellCompDirectiveNoFileComp
|
||||||
|
}
|
|
@ -0,0 +1,216 @@
|
||||||
|
package run
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/flags"
|
||||||
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/stretchr/testify/mock"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testAppRunner struct {
|
||||||
|
m *mocks.MockWorkspaceClient
|
||||||
|
b *bundle.Bundle
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ta *testAppRunner) run(t *testing.T) {
|
||||||
|
r := appRunner{
|
||||||
|
key: "my_app",
|
||||||
|
bundle: ta.b,
|
||||||
|
app: ta.b.Config.Resources.Apps["my_app"],
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := r.Run(ta.ctx, &Options{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWorkspaceClient) {
|
||||||
|
root := t.TempDir()
|
||||||
|
err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
BundleRootPath: root,
|
||||||
|
SyncRoot: vfs.MustNew(root),
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
RootPath: "/Workspace/Users/foo@bar.com/",
|
||||||
|
},
|
||||||
|
Resources: config.Resources{
|
||||||
|
Apps: map[string]*resources.App{
|
||||||
|
"my_app": {
|
||||||
|
App: &apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
},
|
||||||
|
SourceCodePath: "./my_app",
|
||||||
|
Config: map[string]any{
|
||||||
|
"command": []string{"echo", "hello"},
|
||||||
|
"env": []map[string]string{
|
||||||
|
{"name": "MY_APP", "value": "my value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
mwc := mocks.NewMockWorkspaceClient(t)
|
||||||
|
b.SetWorkpaceClient(mwc.WorkspaceClient)
|
||||||
|
bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: "./databricks.yml"}})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
|
||||||
|
ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
|
||||||
|
|
||||||
|
diags := bundle.Apply(ctx, b, bundle.Seq(
|
||||||
|
mutator.DefineDefaultWorkspacePaths(),
|
||||||
|
mutator.TranslatePaths(),
|
||||||
|
))
|
||||||
|
require.Empty(t, diags)
|
||||||
|
|
||||||
|
return ctx, b, mwc
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupTestApp(t *testing.T, initialAppState apps.ApplicationState, initialComputeState apps.ComputeState) *testAppRunner {
|
||||||
|
ctx, b, mwc := setupBundle(t)
|
||||||
|
|
||||||
|
appApi := mwc.GetMockAppsAPI()
|
||||||
|
appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
|
||||||
|
Name: "my_app",
|
||||||
|
}).Return(&apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
AppStatus: &apps.ApplicationStatus{
|
||||||
|
State: initialAppState,
|
||||||
|
},
|
||||||
|
ComputeStatus: &apps.ComputeStatus{
|
||||||
|
State: initialComputeState,
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
|
||||||
|
Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
|
||||||
|
AppName: "my_app",
|
||||||
|
AppDeployment: &apps.AppDeployment{
|
||||||
|
Mode: apps.AppDeploymentModeSnapshot,
|
||||||
|
SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
|
||||||
|
},
|
||||||
|
}).Return(wait, nil)
|
||||||
|
|
||||||
|
return &testAppRunner{
|
||||||
|
m: mwc,
|
||||||
|
b: b,
|
||||||
|
ctx: ctx,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppRunStartedApp(t *testing.T) {
|
||||||
|
r := setupTestApp(t, apps.ApplicationStateRunning, apps.ComputeStateActive)
|
||||||
|
r.run(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppRunStoppedApp(t *testing.T) {
|
||||||
|
r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
|
||||||
|
|
||||||
|
appsApi := r.m.GetMockAppsAPI()
|
||||||
|
appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
|
||||||
|
Name: "my_app",
|
||||||
|
}).Return(&apps.WaitGetAppActive[apps.App]{
|
||||||
|
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
|
||||||
|
return &apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
AppStatus: &apps.ApplicationStatus{
|
||||||
|
State: apps.ApplicationStateRunning,
|
||||||
|
},
|
||||||
|
ComputeStatus: &apps.ComputeStatus{
|
||||||
|
State: apps.ComputeStateActive,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
r.run(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
|
||||||
|
r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
|
||||||
|
|
||||||
|
appsApi := r.m.GetMockAppsAPI()
|
||||||
|
appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
|
||||||
|
Name: "my_app",
|
||||||
|
}).Return(&apps.WaitGetAppActive[apps.App]{
|
||||||
|
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
|
||||||
|
return &apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
AppStatus: &apps.ApplicationStatus{
|
||||||
|
State: apps.ApplicationStateRunning,
|
||||||
|
},
|
||||||
|
ComputeStatus: &apps.ComputeStatus{
|
||||||
|
State: apps.ComputeStateActive,
|
||||||
|
},
|
||||||
|
ActiveDeployment: &apps.AppDeployment{
|
||||||
|
DeploymentId: "active_deployment_id",
|
||||||
|
Status: &apps.AppDeploymentStatus{
|
||||||
|
State: apps.AppDeploymentStateInProgress,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
PendingDeployment: &apps.AppDeployment{
|
||||||
|
DeploymentId: "pending_deployment_id",
|
||||||
|
Status: &apps.AppDeploymentStatus{
|
||||||
|
State: apps.AppDeploymentStateCancelled,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
appsApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "active_deployment_id", mock.Anything, mock.Anything).Return(nil, nil)
|
||||||
|
|
||||||
|
r.run(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopApp(t *testing.T) {
|
||||||
|
ctx, b, mwc := setupBundle(t)
|
||||||
|
appsApi := mwc.GetMockAppsAPI()
|
||||||
|
appsApi.EXPECT().Stop(mock.Anything, apps.StopAppRequest{
|
||||||
|
Name: "my_app",
|
||||||
|
}).Return(&apps.WaitGetAppStopped[apps.App]{
|
||||||
|
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
|
||||||
|
return &apps.App{
|
||||||
|
Name: "my_app",
|
||||||
|
AppStatus: &apps.ApplicationStatus{
|
||||||
|
State: apps.ApplicationStateUnavailable,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
r := appRunner{
|
||||||
|
key: "my_app",
|
||||||
|
bundle: b,
|
||||||
|
app: b.Config.Resources.Apps["my_app"],
|
||||||
|
}
|
||||||
|
|
||||||
|
err := r.Cancel(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
|
@ -42,7 +42,7 @@ type Runner interface {
|
||||||
// IsRunnable returns a filter that only allows runnable resources.
|
// IsRunnable returns a filter that only allows runnable resources.
|
||||||
func IsRunnable(ref refs.Reference) bool {
|
func IsRunnable(ref refs.Reference) bool {
|
||||||
switch ref.Resource.(type) {
|
switch ref.Resource.(type) {
|
||||||
case *resources.Job, *resources.Pipeline:
|
case *resources.Job, *resources.Pipeline, *resources.App:
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
|
@ -56,6 +56,12 @@ func ToRunner(b *bundle.Bundle, ref refs.Reference) (Runner, error) {
|
||||||
return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil
|
return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil
|
||||||
case *resources.Pipeline:
|
case *resources.Pipeline:
|
||||||
return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil
|
return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil
|
||||||
|
case *resources.App:
|
||||||
|
return &appRunner{
|
||||||
|
key: key(ref.KeyWithType),
|
||||||
|
bundle: b,
|
||||||
|
app: resource,
|
||||||
|
}, nil
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("unsupported resource type: %T", resource)
|
return nil, fmt.Errorf("unsupported resource type: %T", resource)
|
||||||
}
|
}
|
||||||
|
|
|
@ -59,6 +59,81 @@
|
||||||
"cli": {
|
"cli": {
|
||||||
"bundle": {
|
"bundle": {
|
||||||
"config": {
|
"config": {
|
||||||
|
"resources.App": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"active_deployment": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment"
|
||||||
|
},
|
||||||
|
"app_status": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus"
|
||||||
|
},
|
||||||
|
"compute_status": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus"
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"$ref": "#/$defs/map/interface"
|
||||||
|
},
|
||||||
|
"create_time": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"creator": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"default_source_code_path": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"description": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"pending_deployment": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment"
|
||||||
|
},
|
||||||
|
"permissions": {
|
||||||
|
"$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission"
|
||||||
|
},
|
||||||
|
"resources": {
|
||||||
|
"$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/apps.AppResource"
|
||||||
|
},
|
||||||
|
"service_principal_client_id": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"service_principal_id": {
|
||||||
|
"$ref": "#/$defs/int64"
|
||||||
|
},
|
||||||
|
"service_principal_name": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"source_code_path": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"update_time": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"updater": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"source_code_path",
|
||||||
|
"name"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"resources.Cluster": {
|
"resources.Cluster": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
@ -1273,6 +1348,9 @@
|
||||||
{
|
{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"apps": {
|
||||||
|
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App"
|
||||||
|
},
|
||||||
"clusters": {
|
"clusters": {
|
||||||
"description": "The cluster definitions for the bundle.",
|
"description": "The cluster definitions for the bundle.",
|
||||||
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster",
|
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster",
|
||||||
|
@ -1528,6 +1606,280 @@
|
||||||
},
|
},
|
||||||
"databricks-sdk-go": {
|
"databricks-sdk-go": {
|
||||||
"service": {
|
"service": {
|
||||||
|
"apps.AppDeployment": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"create_time": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"creator": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"deployment_artifacts": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts"
|
||||||
|
},
|
||||||
|
"deployment_id": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"mode": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentMode"
|
||||||
|
},
|
||||||
|
"source_code_path": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus"
|
||||||
|
},
|
||||||
|
"update_time": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppDeploymentArtifacts": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"source_code_path": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppDeploymentMode": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.AppDeploymentState": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.AppDeploymentStatus": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentState"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResource": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"description": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"job": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob"
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"secret": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret"
|
||||||
|
},
|
||||||
|
"serving_endpoint": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint"
|
||||||
|
},
|
||||||
|
"sql_warehouse": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"name"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResourceJob": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"permission": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJobJobPermission"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"permission"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResourceJobJobPermission": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.AppResourceSecret": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"key": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"permission": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecretSecretPermission"
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"key",
|
||||||
|
"permission",
|
||||||
|
"scope"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResourceSecretSecretPermission": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.AppResourceServingEndpoint": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"name": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"permission": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpointServingEndpointPermission"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"name",
|
||||||
|
"permission"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResourceServingEndpointServingEndpointPermission": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.AppResourceSqlWarehouse": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"permission": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouseSqlWarehousePermission"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"permission"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.AppResourceSqlWarehouseSqlWarehousePermission": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.ApplicationState": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.ApplicationStatus": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationState"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"apps.ComputeState": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"apps.ComputeStatus": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"message": {
|
||||||
|
"$ref": "#/$defs/string"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeState"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"catalog.MonitorCronSchedule": {
|
"catalog.MonitorCronSchedule": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
@ -5718,6 +6070,20 @@
|
||||||
"cli": {
|
"cli": {
|
||||||
"bundle": {
|
"bundle": {
|
||||||
"config": {
|
"config": {
|
||||||
|
"resources.App": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.App"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"resources.Cluster": {
|
"resources.Cluster": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
@ -5947,6 +6313,20 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"interface": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"$ref": "#/$defs/interface"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"string": {
|
"string": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
@ -6015,6 +6395,20 @@
|
||||||
},
|
},
|
||||||
"databricks-sdk-go": {
|
"databricks-sdk-go": {
|
||||||
"service": {
|
"service": {
|
||||||
|
"apps.AppResource": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResource"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"catalog.MonitorMetric": {
|
"catalog.MonitorMetric": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
bundle:
|
||||||
|
name: apps
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
host: https://acme.cloud.databricks.com/
|
||||||
|
|
||||||
|
variables:
|
||||||
|
app_config:
|
||||||
|
type: complex
|
||||||
|
default:
|
||||||
|
command:
|
||||||
|
- "python"
|
||||||
|
- "app.py"
|
||||||
|
env:
|
||||||
|
- name: SOME_ENV_VARIABLE
|
||||||
|
value: "Some value"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
apps:
|
||||||
|
my_app:
|
||||||
|
name: "my-app"
|
||||||
|
description: "My App"
|
||||||
|
source_code_path: ./app
|
||||||
|
config: ${var.app_config}
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- name: "my-sql-warehouse"
|
||||||
|
sql_warehouse:
|
||||||
|
id: 1234
|
||||||
|
permission: "CAN_USE"
|
||||||
|
- name: "my-job"
|
||||||
|
job:
|
||||||
|
id: 5678
|
||||||
|
permission: "CAN_MANAGE_RUN"
|
||||||
|
permissions:
|
||||||
|
- user_name: "foo@bar.com"
|
||||||
|
level: "CAN_VIEW"
|
||||||
|
- service_principal_name: "my_sp"
|
||||||
|
level: "CAN_MANAGE"
|
||||||
|
|
||||||
|
|
||||||
|
targets:
|
||||||
|
default:
|
||||||
|
|
||||||
|
development:
|
||||||
|
variables:
|
||||||
|
app_config:
|
||||||
|
command:
|
||||||
|
- "python"
|
||||||
|
- "dev.py"
|
||||||
|
env:
|
||||||
|
- name: SOME_ENV_VARIABLE_2
|
||||||
|
value: "Some value 2"
|
||||||
|
resources:
|
||||||
|
apps:
|
||||||
|
my_app:
|
||||||
|
source_code_path: ./app-dev
|
||||||
|
resources:
|
||||||
|
- name: "my-sql-warehouse"
|
||||||
|
sql_warehouse:
|
||||||
|
id: 1234
|
||||||
|
permission: "CAN_MANAGE"
|
||||||
|
- name: "my-job"
|
||||||
|
job:
|
||||||
|
id: 5678
|
||||||
|
permission: "CAN_MANAGE"
|
||||||
|
- name: "my-secret"
|
||||||
|
secret:
|
||||||
|
key: "key"
|
||||||
|
scope: "scope"
|
||||||
|
permission: "CAN_USE"
|
|
@ -0,0 +1,60 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApps(t *testing.T) {
|
||||||
|
b := load(t, "./apps")
|
||||||
|
assert.Equal(t, "apps", b.Config.Bundle.Name)
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b,
|
||||||
|
bundle.Seq(
|
||||||
|
mutator.SetVariables(),
|
||||||
|
mutator.ResolveVariableReferences("variables"),
|
||||||
|
))
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
|
||||||
|
app := b.Config.Resources.Apps["my_app"]
|
||||||
|
assert.Equal(t, "my-app", app.Name)
|
||||||
|
assert.Equal(t, "My App", app.Description)
|
||||||
|
assert.Equal(t, []any{"python", "app.py"}, app.Config["command"])
|
||||||
|
assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE", "value": "Some value"}}, app.Config["env"])
|
||||||
|
|
||||||
|
assert.Len(t, app.Resources, 2)
|
||||||
|
assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
|
||||||
|
assert.Equal(t, "CAN_USE", string(app.Resources[0].SqlWarehouse.Permission))
|
||||||
|
assert.Equal(t, "5678", app.Resources[1].Job.Id)
|
||||||
|
assert.Equal(t, "CAN_MANAGE_RUN", string(app.Resources[1].Job.Permission))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppsOverride(t *testing.T) {
|
||||||
|
b := loadTarget(t, "./apps", "development")
|
||||||
|
assert.Equal(t, "apps", b.Config.Bundle.Name)
|
||||||
|
|
||||||
|
diags := bundle.Apply(context.Background(), b,
|
||||||
|
bundle.Seq(
|
||||||
|
mutator.SetVariables(),
|
||||||
|
mutator.ResolveVariableReferences("variables"),
|
||||||
|
))
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
app := b.Config.Resources.Apps["my_app"]
|
||||||
|
assert.Equal(t, "my-app", app.Name)
|
||||||
|
assert.Equal(t, "My App", app.Description)
|
||||||
|
assert.Equal(t, []any{"python", "dev.py"}, app.Config["command"])
|
||||||
|
assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE_2", "value": "Some value 2"}}, app.Config["env"])
|
||||||
|
|
||||||
|
assert.Len(t, app.Resources, 3)
|
||||||
|
assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
|
||||||
|
assert.Equal(t, "CAN_MANAGE", string(app.Resources[0].SqlWarehouse.Permission))
|
||||||
|
assert.Equal(t, "5678", app.Resources[1].Job.Id)
|
||||||
|
assert.Equal(t, "CAN_MANAGE", string(app.Resources[1].Job.Permission))
|
||||||
|
assert.Equal(t, "key", app.Resources[2].Secret.Key)
|
||||||
|
assert.Equal(t, "scope", app.Resources[2].Secret.Scope)
|
||||||
|
assert.Equal(t, "CAN_USE", string(app.Resources[2].Secret.Permission))
|
||||||
|
}
|
|
@ -47,6 +47,7 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
|
||||||
mutator.MergeJobParameters(),
|
mutator.MergeJobParameters(),
|
||||||
mutator.MergeJobTasks(),
|
mutator.MergeJobTasks(),
|
||||||
mutator.MergePipelineClusters(),
|
mutator.MergePipelineClusters(),
|
||||||
|
mutator.MergeApps(),
|
||||||
))
|
))
|
||||||
return b, diags
|
return b, diags
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,6 +17,7 @@ func newGenerateCommand() *cobra.Command {
|
||||||
cmd.AddCommand(generate.NewGenerateJobCommand())
|
cmd.AddCommand(generate.NewGenerateJobCommand())
|
||||||
cmd.AddCommand(generate.NewGeneratePipelineCommand())
|
cmd.AddCommand(generate.NewGeneratePipelineCommand())
|
||||||
cmd.AddCommand(generate.NewGenerateDashboardCommand())
|
cmd.AddCommand(generate.NewGenerateDashboardCommand())
|
||||||
|
cmd.AddCommand(generate.NewGenerateAppCommand())
|
||||||
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
|
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,166 @@
|
||||||
|
package generate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/config/generate"
|
||||||
|
"github.com/databricks/cli/cmd/root"
|
||||||
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/yamlsaver"
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
|
"github.com/databricks/cli/libs/textutil"
|
||||||
|
"github.com/databricks/databricks-sdk-go"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NewGenerateAppCommand() *cobra.Command {
|
||||||
|
var configDir string
|
||||||
|
var sourceDir string
|
||||||
|
var appName string
|
||||||
|
var force bool
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "app",
|
||||||
|
Short: "Generate bundle configuration for a Databricks app",
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`)
|
||||||
|
cmd.MarkFlagRequired("existing-app-name")
|
||||||
|
|
||||||
|
cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join("resources"), `Directory path where the output bundle config will be stored`)
|
||||||
|
cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join("src", "app"), `Directory path where the app files will be stored`)
|
||||||
|
cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
ctx := cmd.Context()
|
||||||
|
b, diags := root.MustConfigureBundle(cmd)
|
||||||
|
if err := diags.Error(); err != nil {
|
||||||
|
return diags.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
w := b.WorkspaceClient()
|
||||||
|
cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName))
|
||||||
|
app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: appName})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Making sure the config directory and source directory are absolute paths.
|
||||||
|
if !filepath.IsAbs(configDir) {
|
||||||
|
configDir = filepath.Join(b.BundleRootPath, configDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !filepath.IsAbs(sourceDir) {
|
||||||
|
sourceDir = filepath.Join(b.BundleRootPath, sourceDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
downloader := newDownloader(w, sourceDir, configDir)
|
||||||
|
|
||||||
|
sourceCodePath := app.DefaultSourceCodePath
|
||||||
|
err = downloader.markDirectoryForDownload(ctx, &sourceCodePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
appConfig, err := getAppConfig(ctx, app, w)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get app config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Making sure the source code path is relative to the config directory.
|
||||||
|
rel, err := filepath.Rel(configDir, sourceDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel), appConfig)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
appKey := cmd.Flag("key").Value.String()
|
||||||
|
if appKey == "" {
|
||||||
|
appKey = textutil.NormalizeString(app.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := map[string]dyn.Value{
|
||||||
|
"resources": dyn.V(map[string]dyn.Value{
|
||||||
|
"apps": dyn.V(map[string]dyn.Value{
|
||||||
|
appKey: v,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle.
|
||||||
|
// We include this configuration inline, so we need to remove these files.
|
||||||
|
for _, configFile := range []string{"app.yml", "app.yaml"} {
|
||||||
|
delete(downloader.files, filepath.Join(sourceDir, configFile))
|
||||||
|
}
|
||||||
|
|
||||||
|
err = downloader.FlushToDisk(ctx, force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := filepath.Join(configDir, appKey+".app.yml")
|
||||||
|
|
||||||
|
saver := yamlsaver.NewSaver()
|
||||||
|
err = saver.SaveAsYAML(result, filename, force)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cmdio.LogString(ctx, "App configuration successfully saved to "+filename)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]any, error) {
|
||||||
|
sourceCodePath := app.DefaultSourceCodePath
|
||||||
|
|
||||||
|
f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The app config is stored in app.yml or app.yaml file in the source code path.
|
||||||
|
configFileNames := []string{"app.yml", "app.yaml"}
|
||||||
|
for _, configFile := range configFileNames {
|
||||||
|
r, err := f.Read(ctx, configFile)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
|
||||||
|
cmdio.LogString(ctx, "Reading app configuration from "+configFile)
|
||||||
|
content, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var appConfig map[string]any
|
||||||
|
err = yaml.Unmarshal(content, &appConfig)
|
||||||
|
if err != nil {
|
||||||
|
cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err))
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return appConfig, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, nil
|
||||||
|
}
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go"
|
"github.com/databricks/databricks-sdk-go"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/workspace"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -63,6 +64,37 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
|
||||||
|
_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
objects, err := n.w.Workspace.RecursiveList(ctx, *dirPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, obj := range objects {
|
||||||
|
if obj.ObjectType == workspace.ObjectTypeDirectory {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err := n.markFileForDownload(ctx, &obj.Path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(n.configDir, n.sourceDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*dirPath = rel
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
|
func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
|
||||||
info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
|
info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -0,0 +1,113 @@
|
||||||
|
package bundle_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/integration/internal/acc"
|
||||||
|
"github.com/databricks/cli/internal/testutil"
|
||||||
|
"github.com/databricks/cli/libs/env"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/apps"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDeployBundleWithApp(t *testing.T) {
|
||||||
|
ctx, wt := acc.WorkspaceTest(t)
|
||||||
|
|
||||||
|
// TODO: should only skip app run when app can be created with no_compute option.
|
||||||
|
if testing.Short() {
|
||||||
|
t.Log("Skip the app creation and run in short mode")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if testutil.GetCloud(t) == testutil.GCP {
|
||||||
|
t.Skip("Skipping test for GCP cloud because /api/2.0/apps is temporarily unavailable there.")
|
||||||
|
}
|
||||||
|
|
||||||
|
uniqueId := uuid.New().String()
|
||||||
|
appId := "app-%s" + uuid.New().String()[0:8]
|
||||||
|
nodeTypeId := testutil.GetCloud(t).NodeTypeID()
|
||||||
|
instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
|
||||||
|
|
||||||
|
root := initTestTemplate(t, ctx, "apps", map[string]any{
|
||||||
|
"unique_id": uniqueId,
|
||||||
|
"app_id": appId,
|
||||||
|
"node_type_id": nodeTypeId,
|
||||||
|
"spark_version": defaultSparkVersion,
|
||||||
|
"instance_pool_id": instancePoolId,
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
destroyBundle(t, ctx, root)
|
||||||
|
app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: "test-app"})
|
||||||
|
if err != nil {
|
||||||
|
require.ErrorContains(t, err, "does not exist")
|
||||||
|
} else {
|
||||||
|
require.Contains(t, []apps.ApplicationState{apps.ApplicationStateUnavailable}, app.AppStatus.State)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
deployBundle(t, ctx, root)
|
||||||
|
|
||||||
|
// App should exists after bundle deployment
|
||||||
|
app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, app)
|
||||||
|
|
||||||
|
// Check app config
|
||||||
|
currentUser, err := wt.W.CurrentUser.Me(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
pathToAppYml := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s/files/app/app.yml", currentUser.UserName, uniqueId)
|
||||||
|
reader, err := wt.W.Workspace.Download(ctx, pathToAppYml)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
job, err := wt.W.Jobs.GetBySettingsName(ctx, "test-job-with-cluster-"+uniqueId)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
content := string(data)
|
||||||
|
require.Contains(t, content, fmt.Sprintf(`command:
|
||||||
|
- flask
|
||||||
|
- --app
|
||||||
|
- app
|
||||||
|
- run
|
||||||
|
env:
|
||||||
|
- name: JOB_ID
|
||||||
|
value: "%d"`, job.JobId))
|
||||||
|
|
||||||
|
// Try to run the app
|
||||||
|
_, out := runResourceWithStderr(t, ctx, root, "test_app")
|
||||||
|
require.Contains(t, out, app.Url)
|
||||||
|
|
||||||
|
// App should be in the running state
|
||||||
|
app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, app)
|
||||||
|
require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
|
||||||
|
|
||||||
|
// Stop the app
|
||||||
|
wait, err := wt.W.Apps.Stop(ctx, apps.StopAppRequest{Name: appId})
|
||||||
|
require.NoError(t, err)
|
||||||
|
app, err = wait.Get()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, app)
|
||||||
|
require.Equal(t, apps.ApplicationStateUnavailable, app.AppStatus.State)
|
||||||
|
|
||||||
|
// Try to run the app again
|
||||||
|
_, out = runResourceWithStderr(t, ctx, root, "test_app")
|
||||||
|
require.Contains(t, out, app.Url)
|
||||||
|
|
||||||
|
// App should be in the running state
|
||||||
|
app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, app)
|
||||||
|
require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
|
||||||
|
|
||||||
|
// Redeploy it again just to check that it can be redeployed
|
||||||
|
deployBundle(t, ctx, root)
|
||||||
|
}
|
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"unique_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Unique ID for job name"
|
||||||
|
},
|
||||||
|
"app_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Unique ID for app name"
|
||||||
|
},
|
||||||
|
"spark_version": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Spark version used for job cluster"
|
||||||
|
},
|
||||||
|
"node_type_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Node type id for job cluster"
|
||||||
|
},
|
||||||
|
"instance_pool_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Instance pool id for job cluster"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from databricks.sdk import WorkspaceClient
|
||||||
|
from flask import Flask
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/")
|
||||||
|
def home():
|
||||||
|
job_id = os.getenv("JOB_ID")
|
||||||
|
|
||||||
|
w = WorkspaceClient()
|
||||||
|
job = w.jobs.get(job_id)
|
||||||
|
return job.settings.name
|
|
@ -0,0 +1,42 @@
|
||||||
|
bundle:
|
||||||
|
name: basic
|
||||||
|
|
||||||
|
workspace:
|
||||||
|
root_path: "~/.bundle/{{.unique_id}}"
|
||||||
|
|
||||||
|
resources:
|
||||||
|
apps:
|
||||||
|
test_app:
|
||||||
|
name: "{{.app_id}}"
|
||||||
|
description: "App which manages job created by this bundle"
|
||||||
|
source_code_path: ./app
|
||||||
|
config:
|
||||||
|
command:
|
||||||
|
- flask
|
||||||
|
- --app
|
||||||
|
- app
|
||||||
|
- run
|
||||||
|
env:
|
||||||
|
- name: JOB_ID
|
||||||
|
value: ${resources.jobs.foo.id}
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- name: "app-job"
|
||||||
|
description: "A job for app to be able to work with"
|
||||||
|
job:
|
||||||
|
id: ${resources.jobs.foo.id}
|
||||||
|
permission: "CAN_MANAGE_RUN"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
foo:
|
||||||
|
name: test-job-with-cluster-{{.unique_id}}
|
||||||
|
tasks:
|
||||||
|
- task_key: my_notebook_task
|
||||||
|
new_cluster:
|
||||||
|
num_workers: 1
|
||||||
|
spark_version: "{{.spark_version}}"
|
||||||
|
node_type_id: "{{.node_type_id}}"
|
||||||
|
data_security_mode: USER_ISOLATION
|
||||||
|
instance_pool_id: "{{.instance_pool_id}}"
|
||||||
|
spark_python_task:
|
||||||
|
python_file: ./hello_world.py
|
|
@ -0,0 +1 @@
|
||||||
|
print("Hello World!")
|
|
@ -119,6 +119,17 @@ func runResource(t testutil.TestingT, ctx context.Context, path, key string) (st
|
||||||
return stdout.String(), err
|
return stdout.String(), err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runResourceWithStderr(t testutil.TestingT, ctx context.Context, path, key string) (string, string) {
|
||||||
|
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
|
||||||
|
ctx = cmdio.NewContext(ctx, cmdio.Default())
|
||||||
|
|
||||||
|
c := testcli.NewRunner(t, ctx, "bundle", "run", key)
|
||||||
|
stdout, stderr, err := c.Run()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return stdout.String(), stderr.String()
|
||||||
|
}
|
||||||
|
|
||||||
func runResourceWithParams(t testutil.TestingT, ctx context.Context, path, key string, params ...string) (string, error) {
|
func runResourceWithParams(t testutil.TestingT, ctx context.Context, path, key string, params ...string) (string, error) {
|
||||||
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
|
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
|
||||||
ctx = cmdio.NewContext(ctx, cmdio.Default())
|
ctx = cmdio.NewContext(ctx, cmdio.Default())
|
||||||
|
|
|
@ -7,7 +7,7 @@ type elementsByKey struct {
|
||||||
keyFunc func(dyn.Value) string
|
keyFunc func(dyn.Value) string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) {
|
||||||
// We know the type of this value is a sequence.
|
// We know the type of this value is a sequence.
|
||||||
// For additional defence, return self if it is not.
|
// For additional defence, return self if it is not.
|
||||||
elements, ok := v.AsSequence()
|
elements, ok := v.AsSequence()
|
||||||
|
@ -33,7 +33,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Merge this instance into the reference.
|
// Merge this instance into the reference.
|
||||||
nv, err := Merge(ref, elements[i])
|
nv, err := mergeFunc(ref, elements[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v, err
|
return v, err
|
||||||
}
|
}
|
||||||
|
@ -55,6 +55,26 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
return dyn.NewValue(out, v.Locations()), nil
|
return dyn.NewValue(out, v.Locations()), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
return e.doMap(nil, v, Merge)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) {
|
||||||
|
return Override(a, b, OverrideVisitor{
|
||||||
|
VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
return v, nil
|
||||||
|
},
|
||||||
|
VisitDelete: func(valuePath dyn.Path, left dyn.Value) error {
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) {
|
||||||
|
return b, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence
|
// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence
|
||||||
// where each element is a map. It groups elements by a key and merges
|
// where each element is a map. It groups elements by a key and merges
|
||||||
// elements with the same key.
|
// elements with the same key.
|
||||||
|
@ -65,3 +85,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
|
func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
|
||||||
return elementsByKey{key, keyFunc}.Map
|
return elementsByKey{key, keyFunc}.Map
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
|
||||||
|
return elementsByKey{key, keyFunc}.MapWithOverride
|
||||||
|
}
|
||||||
|
|
|
@ -50,3 +50,42 @@ func TestElementByKey(t *testing.T) {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestElementByKeyWithOverride(t *testing.T) {
|
||||||
|
vin := dyn.V([]dyn.Value{
|
||||||
|
dyn.V(map[string]dyn.Value{
|
||||||
|
"key": dyn.V("foo"),
|
||||||
|
"value": dyn.V(42),
|
||||||
|
}),
|
||||||
|
dyn.V(map[string]dyn.Value{
|
||||||
|
"key": dyn.V("bar"),
|
||||||
|
"value": dyn.V(43),
|
||||||
|
}),
|
||||||
|
dyn.V(map[string]dyn.Value{
|
||||||
|
"key": dyn.V("foo"),
|
||||||
|
"othervalue": dyn.V(44),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
|
||||||
|
keyFunc := func(v dyn.Value) string {
|
||||||
|
return strings.ToLower(v.MustString())
|
||||||
|
}
|
||||||
|
|
||||||
|
vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, vout.MustSequence(), 2)
|
||||||
|
assert.Equal(t,
|
||||||
|
vout.Index(0).AsAny(),
|
||||||
|
map[string]any{
|
||||||
|
"key": "foo",
|
||||||
|
"othervalue": 44,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert.Equal(t,
|
||||||
|
vout.Index(1).AsAny(),
|
||||||
|
map[string]any{
|
||||||
|
"key": "bar",
|
||||||
|
"value": 43,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue