Compare commits

...

17 Commits

Author SHA1 Message Date
Andrew Nester cb7bca22af
added missing err check 2024-12-16 15:37:53 +01:00
Andrew Nester 1e74a0d257
Added support for bundle generate and bind for Apps 2024-12-16 15:37:53 +01:00
Andrew Nester 6b7f4de652
Added support for Databricks Apps in DABs 2024-12-16 15:37:51 +01:00
Andrew Nester dde1b020fc
fixed tests after rebase 2024-12-16 15:29:04 +01:00
Andrew Nester 438c19f2af
fixed lint 2024-12-16 15:27:23 +01:00
Andrew Nester 59c8254770
test fixes 2024-12-16 15:27:23 +01:00
Andrew Nester cecc5695cd
fixes 2024-12-16 15:27:22 +01:00
Andrew Nester b6af5b44df
fixes after rebase 2024-12-16 15:27:22 +01:00
Andrew Nester 298d745fb5
use TF provider 1.61.0 2024-12-16 15:27:22 +01:00
Andrew Nester 6915222ca5
interpolate after 2024-12-16 15:27:22 +01:00
Andrew Nester fc00adb427
addressed feedback 2024-12-16 15:27:22 +01:00
Andrew Nester c4e7b52832
fix fmt 2024-12-16 15:27:21 +01:00
Andrew Nester d8a9b2f4df
addressed feedback 2024-12-16 15:27:21 +01:00
Andrew Nester d8f210a2ad
chnaged to wait for deployment on start 2024-12-16 15:27:21 +01:00
Andrew Nester 79dd75a926
fix test 2024-12-16 15:27:21 +01:00
Andrew Nester 79323ee356
Added integration test 2024-12-16 15:27:21 +01:00
Andrew Nester 98140b4087
Added support for Databricks Apps in DABs 2024-12-16 15:27:18 +01:00
46 changed files with 2055 additions and 3 deletions

View File

@ -0,0 +1,69 @@
package apps
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/dynvar"
)
type interpolateVariables struct{}
func (i *interpolateVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
pattern := dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("apps"),
dyn.AnyKey(),
dyn.Key("config"),
)
err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
switch path[0] {
case dyn.Key("databricks_pipeline"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines")).Append(path[1:]...)
case dyn.Key("databricks_job"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")).Append(path[1:]...)
case dyn.Key("databricks_mlflow_model"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("models")).Append(path[1:]...)
case dyn.Key("databricks_mlflow_experiment"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("experiments")).Append(path[1:]...)
case dyn.Key("databricks_model_serving"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("model_serving_endpoints")).Append(path[1:]...)
case dyn.Key("databricks_registered_model"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("registered_models")).Append(path[1:]...)
case dyn.Key("databricks_quality_monitor"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("quality_monitors")).Append(path[1:]...)
case dyn.Key("databricks_schema"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("schemas")).Append(path[1:]...)
case dyn.Key("databricks_volume"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("volumes")).Append(path[1:]...)
case dyn.Key("databricks_cluster"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters")).Append(path[1:]...)
case dyn.Key("databricks_dashboard"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("dashboards")).Append(path[1:]...)
case dyn.Key("databricks_app"):
path = dyn.NewPath(dyn.Key("resources"), dyn.Key("apps")).Append(path[1:]...)
default:
// Trigger "key not found" for unknown resource types.
return dyn.GetByPath(root, path)
}
return dyn.GetByPath(root, path)
})
})
})
return diag.FromErr(err)
}
func (i *interpolateVariables) Name() string {
return "apps.InterpolateVariables"
}
func InterpolateVariables() bundle.Mutator {
return &interpolateVariables{}
}

View File

@ -0,0 +1,49 @@
package apps
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/require"
)
func TestAppInterpolateVariables(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Apps: map[string]*resources.App{
"my_app_1": {
App: &apps.App{
Name: "my_app_1",
},
Config: map[string]any{
"command": []string{"echo", "hello"},
"env": []map[string]string{
{"name": "JOB_ID", "value": "${resources.jobs.my_job.id}"},
},
},
},
"my_app_2": {
App: &apps.App{
Name: "my_app_2",
},
},
},
Jobs: map[string]*resources.Job{
"my_job": {
ID: "123",
},
},
},
},
}
diags := bundle.Apply(context.Background(), b, InterpolateVariables())
require.Empty(t, diags)
require.Equal(t, []any([]any{map[string]any{"name": "JOB_ID", "value": "123"}}), b.Config.Resources.Apps["my_app_1"].Config["env"])
require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config)
}

View File

@ -0,0 +1,105 @@
package apps
import (
"bytes"
"context"
"fmt"
"path"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/deploy"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/filer"
"golang.org/x/sync/errgroup"
"gopkg.in/yaml.v3"
)
type uploadConfig struct {
filerFactory deploy.FilerFactory
}
func (u *uploadConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
var diags diag.Diagnostics
errGroup, ctx := errgroup.WithContext(ctx)
for key, app := range b.Config.Resources.Apps {
// If the app has a config, we need to deploy it first.
// It means we need to write app.yml file with the content of the config field
// to the remote source code path of the app.
if app.Config != nil {
if !strings.HasPrefix(app.SourceCodePath, b.Config.Workspace.FilePath) {
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: "App source code invalid",
Detail: fmt.Sprintf("App source code path %s is not within file path %s", app.SourceCodePath, b.Config.Workspace.FilePath),
Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
})
continue
}
appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
buf, err := configToYaml(app)
if err != nil {
return diag.FromErr(err)
}
// When the app is started, create a new app deployment and wait for it to complete.
f, err := u.filerFactory(b)
if err != nil {
return diag.FromErr(err)
}
errGroup.Go(func() error {
err = f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists)
if err != nil {
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: "Failed to save config",
Detail: fmt.Sprintf("Failed to write %s file: %s", path.Join(app.SourceCodePath, "app.yml"), err),
Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s", key)),
})
}
return nil
})
}
}
if err := errGroup.Wait(); err != nil {
return diag.FromErr(err)
}
return diags
}
// Name implements bundle.Mutator.
func (u *uploadConfig) Name() string {
return "apps:UploadConfig"
}
func UploadConfig() bundle.Mutator {
return &uploadConfig{
filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath)
},
}
}
func configToYaml(app *resources.App) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
enc := yaml.NewEncoder(buf)
enc.SetIndent(2)
err := enc.Encode(app.Config)
defer enc.Close()
if err != nil {
return nil, fmt.Errorf("failed to encode app config to yaml: %w", err)
}
return buf, nil
}

View File

@ -0,0 +1,69 @@
package apps
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestAppUploadConfig(t *testing.T) {
root := t.TempDir()
err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
require.NoError(t, err)
b := &bundle.Bundle{
BundleRootPath: root,
SyncRoot: vfs.MustNew(root),
Config: config.Root{
Workspace: config.Workspace{
RootPath: "/Workspace/Users/foo@bar.com/",
},
Resources: config.Resources{
Apps: map[string]*resources.App{
"my_app": {
App: &apps.App{
Name: "my_app",
},
SourceCodePath: "./my_app",
Config: map[string]any{
"command": []string{"echo", "hello"},
"env": []map[string]string{
{"name": "MY_APP", "value": "my value"},
},
},
},
},
},
},
}
mockFiler := mockfiler.NewMockFiler(t)
mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command:
- echo
- hello
env:
- name: MY_APP
value: my value
`), filer.OverwriteIfExists).Return(nil)
u := uploadConfig{
filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
return mockFiler, nil
},
}
diags := bundle.Apply(context.Background(), b, &u)
require.NoError(t, diags.Error())
}

40
bundle/apps/validate.go Normal file
View File

@ -0,0 +1,40 @@
package apps
import (
"context"
"fmt"
"path"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
)
type validate struct{}
func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
var diags diag.Diagnostics
possibleConfigFiles := []string{"app.yml", "app.yaml"}
for _, app := range b.Config.Resources.Apps {
for _, configFile := range possibleConfigFiles {
cf := path.Join(app.SourceCodePath, configFile)
if _, err := b.SyncRoot.Stat(cf); err == nil {
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("%s detected", configFile),
Detail: fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name),
})
}
}
}
return diags
}
func (v *validate) Name() string {
return "apps.Validate"
}
func Validate() bundle.Mutator {
return &validate{}
}

View File

@ -0,0 +1,55 @@
package apps
import (
"context"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/require"
)
func TestAppsValidate(t *testing.T) {
tmpDir := t.TempDir()
testutil.Touch(t, tmpDir, "app1", "app.yml")
testutil.Touch(t, tmpDir, "app2", "app.py")
b := &bundle.Bundle{
BundleRootPath: tmpDir,
SyncRootPath: tmpDir,
SyncRoot: vfs.MustNew(tmpDir),
Config: config.Root{
Resources: config.Resources{
Apps: map[string]*resources.App{
"app1": {
App: &apps.App{
Name: "app1",
},
SourceCodePath: "./app1",
},
"app2": {
App: &apps.App{
Name: "app2",
},
SourceCodePath: "./app2",
},
},
},
},
}
bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
require.Len(t, diags, 1)
require.Equal(t, "app.yml detected", diags[0].Summary)
require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource")
}

View File

@ -0,0 +1,37 @@
package generate
import (
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/apps"
)
func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]interface{}) (dyn.Value, error) {
ac, err := convert.FromTyped(appConfig, dyn.NilValue)
if err != nil {
return dyn.NilValue, err
}
ar, err := convert.FromTyped(app.Resources, dyn.NilValue)
if err != nil {
return dyn.NilValue, err
}
// The majority of fields of the app struct are read-only.
// We copy the relevant fields manually.
dv := map[string]dyn.Value{
"name": dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}),
"description": dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}),
"source_code_path": dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 3}}),
}
if ac.Kind() != dyn.KindNil {
dv["config"] = ac.WithLocations([]dyn.Location{{Line: 4}})
}
if ar.Kind() != dyn.KindNil {
dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}})
}
return dyn.V(dv), nil
}

View File

@ -222,6 +222,8 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
dashboard.DisplayName = prefix + dashboard.DisplayName
}
// Apps: No presets
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
if !isDatabricksWorkspace {

View File

@ -12,6 +12,7 @@ import (
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/require"
@ -482,3 +483,59 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
})
}
}
func TestApplyPresetsPrefixForApps(t *testing.T) {
tests := []struct {
name string
prefix string
app *resources.App
want string
}{
{
name: "add prefix to app",
prefix: "[prefix] ",
app: &resources.App{
App: &apps.App{
Name: "app1",
},
},
want: "prefix-app1",
},
{
name: "add empty prefix to app",
prefix: "",
app: &resources.App{
App: &apps.App{
Name: "app1",
},
},
want: "app1",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Apps: map[string]*resources.App{
"app1": tt.app,
},
},
Presets: config.Presets{
NamePrefix: tt.prefix,
},
},
}
ctx := context.Background()
diag := bundle.Apply(ctx, b, mutator.ApplyPresets())
if diag.HasError() {
t.Fatalf("unexpected error: %v", diag)
}
require.Equal(t, tt.want, b.Config.Resources.Apps["app1"].Name)
})
}
}

View File

@ -0,0 +1,45 @@
package mutator
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/merge"
)
type mergeApps struct{}
func MergeApps() bundle.Mutator {
return &mergeApps{}
}
func (m *mergeApps) Name() string {
return "MergeApps"
}
func (m *mergeApps) resourceName(v dyn.Value) string {
switch v.Kind() {
case dyn.KindInvalid, dyn.KindNil:
return ""
case dyn.KindString:
return v.MustString()
default:
panic("app name must be a string")
}
}
func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
if v.Kind() == dyn.KindNil {
return v, nil
}
return dyn.Map(v, "resources.apps", dyn.Foreach(func(_ dyn.Path, app dyn.Value) (dyn.Value, error) {
return dyn.Map(app, "resources", merge.ElementsByKeyWithOverride("name", m.resourceName))
}))
})
return diag.FromErr(err)
}

View File

@ -0,0 +1,73 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/assert"
)
func TestMergeApps(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Apps: map[string]*resources.App{
"foo": {
App: &apps.App{
Name: "foo",
Resources: []apps.AppResource{
{
Name: "job1",
Job: &apps.AppResourceJob{
Id: "1234",
Permission: "CAN_MANAGE_RUN",
},
},
{
Name: "sql1",
SqlWarehouse: &apps.AppResourceSqlWarehouse{
Id: "5678",
Permission: "CAN_USE",
},
},
{
Name: "job1",
Job: &apps.AppResourceJob{
Id: "1234",
Permission: "CAN_MANAGE",
},
},
{
Name: "sql1",
Job: &apps.AppResourceJob{
Id: "9876",
Permission: "CAN_MANAGE",
},
},
},
},
},
},
},
},
}
diags := bundle.Apply(context.Background(), b, mutator.MergeApps())
assert.NoError(t, diags.Error())
j := b.Config.Resources.Apps["foo"]
assert.Len(t, j.Resources, 2)
assert.Equal(t, "job1", j.Resources[0].Name)
assert.Equal(t, "sql1", j.Resources[1].Name)
assert.Equal(t, "CAN_MANAGE", string(j.Resources[0].Job.Permission))
assert.Nil(t, j.Resources[1].SqlWarehouse)
assert.Equal(t, "CAN_MANAGE", string(j.Resources[1].Job.Permission))
}

View File

@ -15,6 +15,7 @@ import (
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/vfs"
sdkconfig "github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/dashboards"
@ -144,6 +145,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
},
},
},
Apps: map[string]*resources.App{
"app1": {
App: &apps.App{
Name: "app1",
},
},
},
},
},
SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
@ -418,6 +426,13 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
for _, key := range field.MapKeys() {
resource := field.MapIndex(key)
nameField := resource.Elem().FieldByName("Name")
resourceType := resources.Type().Field(i).Name
// Skip apps, as they are not renamed
if resourceType == "Apps" {
continue
}
if !nameField.IsValid() || nameField.Kind() != reflect.String {
continue
}

View File

@ -119,6 +119,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
))
}
// Apps do not support run_as in the API.
if len(b.Config.Resources.Apps) > 0 {
diags = diags.Extend(reportRunAsNotSupported(
"apps",
b.Config.GetLocation("resources.apps"),
b.Config.Workspace.CurrentUser.UserName,
identity,
))
}
return diags
}

View File

@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string {
// the dyn library gives us the correct list of all resources supported. Please
// also update this check when adding a new resource
require.Equal(t, []string{
"apps",
"clusters",
"dashboards",
"experiments",
@ -143,6 +144,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
"experiments",
"schemas",
"volumes",
"apps",
}
base := config.Root{

View File

@ -262,6 +262,7 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
t.applyPipelineTranslations,
t.applyArtifactTranslations,
t.applyDashboardTranslations,
t.applyAppsTranslations,
} {
v, err = fn(v)
if err != nil {

View File

@ -0,0 +1,28 @@
package mutator
import (
"fmt"
"github.com/databricks/cli/libs/dyn"
)
func (t *translateContext) applyAppsTranslations(v dyn.Value) (dyn.Value, error) {
// Convert the `source_code_path` field to a remote absolute path.
// We use this path for app deployment to point to the source code.
pattern := dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("apps"),
dyn.AnyKey(),
dyn.Key("source_code_path"),
)
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
key := p[2].Key()
dir, err := v.Location().Directory()
if err != nil {
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for app %s: %w", key, err)
}
return t.rewriteRelativeTo(p, v, t.translateDirectoryPath, dir, "")
})
}

View File

@ -0,0 +1,57 @@
package mutator_test
import (
"context"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) {
dir := t.TempDir()
touchEmptyFile(t, filepath.Join(dir, "src", "app", "app.py"))
b := &bundle.Bundle{
SyncRootPath: dir,
SyncRoot: vfs.MustNew(dir),
Config: config.Root{
Workspace: config.Workspace{
FilePath: "/bundle/files",
},
Resources: config.Resources{
Apps: map[string]*resources.App{
"app": {
App: &apps.App{
Name: "My App",
},
SourceCodePath: "../src/app",
},
},
},
},
}
bundletest.SetLocation(b, "resources.apps", []dyn.Location{{
File: filepath.Join(dir, "resources/app.yml"),
}})
diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
require.NoError(t, diags.Error())
// Assert that the file path for the app has been converted to its local absolute path.
assert.Equal(
t,
"/bundle/files/src/app",
b.Config.Resources.Apps["app"].SourceCodePath,
)
}

View File

@ -23,6 +23,7 @@ type Resources struct {
Volumes map[string]*resources.Volume `json:"volumes,omitempty"`
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
Apps map[string]*resources.App `json:"apps,omitempty"`
}
type ConfigResource interface {
@ -87,6 +88,7 @@ func (r *Resources) AllResources() []ResourceGroup {
collectResourceMap(descriptions["clusters"], r.Clusters),
collectResourceMap(descriptions["dashboards"], r.Dashboards),
collectResourceMap(descriptions["volumes"], r.Volumes),
collectResourceMap(descriptions["apps"], r.Apps),
}
}
@ -97,12 +99,19 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
found = append(found, r.Jobs[k])
}
}
for k := range r.Pipelines {
if k == key {
found = append(found, r.Pipelines[k])
}
}
for k := range r.Apps {
if k == key {
found = append(found, r.Apps[k])
}
}
if len(found) == 0 {
return nil, fmt.Errorf("no such resource: %s", key)
}
@ -197,5 +206,11 @@ func SupportedResources() map[string]ResourceDescription {
SingularTitle: "Volume",
PluralTitle: "Volumes",
},
"apps": {
SingularName: "app",
PluralName: "apps",
SingularTitle: "App",
PluralTitle: "Apps",
},
}
}

View File

@ -0,0 +1,76 @@
package resources
import (
"context"
"fmt"
"net/url"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/apps"
)
type App struct {
// This represents the id which is the name of the app that can be used
// as a reference in other resources. This value is returned by terraform.
// This equals to app name and added for symmetry with other resources.
ID string `json:"id,omitempty" bundle:"readonly"`
// SourceCodePath is a required field used by DABs to point to Databricks app source code
// on local disk and to the corresponding workspace path during app deployment.
SourceCodePath string `json:"source_code_path"`
// Config is an optional field which allows configuring the app following Databricks app configuration format like in app.yml.
// When this field is set, DABs read the configuration set in this field and write
// it to app.yml in the root of the source code folder in Databricks workspace.
// If theres app.yml defined locally, DABs will raise an error.
Config map[string]any `json:"config,omitempty"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`
*apps.App
}
func (a *App) UnmarshalJSON(b []byte) error {
return marshal.Unmarshal(b, a)
}
func (a App) MarshalJSON() ([]byte, error) {
return marshal.Marshal(a)
}
func (a *App) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) {
_, err := w.Apps.GetByName(ctx, name)
if err != nil {
log.Debugf(ctx, "app %s does not exist", name)
return false, err
}
return true, nil
}
func (a *App) TerraformResourceName() string {
return "databricks_app"
}
func (a *App) InitializeURL(baseURL url.URL) {
if a.ID == "" {
return
}
baseURL.Path = fmt.Sprintf("apps/%s", a.ID)
a.URL = baseURL.String()
}
func (a *App) GetName() string {
return a.Name
}
func (a *App) GetURL() string {
return a.URL
}
func (a *App) IsNil() bool {
return a.App == nil
}

View File

@ -196,6 +196,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
}
cur.ID = instance.Attributes.ID
config.Resources.Dashboards[resource.Name] = cur
case "databricks_app":
if config.Resources.Apps == nil {
config.Resources.Apps = make(map[string]*resources.App)
}
cur := config.Resources.Apps[resource.Name]
if cur == nil {
cur = &resources.App{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Apps[resource.Name] = cur
case "databricks_permissions":
case "databricks_grants":
// Ignore; no need to pull these back into the configuration.
@ -260,6 +270,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
for _, src := range config.Resources.Apps {
if src.ModifiedStatus == "" && src.ID == "" {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
return nil
}

View File

@ -10,6 +10,7 @@ import (
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/dashboards"
@ -694,6 +695,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_app",
Mode: "managed",
Name: "test_app",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
},
}
err := TerraformToBundle(&tfState, &config)
@ -732,6 +741,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
assert.Equal(t, "1", config.Resources.Apps["test_app"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -815,6 +827,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
},
},
},
Apps: map[string]*resources.App{
"test_app": {
App: &apps.App{
Name: "test_app",
},
},
},
},
}
tfState := resourcesState{
@ -856,6 +875,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
assert.Equal(t, "", config.Resources.Apps["test_app"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -994,6 +1016,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
},
},
},
Apps: map[string]*resources.App{
"test_app": {
App: &apps.App{
Name: "test_app",
},
},
"test_app_new": {
App: &apps.App{
Name: "test_app_new",
},
},
},
},
}
tfState := resourcesState{
@ -1174,6 +1208,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_app",
Mode: "managed",
Name: "test_app",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_app",
Mode: "managed",
Name: "test_app_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
},
}
err := TerraformToBundle(&tfState, &config)
@ -1256,6 +1306,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus)
assert.Equal(t, "1", config.Resources.Apps["test_app"].ID)
assert.Equal(t, "", config.Resources.Apps["test_app"].ModifiedStatus)
assert.Equal(t, "2", config.Resources.Apps["test_app_old"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app_old"].ModifiedStatus)
assert.Equal(t, "", config.Resources.Apps["test_app_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app_new"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}

View File

@ -63,6 +63,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
case dyn.Key("dashboards"):
path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...)
case dyn.Key("apps"):
path = dyn.NewPath(dyn.Key("databricks_app")).Append(path[2:]...)
default:
// Trigger "key not found" for unknown resource types.
return dyn.GetByPath(root, path)

View File

@ -34,6 +34,7 @@ func TestInterpolate(t *testing.T) {
"other_volume": "${resources.volumes.other_volume.id}",
"other_cluster": "${resources.clusters.other_cluster.id}",
"other_dashboard": "${resources.dashboards.other_dashboard.id}",
"other_app": "${resources.apps.other_app.id}",
},
Tasks: []jobs.Task{
{
@ -73,6 +74,7 @@ func TestInterpolate(t *testing.T) {
assert.Equal(t, "${databricks_volume.other_volume.id}", j.Tags["other_volume"])
assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"])
assert.Equal(t, "${databricks_app.other_app.id}", j.Tags["other_app"])
m := b.Config.Resources.Models["my_model"]
assert.Equal(t, "my_model", m.Model.Name)

View File

@ -0,0 +1,46 @@
package tfdyn
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go/service/apps"
)
func convertAppResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
// Normalize the output value to the target schema.
vout, diags := convert.Normalize(apps.App{}, vin)
for _, diag := range diags {
log.Debugf(ctx, "app normalization diagnostic: %s", diag.Summary)
}
return vout, nil
}
type appConverter struct{}
func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
vout, err := convertAppResource(ctx, vin)
if err != nil {
return err
}
// Add the converted resource to the output.
out.App[key] = vout.AsAny()
// Configure permissions for this resource.
if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
permissions.AppName = fmt.Sprintf("${databricks_app.%s.name}", key)
out.Permissions["app_"+key] = permissions
}
return nil
}
func init() {
registerConverter("apps", appConverter{})
}

View File

@ -0,0 +1,98 @@
package tfdyn
import (
"context"
"testing"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertApp(t *testing.T) {
src := resources.App{
SourceCodePath: "./app",
Config: map[string]any{
"command": []string{"python", "app.py"},
},
App: &apps.App{
Name: "app_id",
Description: "app description",
Resources: []apps.AppResource{
{
Name: "job1",
Job: &apps.AppResourceJob{
Id: "1234",
Permission: "CAN_MANAGE_RUN",
},
},
{
Name: "sql1",
SqlWarehouse: &apps.AppResourceSqlWarehouse{
Id: "5678",
Permission: "CAN_USE",
},
},
},
},
Permissions: []resources.Permission{
{
Level: "CAN_RUN",
UserName: "jack@gmail.com",
},
{
Level: "CAN_MANAGE",
ServicePrincipalName: "sp",
},
},
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = appConverter{}.Convert(ctx, "my_app", vin, out)
require.NoError(t, err)
app := out.App["my_app"]
assert.Equal(t, map[string]any{
"description": "app description",
"name": "app_id",
"resources": []any{
map[string]any{
"name": "job1",
"job": map[string]any{
"id": "1234",
"permission": "CAN_MANAGE_RUN",
},
},
map[string]any{
"name": "sql1",
"sql_warehouse": map[string]any{
"id": "5678",
"permission": "CAN_USE",
},
},
},
}, app)
// Assert equality on the permissions
assert.Equal(t, &schema.ResourcePermissions{
AppName: "${databricks_app.my_app.name}",
AccessControl: []schema.ResourcePermissionsAccessControl{
{
PermissionLevel: "CAN_RUN",
UserName: "jack@gmail.com",
},
{
PermissionLevel: "CAN_MANAGE",
ServicePrincipalName: "sp",
},
},
}, out.Permissions["app_my_app"])
}

View File

@ -51,6 +51,10 @@ var (
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_READ",
},
"apps": {
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_USE",
},
}
)

View File

@ -58,6 +58,10 @@ func TestApplyBundlePermissions(t *testing.T) {
"dashboard_1": {},
"dashboard_2": {},
},
Apps: map[string]*resources.App{
"app_1": {},
"app_2": {},
},
},
},
}
@ -114,6 +118,10 @@ func TestApplyBundlePermissions(t *testing.T) {
require.Len(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, 2)
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
require.Contains(t, b.Config.Resources.Dashboards["dashboard_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"})
require.Len(t, b.Config.Resources.Apps["app_1"].Permissions, 2)
require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"})
require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"})
}
func TestWarningOnOverlapPermission(t *testing.T) {

View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/apps"
"github.com/databricks/cli/bundle/artifacts"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
@ -135,6 +136,8 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
bundle.Seq(
terraform.StatePush(),
terraform.Load(),
apps.InterpolateVariables(),
apps.UploadConfig(),
metadata.Compute(),
metadata.Upload(),
bundle.LogString("Deployment complete!"),

View File

@ -2,6 +2,7 @@ package phases
import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/apps"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
pythonmutator "github.com/databricks/cli/bundle/config/mutator/python"
@ -37,6 +38,8 @@ func Initialize() bundle.Mutator {
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),
mutator.MergePipelineClusters(),
mutator.MergeApps(),
mutator.InitializeWorkspaceClient(),
mutator.PopulateCurrentUser(),
mutator.LoadGitDetails(),
@ -80,6 +83,8 @@ func Initialize() bundle.Mutator {
mutator.TranslatePaths(),
trampoline.WrapperWarning(),
apps.Validate(),
permissions.ValidateSharedRootPermissions(),
permissions.ApplyBundlePermissions(),
permissions.FilterCurrentUser(),

211
bundle/run/app.go Normal file
View File

@ -0,0 +1,211 @@
package run
import (
"context"
"fmt"
"time"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/run/output"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/spf13/cobra"
)
func logProgress(ctx context.Context, msg string) {
if msg == "" {
return
}
cmdio.LogString(ctx, fmt.Sprintf("✓ %s", msg))
}
type appRunner struct {
key
bundle *bundle.Bundle
app *resources.App
}
func (a *appRunner) Name() string {
if a.app == nil {
return ""
}
return a.app.Name
}
func isAppStopped(app *apps.App) bool {
return app.ComputeStatus == nil ||
(app.ComputeStatus.State == apps.ComputeStateStopped || app.ComputeStatus.State == apps.ComputeStateError)
}
func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) {
app := a.app
b := a.bundle
if app == nil {
return nil, fmt.Errorf("app is not defined")
}
logProgress(ctx, fmt.Sprintf("Getting the status of the app %s", app.Name))
w := b.WorkspaceClient()
// Check the status of the app first.
createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
if err != nil {
return nil, err
}
if createdApp.AppStatus != nil {
logProgress(ctx, fmt.Sprintf("App is in %s state", createdApp.AppStatus.State))
}
if createdApp.ComputeStatus != nil {
logProgress(ctx, fmt.Sprintf("App compute is in %s state", createdApp.ComputeStatus.State))
}
// There could be 2 reasons why the app is not running:
// 1. The app is new and was never deployed yet.
// 2. The app was stopped (compute not running).
// We need to start the app only if the compute is not running.
if isAppStopped(createdApp) {
err := a.start(ctx)
if err != nil {
return nil, err
}
}
// Deploy the app.
err = a.deploy(ctx)
if err != nil {
return nil, err
}
cmdio.LogString(ctx, fmt.Sprintf("You can access the app at %s", createdApp.Url))
return nil, nil
}
func (a *appRunner) start(ctx context.Context) error {
app := a.app
b := a.bundle
w := b.WorkspaceClient()
logProgress(ctx, fmt.Sprintf("Starting the app %s", app.Name))
wait, err := w.Apps.Start(ctx, apps.StartAppRequest{Name: app.Name})
if err != nil {
return err
}
startedApp, err := wait.OnProgress(func(p *apps.App) {
if p.AppStatus == nil {
return
}
logProgress(ctx, "App is starting...")
}).Get()
if err != nil {
return err
}
// After the app is started (meaning the compute is running), the API will return the app object with the
// active and pending deployments fields (if any). If there are active or pending deployments,
// we need to wait for them to complete before we can do the new deployment.
// Otherwise, the new deployment will fail.
// Thus, we first wait for the active deployment to complete.
if startedApp.ActiveDeployment != nil &&
startedApp.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
logProgress(ctx, "Waiting for the active deployment to complete...")
_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
if err != nil {
return err
}
logProgress(ctx, "Active deployment is completed!")
}
// Then, we wait for the pending deployment to complete.
if startedApp.PendingDeployment != nil &&
startedApp.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
logProgress(ctx, "Waiting for the pending deployment to complete...")
_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.PendingDeployment.DeploymentId, 20*time.Minute, nil)
if err != nil {
return err
}
logProgress(ctx, "Pending deployment is completed!")
}
logProgress(ctx, "App is started!")
return nil
}
func (a *appRunner) deploy(ctx context.Context) error {
app := a.app
b := a.bundle
w := b.WorkspaceClient()
wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
AppName: app.Name,
AppDeployment: &apps.AppDeployment{
Mode: apps.AppDeploymentModeSnapshot,
SourceCodePath: app.SourceCodePath,
},
})
// If deploy returns an error, then there's an active deployment in progress, wait for it to complete.
if err != nil {
return err
}
_, err = wait.OnProgress(func(ad *apps.AppDeployment) {
if ad.Status == nil {
return
}
logProgress(ctx, ad.Status.Message)
}).Get()
if err != nil {
return err
}
return nil
}
func (a *appRunner) Cancel(ctx context.Context) error {
// We should cancel the app by stopping it.
app := a.app
b := a.bundle
if app == nil {
return fmt.Errorf("app is not defined")
}
w := b.WorkspaceClient()
logProgress(ctx, fmt.Sprintf("Stopping app %s", app.Name))
wait, err := w.Apps.Stop(ctx, apps.StopAppRequest{Name: app.Name})
if err != nil {
return err
}
_, err = wait.OnProgress(func(p *apps.App) {
if p.AppStatus == nil {
return
}
logProgress(ctx, p.AppStatus.Message)
}).Get()
logProgress(ctx, "App is stopped!")
return err
}
func (a *appRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) {
// We should restart the app by just running it again meaning a new app deployment will be done.
return a.Run(ctx, opts)
}
func (a *appRunner) ParseArgs(args []string, opts *Options) error {
if len(args) == 0 {
return nil
}
return fmt.Errorf("received %d unexpected positional arguments", len(args))
}
func (a *appRunner) CompleteArgs(args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return nil, cobra.ShellCompDirectiveNoFileComp
}

216
bundle/run/app_test.go Normal file
View File

@ -0,0 +1,216 @@
package run
import (
"bytes"
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
type testAppRunner struct {
m *mocks.MockWorkspaceClient
b *bundle.Bundle
ctx context.Context
}
func (ta *testAppRunner) run(t *testing.T) {
r := appRunner{
key: "my_app",
bundle: ta.b,
app: ta.b.Config.Resources.Apps["my_app"],
}
_, err := r.Run(ta.ctx, &Options{})
require.NoError(t, err)
}
func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWorkspaceClient) {
root := t.TempDir()
err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700)
require.NoError(t, err)
b := &bundle.Bundle{
BundleRootPath: root,
SyncRoot: vfs.MustNew(root),
Config: config.Root{
Workspace: config.Workspace{
RootPath: "/Workspace/Users/foo@bar.com/",
},
Resources: config.Resources{
Apps: map[string]*resources.App{
"my_app": {
App: &apps.App{
Name: "my_app",
},
SourceCodePath: "./my_app",
Config: map[string]any{
"command": []string{"echo", "hello"},
"env": []map[string]string{
{"name": "MY_APP", "value": "my value"},
},
},
},
},
},
},
}
mwc := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(mwc.WorkspaceClient)
bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: "./databricks.yml"}})
ctx := context.Background()
ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "..."))
ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend))
diags := bundle.Apply(ctx, b, bundle.Seq(
mutator.DefineDefaultWorkspacePaths(),
mutator.TranslatePaths(),
))
require.Empty(t, diags)
return ctx, b, mwc
}
func setupTestApp(t *testing.T, initialAppState apps.ApplicationState, initialComputeState apps.ComputeState) *testAppRunner {
ctx, b, mwc := setupBundle(t)
appApi := mwc.GetMockAppsAPI()
appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
Name: "my_app",
}).Return(&apps.App{
Name: "my_app",
AppStatus: &apps.ApplicationStatus{
State: initialAppState,
},
ComputeStatus: &apps.ComputeStatus{
State: initialComputeState,
},
}, nil)
wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
return nil, nil
},
}
appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
AppName: "my_app",
AppDeployment: &apps.AppDeployment{
Mode: apps.AppDeploymentModeSnapshot,
SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
},
}).Return(wait, nil)
return &testAppRunner{
m: mwc,
b: b,
ctx: ctx,
}
}
func TestAppRunStartedApp(t *testing.T) {
r := setupTestApp(t, apps.ApplicationStateRunning, apps.ComputeStateActive)
r.run(t)
}
func TestAppRunStoppedApp(t *testing.T) {
r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
appsApi := r.m.GetMockAppsAPI()
appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
Name: "my_app",
}).Return(&apps.WaitGetAppActive[apps.App]{
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
return &apps.App{
Name: "my_app",
AppStatus: &apps.ApplicationStatus{
State: apps.ApplicationStateRunning,
},
ComputeStatus: &apps.ComputeStatus{
State: apps.ComputeStateActive,
},
}, nil
},
}, nil)
r.run(t)
}
func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
r := setupTestApp(t, apps.ApplicationStateCrashed, apps.ComputeStateStopped)
appsApi := r.m.GetMockAppsAPI()
appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{
Name: "my_app",
}).Return(&apps.WaitGetAppActive[apps.App]{
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
return &apps.App{
Name: "my_app",
AppStatus: &apps.ApplicationStatus{
State: apps.ApplicationStateRunning,
},
ComputeStatus: &apps.ComputeStatus{
State: apps.ComputeStateActive,
},
ActiveDeployment: &apps.AppDeployment{
DeploymentId: "active_deployment_id",
Status: &apps.AppDeploymentStatus{
State: apps.AppDeploymentStateInProgress,
},
},
PendingDeployment: &apps.AppDeployment{
DeploymentId: "pending_deployment_id",
Status: &apps.AppDeploymentStatus{
State: apps.AppDeploymentStateCancelled,
},
},
}, nil
},
}, nil)
appsApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "active_deployment_id", mock.Anything, mock.Anything).Return(nil, nil)
r.run(t)
}
func TestStopApp(t *testing.T) {
ctx, b, mwc := setupBundle(t)
appsApi := mwc.GetMockAppsAPI()
appsApi.EXPECT().Stop(mock.Anything, apps.StopAppRequest{
Name: "my_app",
}).Return(&apps.WaitGetAppStopped[apps.App]{
Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) {
return &apps.App{
Name: "my_app",
AppStatus: &apps.ApplicationStatus{
State: apps.ApplicationStateUnavailable,
},
}, nil
},
}, nil)
r := appRunner{
key: "my_app",
bundle: b,
app: b.Config.Resources.Apps["my_app"],
}
err := r.Cancel(ctx)
require.NoError(t, err)
}

View File

@ -42,7 +42,7 @@ type Runner interface {
// IsRunnable returns a filter that only allows runnable resources.
func IsRunnable(ref refs.Reference) bool {
switch ref.Resource.(type) {
case *resources.Job, *resources.Pipeline:
case *resources.Job, *resources.Pipeline, *resources.App:
return true
default:
return false
@ -56,6 +56,12 @@ func ToRunner(b *bundle.Bundle, ref refs.Reference) (Runner, error) {
return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil
case *resources.Pipeline:
return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil
case *resources.App:
return &appRunner{
key: key(ref.KeyWithType),
bundle: b,
app: resource,
}, nil
default:
return nil, fmt.Errorf("unsupported resource type: %T", resource)
}

View File

@ -0,0 +1,71 @@
bundle:
name: apps
workspace:
host: https://acme.cloud.databricks.com/
variables:
app_config:
type: complex
default:
command:
- "python"
- "app.py"
env:
- name: SOME_ENV_VARIABLE
value: "Some value"
resources:
apps:
my_app:
name: "my-app"
description: "My App"
source_code_path: ./app
config: ${var.app_config}
resources:
- name: "my-sql-warehouse"
sql_warehouse:
id: 1234
permission: "CAN_USE"
- name: "my-job"
job:
id: 5678
permission: "CAN_MANAGE_RUN"
permissions:
- user_name: "foo@bar.com"
level: "CAN_VIEW"
- service_principal_name: "my_sp"
level: "CAN_MANAGE"
targets:
default:
development:
variables:
app_config:
command:
- "python"
- "dev.py"
env:
- name: SOME_ENV_VARIABLE_2
value: "Some value 2"
resources:
apps:
my_app:
source_code_path: ./app-dev
resources:
- name: "my-sql-warehouse"
sql_warehouse:
id: 1234
permission: "CAN_MANAGE"
- name: "my-job"
job:
id: 5678
permission: "CAN_MANAGE"
- name: "my-secret"
secret:
key: "key"
scope: "scope"
permission: "CAN_USE"

60
bundle/tests/apps_test.go Normal file
View File

@ -0,0 +1,60 @@
package config_tests
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/stretchr/testify/assert"
)
func TestApps(t *testing.T) {
b := load(t, "./apps")
assert.Equal(t, "apps", b.Config.Bundle.Name)
diags := bundle.Apply(context.Background(), b,
bundle.Seq(
mutator.SetVariables(),
mutator.ResolveVariableReferences("variables"),
))
assert.Empty(t, diags)
app := b.Config.Resources.Apps["my_app"]
assert.Equal(t, "my-app", app.Name)
assert.Equal(t, "My App", app.Description)
assert.Equal(t, []any{"python", "app.py"}, app.Config["command"])
assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE", "value": "Some value"}}, app.Config["env"])
assert.Len(t, app.Resources, 2)
assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
assert.Equal(t, "CAN_USE", string(app.Resources[0].SqlWarehouse.Permission))
assert.Equal(t, "5678", app.Resources[1].Job.Id)
assert.Equal(t, "CAN_MANAGE_RUN", string(app.Resources[1].Job.Permission))
}
func TestAppsOverride(t *testing.T) {
b := loadTarget(t, "./apps", "development")
assert.Equal(t, "apps", b.Config.Bundle.Name)
diags := bundle.Apply(context.Background(), b,
bundle.Seq(
mutator.SetVariables(),
mutator.ResolveVariableReferences("variables"),
))
assert.Empty(t, diags)
app := b.Config.Resources.Apps["my_app"]
assert.Equal(t, "my-app", app.Name)
assert.Equal(t, "My App", app.Description)
assert.Equal(t, []any{"python", "dev.py"}, app.Config["command"])
assert.Equal(t, []any{map[string]any{"name": "SOME_ENV_VARIABLE_2", "value": "Some value 2"}}, app.Config["env"])
assert.Len(t, app.Resources, 3)
assert.Equal(t, "1234", app.Resources[0].SqlWarehouse.Id)
assert.Equal(t, "CAN_MANAGE", string(app.Resources[0].SqlWarehouse.Permission))
assert.Equal(t, "5678", app.Resources[1].Job.Id)
assert.Equal(t, "CAN_MANAGE", string(app.Resources[1].Job.Permission))
assert.Equal(t, "key", app.Resources[2].Secret.Key)
assert.Equal(t, "scope", app.Resources[2].Secret.Scope)
assert.Equal(t, "CAN_USE", string(app.Resources[2].Secret.Permission))
}

View File

@ -46,6 +46,7 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
mutator.MergeJobParameters(),
mutator.MergeJobTasks(),
mutator.MergePipelineClusters(),
mutator.MergeApps(),
))
return b, diags
}

View File

@ -17,6 +17,7 @@ func newGenerateCommand() *cobra.Command {
cmd.AddCommand(generate.NewGenerateJobCommand())
cmd.AddCommand(generate.NewGeneratePipelineCommand())
cmd.AddCommand(generate.NewGenerateDashboardCommand())
cmd.AddCommand(generate.NewGenerateAppCommand())
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
return cmd
}

154
cmd/bundle/generate/app.go Normal file
View File

@ -0,0 +1,154 @@
package generate
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/databricks/cli/bundle/config/generate"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/yamlsaver"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/textutil"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
)
func NewGenerateAppCommand() *cobra.Command {
var configDir string
var sourceDir string
var appName string
var force bool
cmd := &cobra.Command{
Use: "app",
Short: "Generate bundle configuration for a Databricks app",
}
cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`)
cmd.MarkFlagRequired("existing-app-name")
wd, err := os.Getwd()
if err != nil {
wd = "."
}
cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join(wd, "resources"), `Directory path where the output bundle config will be stored`)
cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join(wd, "src", "app"), `Directory path where the app files will be stored`)
cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
b, diags := root.MustConfigureBundle(cmd)
if err := diags.Error(); err != nil {
return diags.Error()
}
w := b.WorkspaceClient()
cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName))
app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: appName})
if err != nil {
return err
}
downloader := newDownloader(w, sourceDir, configDir)
sourceCodePath := app.DefaultSourceCodePath
downloader.markDirectoryForDownload(ctx, &sourceCodePath)
appConfig, err := getAppConfig(ctx, app, w)
if err != nil {
return fmt.Errorf("failed to get app config: %w", err)
}
v, err := generate.ConvertAppToValue(app, sourceCodePath, appConfig)
if err != nil {
return err
}
appKey := cmd.Flag("key").Value.String()
if appKey == "" {
appKey = textutil.NormalizeString(app.Name)
}
result := map[string]dyn.Value{
"resources": dyn.V(map[string]dyn.Value{
"apps": dyn.V(map[string]dyn.Value{
appKey: v,
}),
}),
}
// If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle.
// We include this configuration inline, so we need to remove these files.
for _, configFile := range []string{"app.yml", "app.yaml"} {
delete(downloader.files, filepath.Join(sourceDir, configFile))
}
err = downloader.FlushToDisk(ctx, force)
if err != nil {
return err
}
filename := filepath.Join(configDir, fmt.Sprintf("%s.app.yml", appKey))
saver := yamlsaver.NewSaver()
err = saver.SaveAsYAML(result, filename, force)
if err != nil {
return err
}
cmdio.LogString(ctx, fmt.Sprintf("App configuration successfully saved to %s", filename))
return nil
}
return cmd
}
func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]interface{}, error) {
sourceCodePath := app.DefaultSourceCodePath
f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath)
if err != nil {
return nil, err
}
// The app config is stored in app.yml or app.yaml file in the source code path.
configFileNames := []string{"app.yml", "app.yaml"}
for _, configFile := range configFileNames {
r, err := f.Read(ctx, configFile)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
return nil, err
}
defer r.Close()
cmdio.LogString(ctx, fmt.Sprintf("Reading app configuration from %s", configFile))
content, err := io.ReadAll(r)
if err != nil {
return nil, err
}
var appConfig map[string]interface{}
err = yaml.Unmarshal(content, &appConfig)
if err != nil {
cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err))
return nil, nil
}
return appConfig, nil
}
return nil, nil
}

View File

@ -13,6 +13,7 @@ import (
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/workspace"
"golang.org/x/sync/errgroup"
)
@ -63,6 +64,37 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
return nil
}
func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
if err != nil {
return err
}
objects, err := n.w.Workspace.RecursiveList(ctx, *dirPath)
if err != nil {
return err
}
for _, obj := range objects {
if obj.ObjectType == workspace.ObjectTypeDirectory {
continue
}
err := n.markFileForDownload(ctx, &obj.Path)
if err != nil {
return err
}
}
rel, err := filepath.Rel(n.configDir, n.sourceDir)
if err != nil {
return err
}
*dirPath = rel
return nil
}
func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
if err != nil {

View File

@ -0,0 +1,101 @@
package bundle_test
import (
"fmt"
"io"
"testing"
"github.com/databricks/cli/internal/acc"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/env"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
)
func TestAccDeployBundleWithApp(t *testing.T) {
ctx, wt := acc.WorkspaceTest(t)
uniqueId := uuid.New().String()
appId := fmt.Sprintf("app-%s", uuid.New().String()[0:8])
nodeTypeId := testutil.GetCloud(t).NodeTypeID()
instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
root := initTestTemplate(t, ctx, "apps", map[string]any{
"unique_id": uniqueId,
"app_id": appId,
"node_type_id": nodeTypeId,
"spark_version": defaultSparkVersion,
"instance_pool_id": instancePoolId,
})
t.Cleanup(func() {
destroyBundle(t, ctx, root)
app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: "test-app"})
if err != nil {
require.ErrorContains(t, err, "does not exist")
} else {
require.Contains(t, []apps.ApplicationState{apps.ApplicationStateUnavailable}, app.AppStatus.State)
}
})
deployBundle(t, ctx, root)
// App should exists after bundle deployment
app, err := wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
require.NoError(t, err)
require.NotNil(t, app)
// Check app config
currentUser, err := wt.W.CurrentUser.Me(ctx)
require.NoError(t, err)
pathToAppYml := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s/files/app/app.yml", currentUser.UserName, uniqueId)
reader, err := wt.W.Workspace.Download(ctx, pathToAppYml)
require.NoError(t, err)
data, err := io.ReadAll(reader)
require.NoError(t, err)
job, err := wt.W.Jobs.GetBySettingsName(ctx, fmt.Sprintf("test-job-with-cluster-%s", uniqueId))
require.NoError(t, err)
content := string(data)
require.Contains(t, content, fmt.Sprintf(`command:
- flask
- --app
- app
- run
env:
- name: JOB_ID
value: "%d"`, job.JobId))
// Try to run the app
_, out, err := runResourceWithStderr(t, ctx, root, "test_app")
require.NoError(t, err)
require.Contains(t, out, app.Url)
// App should be in the running state
app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
require.NoError(t, err)
require.NotNil(t, app)
require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
// Stop the app
wait, err := wt.W.Apps.Stop(ctx, apps.StopAppRequest{Name: appId})
require.NoError(t, err)
app, err = wait.Get()
require.NoError(t, err)
require.NotNil(t, app)
require.Equal(t, apps.ApplicationStateUnavailable, app.AppStatus.State)
// Try to run the app again
_, out, err = runResourceWithStderr(t, ctx, root, "test_app")
require.NoError(t, err)
require.Contains(t, out, app.Url)
// App should be in the running state
app, err = wt.W.Apps.Get(ctx, apps.GetAppRequest{Name: appId})
require.NoError(t, err)
require.NotNil(t, app)
require.Equal(t, apps.ApplicationStateRunning, app.AppStatus.State)
}

View File

@ -0,0 +1,24 @@
{
"properties": {
"unique_id": {
"type": "string",
"description": "Unique ID for job name"
},
"app_id": {
"type": "string",
"description": "Unique ID for app name"
},
"spark_version": {
"type": "string",
"description": "Spark version used for job cluster"
},
"node_type_id": {
"type": "string",
"description": "Node type id for job cluster"
},
"instance_pool_id": {
"type": "string",
"description": "Instance pool id for job cluster"
}
}
}

View File

@ -0,0 +1,15 @@
import os
from databricks.sdk import WorkspaceClient
from flask import Flask
app = Flask(__name__)
@app.route("/")
def home():
job_id = os.getenv("JOB_ID")
w = WorkspaceClient()
job = w.jobs.get(job_id)
return job.settings.name

View File

@ -0,0 +1,42 @@
bundle:
name: basic
workspace:
root_path: "~/.bundle/{{.unique_id}}"
resources:
apps:
test_app:
name: "{{.app_id}}"
description: "App which manages job created by this bundle"
source_code_path: ./app
config:
command:
- flask
- --app
- app
- run
env:
- name: JOB_ID
value: ${resources.jobs.foo.id}
resources:
- name: "app-job"
description: "A job for app to be able to work with"
job:
id: ${resources.jobs.foo.id}
permission: "CAN_MANAGE_RUN"
jobs:
foo:
name: test-job-with-cluster-{{.unique_id}}
tasks:
- task_key: my_notebook_task
new_cluster:
num_workers: 1
spark_version: "{{.spark_version}}"
node_type_id: "{{.node_type_id}}"
data_security_mode: USER_ISOLATION
instance_pool_id: "{{.instance_pool_id}}"
spark_python_task:
python_file: ./hello_world.py

View File

@ -0,0 +1 @@
print("Hello World!")

View File

@ -119,6 +119,15 @@ func runResource(t testutil.TestingT, ctx context.Context, path, key string) (st
return stdout.String(), err
}
func runResourceWithStderr(t testutil.TestingT, ctx context.Context, path, key string) (string, string, error) {
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
ctx = cmdio.NewContext(ctx, cmdio.Default())
c := testcli.NewRunner(t, ctx, "bundle", "run", key)
stdout, stderr, err := c.Run()
return stdout.String(), stderr.String(), err
}
func runResourceWithParams(t testutil.TestingT, ctx context.Context, path, key string, params ...string) (string, error) {
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
ctx = cmdio.NewContext(ctx, cmdio.Default())

View File

@ -7,7 +7,7 @@ type elementsByKey struct {
keyFunc func(dyn.Value) string
}
func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
func (e elementsByKey) doMap(_ dyn.Path, v dyn.Value, mergeFunc func(a, b dyn.Value) (dyn.Value, error)) (dyn.Value, error) {
// We know the type of this value is a sequence.
// For additional defence, return self if it is not.
elements, ok := v.AsSequence()
@ -33,7 +33,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
}
// Merge this instance into the reference.
nv, err := Merge(ref, elements[i])
nv, err := mergeFunc(ref, elements[i])
if err != nil {
return v, err
}
@ -55,6 +55,26 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
return dyn.NewValue(out, v.Locations()), nil
}
func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
return e.doMap(nil, v, Merge)
}
func (e elementsByKey) MapWithOverride(p dyn.Path, v dyn.Value) (dyn.Value, error) {
return e.doMap(nil, v, func(a, b dyn.Value) (dyn.Value, error) {
return Override(a, b, OverrideVisitor{
VisitInsert: func(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
return v, nil
},
VisitDelete: func(valuePath dyn.Path, left dyn.Value) error {
return nil
},
VisitUpdate: func(_ dyn.Path, a, b dyn.Value) (dyn.Value, error) {
return b, nil
},
})
})
}
// ElementsByKey returns a [dyn.MapFunc] that operates on a sequence
// where each element is a map. It groups elements by a key and merges
// elements with the same key.
@ -65,3 +85,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) {
func ElementsByKey(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
return elementsByKey{key, keyFunc}.Map
}
func ElementsByKeyWithOverride(key string, keyFunc func(dyn.Value) string) dyn.MapFunc {
return elementsByKey{key, keyFunc}.MapWithOverride
}

View File

@ -50,3 +50,42 @@ func TestElementByKey(t *testing.T) {
},
)
}
func TestElementByKeyWithOverride(t *testing.T) {
vin := dyn.V([]dyn.Value{
dyn.V(map[string]dyn.Value{
"key": dyn.V("foo"),
"value": dyn.V(42),
}),
dyn.V(map[string]dyn.Value{
"key": dyn.V("bar"),
"value": dyn.V(43),
}),
dyn.V(map[string]dyn.Value{
"key": dyn.V("foo"),
"othervalue": dyn.V(44),
}),
})
keyFunc := func(v dyn.Value) string {
return strings.ToLower(v.MustString())
}
vout, err := dyn.MapByPath(vin, dyn.EmptyPath, ElementsByKeyWithOverride("key", keyFunc))
require.NoError(t, err)
assert.Len(t, vout.MustSequence(), 2)
assert.Equal(t,
vout.Index(0).AsAny(),
map[string]any{
"key": "foo",
"othervalue": 44,
},
)
assert.Equal(t,
vout.Index(1).AsAny(),
map[string]any{
"key": "bar",
"value": 43,
},
)
}