From ee0d261b25be7145c699b6b46f3abd54b40f5838 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 26 Nov 2024 16:08:26 +0100 Subject: [PATCH] move app deployment to bundle run --- bundle/config/mutator/apply_presets.go | 13 +- bundle/config/mutator/apply_presets_test.go | 57 ++++ .../mutator/process_target_mode_test.go | 8 + bundle/config/mutator/run_as_test.go | 2 + .../mutator/translate_paths_apps_test.go | 6 +- bundle/deploy/apps/deploy.go | 106 -------- bundle/deploy/apps/deploy_test.go | 113 -------- bundle/deploy/filer.go | 5 - bundle/deploy/terraform/tfdyn/convert_app.go | 8 +- .../terraform/tfdyn/convert_app_test.go | 29 +- bundle/internal/tf/schema/resource_app.go | 33 +-- .../tf/schema/resource_permissions.go | 1 + .../tf/schema/resource_quality_monitor.go | 46 ++-- bundle/phases/deploy.go | 2 - bundle/run/app.go | 248 ++++++++++++++++++ bundle/run/app_test.go | 208 +++++++++++++++ bundle/run/runner.go | 12 +- 17 files changed, 606 insertions(+), 291 deletions(-) delete mode 100644 bundle/deploy/apps/deploy.go delete mode 100644 bundle/deploy/apps/deploy_test.go create mode 100644 bundle/run/app.go create mode 100644 bundle/run/app_test.go diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 80a04433..33b2ea42 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -222,7 +222,18 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } - // Apps doesn't support tags or prefixes yet. + // Apps: Prefix + for key, app := range r.Apps { + if app == nil || app.App == nil { + diags = diags.Extend(diag.Errorf("app %s is not defined", key)) + continue + } + + app.Name = textutil.NormalizeString(prefix + app.Name) + // Normalize the app name to ensure it is a valid identifier. + // App supports only alphanumeric characters and hyphens. + app.Name = strings.ReplaceAll(app.Name, "_", "-") + } if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/") diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index f11a45d6..2f0e9c9c 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/dbr" + "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -449,3 +450,59 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { } } + +func TestApplyPresetsPrefixForApps(t *testing.T) { + tests := []struct { + name string + prefix string + app *resources.App + want string + }{ + { + name: "add prefix to app", + prefix: "[prefix] ", + app: &resources.App{ + App: &apps.App{ + Name: "app1", + }, + }, + want: "prefix-app1", + }, + { + name: "add empty prefix to app", + prefix: "", + app: &resources.App{ + App: &apps.App{ + Name: "app1", + }, + }, + want: "app1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "app1": tt.app, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Apps["app1"].Name) + }) + } +} diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index c5ea9ade..2a3ecff9 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/tags" "github.com/databricks/cli/libs/vfs" sdkconfig "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -141,6 +142,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Apps: map[string]*resources.App{ + "app1": { + App: &apps.App{ + Name: "app1", + }, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index acb6c3a4..aa672b8f 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string { // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource require.Equal(t, []string{ + "apps", "clusters", "dashboards", "experiments", @@ -141,6 +142,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { "registered_models", "experiments", "schemas", + "apps", } base := config.Root{ diff --git a/bundle/config/mutator/translate_paths_apps_test.go b/bundle/config/mutator/translate_paths_apps_test.go index e0293887..7ca0412d 100644 --- a/bundle/config/mutator/translate_paths_apps_test.go +++ b/bundle/config/mutator/translate_paths_apps_test.go @@ -19,7 +19,7 @@ import ( func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) { dir := t.TempDir() - touchEmptyFile(t, filepath.Join(dir, "src", "my_app.lvdash.json")) + touchEmptyFile(t, filepath.Join(dir, "src", "app", "app.py")) b := &bundle.Bundle{ SyncRootPath: dir, @@ -31,7 +31,7 @@ func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) { App: &apps.App{ Name: "My App", }, - SourceCodePath: "../src/", + SourceCodePath: "../src/app", }, }, }, @@ -48,7 +48,7 @@ func TestTranslatePathsApps_FilePathRelativeSubDirectory(t *testing.T) { // Assert that the file path for the app has been converted to its local absolute path. assert.Equal( t, - filepath.Join(dir, "src"), + filepath.Join("src", "app"), b.Config.Resources.Apps["app"].SourceCodePath, ) } diff --git a/bundle/deploy/apps/deploy.go b/bundle/deploy/apps/deploy.go deleted file mode 100644 index ff4cfa51..00000000 --- a/bundle/deploy/apps/deploy.go +++ /dev/null @@ -1,106 +0,0 @@ -package apps - -import ( - "bytes" - "context" - "fmt" - "path" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deploy" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/service/apps" - "golang.org/x/sync/errgroup" - - "gopkg.in/yaml.v3" -) - -type appsDeploy struct { - filerFactory deploy.FilerFactory -} - -func Deploy() bundle.Mutator { - return appsDeploy{deploy.AppFiler} -} - -func (a appsDeploy) Name() string { - return "apps.Deploy" -} - -func (a appsDeploy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if len(b.Config.Resources.Apps) == 0 { - return nil - } - - errGrp, ctx := errgroup.WithContext(ctx) - w := b.WorkspaceClient() - f, err := a.filerFactory(b) - if err != nil { - return diag.FromErr(err) - } - - for _, app := range b.Config.Resources.Apps { - cmdio.LogString(ctx, fmt.Sprintf("Deploying app %s...", app.Name)) - errGrp.Go(func() error { - // If the app has a config, we need to deploy it first. - // It means we need to write app.yml file with the content of the config field - // to the remote source code path of the app. - if app.Config != nil { - appPath, err := filepath.Rel(b.Config.Workspace.FilePath, app.SourceCodePath) - if err != nil { - return fmt.Errorf("failed to get relative path of app source code path: %w", err) - } - - buf, err := configToYaml(app) - if err != nil { - return err - } - - err = f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists) - if err != nil { - return fmt.Errorf("failed to write %s file: %w", path.Join(app.SourceCodePath, "app.yml"), err) - } - } - - wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: &apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: app.SourceCodePath, - }, - }) - - if err != nil { - return err - } - - _, err = wait.Get() - return err - }) - } - - if err := errGrp.Wait(); err != nil { - return diag.FromErr(err) - } - - return nil -} - -func configToYaml(app *resources.App) (*bytes.Buffer, error) { - buf := bytes.NewBuffer(nil) - enc := yaml.NewEncoder(buf) - enc.SetIndent(2) - - err := enc.Encode(app.Config) - defer enc.Close() - - if err != nil { - return nil, fmt.Errorf("failed to encode app config to yaml: %w", err) - } - - return buf, nil -} diff --git a/bundle/deploy/apps/deploy_test.go b/bundle/deploy/apps/deploy_test.go deleted file mode 100644 index dacd0a28..00000000 --- a/bundle/deploy/apps/deploy_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package apps - -import ( - "bytes" - "context" - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/cli/libs/vfs" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/apps" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestAppDeploy(t *testing.T) { - root := t.TempDir() - err := os.MkdirAll(filepath.Join(root, "app1"), 0700) - require.NoError(t, err) - - err = os.MkdirAll(filepath.Join(root, "app2"), 0700) - require.NoError(t, err) - - b := &bundle.Bundle{ - BundleRootPath: root, - SyncRoot: vfs.MustNew(root), - Config: config.Root{ - Workspace: config.Workspace{ - RootPath: "/Workspace/Users/foo@bar.com/", - }, - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "app1": { - App: &apps.App{ - Name: "app1", - }, - SourceCodePath: "./app1", - Config: map[string]interface{}{ - "command": []string{"echo", "hello"}, - "env": []map[string]string{ - {"name": "MY_APP", "value": "my value"}, - }, - }, - }, - "app2": { - App: &apps.App{ - Name: "app2", - }, - SourceCodePath: "./app2", - }, - }, - }, - }, - } - - mwc := mocks.NewMockWorkspaceClient(t) - b.SetWorkpaceClient(mwc.WorkspaceClient) - - wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{ - Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) { - return nil, nil - }, - } - appApi := mwc.GetMockAppsAPI() - appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{ - AppName: "app1", - AppDeployment: &apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: "/Workspace/Users/foo@bar.com/files/app1", - }, - }).Return(wait, nil) - - appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{ - AppName: "app2", - AppDeployment: &apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: "/Workspace/Users/foo@bar.com/files/app2", - }, - }).Return(wait, nil) - - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write(mock.Anything, "app1/app.yml", bytes.NewBufferString(`command: - - echo - - hello -env: - - name: MY_APP - value: my value -`), filer.OverwriteIfExists).Return(nil) - - bundletest.SetLocation(b, "resources.apps.app1", []dyn.Location{{File: "./databricks.yml"}}) - bundletest.SetLocation(b, "resources.apps.app2", []dyn.Location{{File: "./databricks.yml"}}) - - ctx := context.Background() - diags := bundle.Apply(ctx, b, bundle.Seq( - mutator.DefineDefaultWorkspacePaths(), - mutator.TranslatePaths(), - appsDeploy{ - func(b *bundle.Bundle) (filer.Filer, error) { - return mockFiler, nil - }, - })) - require.Empty(t, diags) -} diff --git a/bundle/deploy/filer.go b/bundle/deploy/filer.go index b6acb4c5..c0fd839e 100644 --- a/bundle/deploy/filer.go +++ b/bundle/deploy/filer.go @@ -12,8 +12,3 @@ type FilerFactory func(b *bundle.Bundle) (filer.Filer, error) func StateFiler(b *bundle.Bundle) (filer.Filer, error) { return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.StatePath) } - -// AppFiler returns a filer.Filer that can be used to read/write Databricks apps related files. -func AppFiler(b *bundle.Bundle) (filer.Filer, error) { - return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath) -} diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index 621dc4e4..f4849cab 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -2,6 +2,7 @@ package tfdyn import ( "context" + "fmt" "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/libs/dyn" @@ -42,11 +43,8 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out // Configure permissions for this resource. if permissions := convertPermissionsResource(ctx, vin); permissions != nil { - // TODO: add when permissions are supported in TF - /* - permissions.AppId = fmt.Sprintf("${databricks_app.%s.id}", key) - out.Permissions["app_"+key] = permissions - */ + permissions.AppName = fmt.Sprintf("${databricks_app.%s.name}", key) + out.Permissions["app_"+key] = permissions } return nil diff --git a/bundle/deploy/terraform/tfdyn/convert_app_test.go b/bundle/deploy/terraform/tfdyn/convert_app_test.go index 81613d8f..95b9bdae 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_app_test.go @@ -81,22 +81,19 @@ func TestConvertApp(t *testing.T) { }, }, app) - // TODO: Add when permissions are supported in TF - /* - // Assert equality on the permissions - assert.Equal(t, &schema.ResourcePermissions{ - AppId: "${databricks_app.my_app.id}", - AccessControl: []schema.ResourcePermissionsAccessControl{ - { - PermissionLevel: "CAN_RUN", - UserName: "jack@gmail.com", - }, - { - PermissionLevel: "CAN_MANAGE", - ServicePrincipalName: "sp", - }, + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + AppName: "${databricks_app.my_app.name}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_RUN", + UserName: "jack@gmail.com", }, - }, out.Permissions["app_my_app"]) - */ + { + PermissionLevel: "CAN_MANAGE", + ServicePrincipalName: "sp", + }, + }, + }, out.Permissions["app_my_app"]) } diff --git a/bundle/internal/tf/schema/resource_app.go b/bundle/internal/tf/schema/resource_app.go index 52b6d0e4..bd787f3f 100644 --- a/bundle/internal/tf/schema/resource_app.go +++ b/bundle/internal/tf/schema/resource_app.go @@ -83,20 +83,21 @@ type ResourceAppResource struct { } type ResourceApp struct { - CreateTime string `json:"create_time,omitempty"` - Creator string `json:"creator,omitempty"` - DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` - Description string `json:"description,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name"` - ServicePrincipalId int `json:"service_principal_id,omitempty"` - ServicePrincipalName string `json:"service_principal_name,omitempty"` - UpdateTime string `json:"update_time,omitempty"` - Updater string `json:"updater,omitempty"` - Url string `json:"url,omitempty"` - ActiveDeployment *ResourceAppActiveDeployment `json:"active_deployment,omitempty"` - AppStatus *ResourceAppAppStatus `json:"app_status,omitempty"` - ComputeStatus *ResourceAppComputeStatus `json:"compute_status,omitempty"` - PendingDeployment *ResourceAppPendingDeployment `json:"pending_deployment,omitempty"` - Resource []ResourceAppResource `json:"resource,omitempty"` + CreateTime string `json:"create_time,omitempty"` + Creator string `json:"creator,omitempty"` + DefaultSourceCodePath string `json:"default_source_code_path,omitempty"` + Description string `json:"description,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + ServicePrincipalClientId string `json:"service_principal_client_id,omitempty"` + ServicePrincipalId int `json:"service_principal_id,omitempty"` + ServicePrincipalName string `json:"service_principal_name,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + Updater string `json:"updater,omitempty"` + Url string `json:"url,omitempty"` + ActiveDeployment *ResourceAppActiveDeployment `json:"active_deployment,omitempty"` + AppStatus *ResourceAppAppStatus `json:"app_status,omitempty"` + ComputeStatus *ResourceAppComputeStatus `json:"compute_status,omitempty"` + PendingDeployment *ResourceAppPendingDeployment `json:"pending_deployment,omitempty"` + Resource []ResourceAppResource `json:"resource,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index a3d05e6f..7dfb84b5 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -10,6 +10,7 @@ type ResourcePermissionsAccessControl struct { } type ResourcePermissions struct { + AppName string `json:"app_name,omitempty"` Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_quality_monitor.go b/bundle/internal/tf/schema/resource_quality_monitor.go index 61d18e93..0fc2abd6 100644 --- a/bundle/internal/tf/schema/resource_quality_monitor.go +++ b/bundle/internal/tf/schema/resource_quality_monitor.go @@ -33,8 +33,8 @@ type ResourceQualityMonitorNotificationsOnNewClassificationTagDetected struct { } type ResourceQualityMonitorNotifications struct { - OnFailure []ResourceQualityMonitorNotificationsOnFailure `json:"on_failure,omitempty"` - OnNewClassificationTagDetected []ResourceQualityMonitorNotificationsOnNewClassificationTagDetected `json:"on_new_classification_tag_detected,omitempty"` + OnFailure *ResourceQualityMonitorNotificationsOnFailure `json:"on_failure,omitempty"` + OnNewClassificationTagDetected *ResourceQualityMonitorNotificationsOnNewClassificationTagDetected `json:"on_new_classification_tag_detected,omitempty"` } type ResourceQualityMonitorSchedule struct { @@ -52,25 +52,25 @@ type ResourceQualityMonitorTimeSeries struct { } type ResourceQualityMonitor struct { - AssetsDir string `json:"assets_dir"` - BaselineTableName string `json:"baseline_table_name,omitempty"` - DashboardId string `json:"dashboard_id,omitempty"` - DriftMetricsTableName string `json:"drift_metrics_table_name,omitempty"` - Id string `json:"id,omitempty"` - LatestMonitorFailureMsg string `json:"latest_monitor_failure_msg,omitempty"` - MonitorVersion string `json:"monitor_version,omitempty"` - OutputSchemaName string `json:"output_schema_name"` - ProfileMetricsTableName string `json:"profile_metrics_table_name,omitempty"` - SkipBuiltinDashboard bool `json:"skip_builtin_dashboard,omitempty"` - SlicingExprs []string `json:"slicing_exprs,omitempty"` - Status string `json:"status,omitempty"` - TableName string `json:"table_name"` - WarehouseId string `json:"warehouse_id,omitempty"` - CustomMetrics []ResourceQualityMonitorCustomMetrics `json:"custom_metrics,omitempty"` - DataClassificationConfig []ResourceQualityMonitorDataClassificationConfig `json:"data_classification_config,omitempty"` - InferenceLog []ResourceQualityMonitorInferenceLog `json:"inference_log,omitempty"` - Notifications []ResourceQualityMonitorNotifications `json:"notifications,omitempty"` - Schedule []ResourceQualityMonitorSchedule `json:"schedule,omitempty"` - Snapshot []ResourceQualityMonitorSnapshot `json:"snapshot,omitempty"` - TimeSeries []ResourceQualityMonitorTimeSeries `json:"time_series,omitempty"` + AssetsDir string `json:"assets_dir"` + BaselineTableName string `json:"baseline_table_name,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DriftMetricsTableName string `json:"drift_metrics_table_name,omitempty"` + Id string `json:"id,omitempty"` + LatestMonitorFailureMsg string `json:"latest_monitor_failure_msg,omitempty"` + MonitorVersion string `json:"monitor_version,omitempty"` + OutputSchemaName string `json:"output_schema_name"` + ProfileMetricsTableName string `json:"profile_metrics_table_name,omitempty"` + SkipBuiltinDashboard bool `json:"skip_builtin_dashboard,omitempty"` + SlicingExprs []string `json:"slicing_exprs,omitempty"` + Status string `json:"status,omitempty"` + TableName string `json:"table_name"` + WarehouseId string `json:"warehouse_id,omitempty"` + CustomMetrics []ResourceQualityMonitorCustomMetrics `json:"custom_metrics,omitempty"` + DataClassificationConfig *ResourceQualityMonitorDataClassificationConfig `json:"data_classification_config,omitempty"` + InferenceLog *ResourceQualityMonitorInferenceLog `json:"inference_log,omitempty"` + Notifications *ResourceQualityMonitorNotifications `json:"notifications,omitempty"` + Schedule *ResourceQualityMonitorSchedule `json:"schedule,omitempty"` + Snapshot *ResourceQualityMonitorSnapshot `json:"snapshot,omitempty"` + TimeSeries *ResourceQualityMonitorTimeSeries `json:"time_series,omitempty"` } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 2e16bc47..e623c364 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy" - "github.com/databricks/cli/bundle/deploy/apps" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/metadata" @@ -137,7 +136,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { bundle.Seq( bundle.LogString("Deploying resources..."), terraform.Apply(), - apps.Deploy(), ), bundle.Seq( terraform.StatePush(), diff --git a/bundle/run/app.go b/bundle/run/app.go new file mode 100644 index 00000000..cbf2d013 --- /dev/null +++ b/bundle/run/app.go @@ -0,0 +1,248 @@ +package run + +import ( + "bytes" + "context" + "fmt" + "path" + "path/filepath" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/run/output" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/service/apps" + "github.com/spf13/cobra" + + "gopkg.in/yaml.v3" +) + +func logProgress(ctx context.Context, msg string) { + if msg == "" { + return + } + cmdio.LogString(ctx, fmt.Sprintf("✓ %s", msg)) +} + +type appRunner struct { + key + + bundle *bundle.Bundle + app *resources.App + + filerFactory deploy.FilerFactory +} + +func (a *appRunner) Name() string { + if a.app == nil { + return "" + } + + return a.app.Name +} + +func (a *appRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) { + app := a.app + b := a.bundle + if app == nil { + return nil, fmt.Errorf("app is not defined") + } + + logProgress(ctx, fmt.Sprintf("Getting the status of the app %s", app.Name)) + w := b.WorkspaceClient() + + // Check the status of the app first. + createdApp, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) + if err != nil { + return nil, err + } + + if createdApp.AppStatus != nil { + logProgress(ctx, fmt.Sprintf("App is in %s state", createdApp.AppStatus.State)) + } + + // If the app is not running, start it. + if createdApp.AppStatus == nil || createdApp.AppStatus.State != apps.ApplicationStateRunning { + err := a.start(ctx) + if err != nil { + return nil, err + } + } + + // Deploy the app. + err = a.deploy(ctx) + if err != nil { + return nil, err + } + + // TODO: We should return the app URL here. + cmdio.LogString(ctx, "You can access the app at ") + return nil, nil +} + +func (a *appRunner) start(ctx context.Context) error { + app := a.app + b := a.bundle + w := b.WorkspaceClient() + + logProgress(ctx, fmt.Sprintf("Starting the app %s", app.Name)) + wait, err := w.Apps.Start(ctx, apps.StartAppRequest{Name: app.Name}) + if err != nil { + return err + } + + startedApp, err := wait.OnProgress(func(p *apps.App) { + if p.AppStatus == nil { + return + } + logProgress(ctx, "App is starting...") + }).Get() + + if err != nil { + return err + } + + // If the app has a pending deployment, wait for it to complete. + if startedApp.PendingDeployment != nil { + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, + startedApp.Name, startedApp.PendingDeployment.DeploymentId, + 20*time.Minute, nil) + + if err != nil { + return err + } + } + + // If the app has an active deployment, wait for it to complete as well + if startedApp.ActiveDeployment != nil { + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, + startedApp.Name, startedApp.ActiveDeployment.DeploymentId, + 20*time.Minute, nil) + + if err != nil { + return err + } + } + + logProgress(ctx, "App is started!") + return nil +} + +func (a *appRunner) deploy(ctx context.Context) error { + app := a.app + b := a.bundle + w := b.WorkspaceClient() + + // If the app has a config, we need to deploy it first. + // It means we need to write app.yml file with the content of the config field + // to the remote source code path of the app. + if app.Config != nil { + appPath, err := filepath.Rel(b.Config.Workspace.FilePath, app.SourceCodePath) + if err != nil { + return fmt.Errorf("failed to get relative path of app source code path: %w", err) + } + + buf, err := configToYaml(app) + if err != nil { + return err + } + + // When the app is started, create a new app deployment and wait for it to complete. + f, err := a.filerFactory(b) + if err != nil { + return err + } + + err = f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists) + if err != nil { + return fmt.Errorf("failed to write %s file: %w", path.Join(app.SourceCodePath, "app.yml"), err) + } + } + + wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ + AppName: app.Name, + AppDeployment: &apps.AppDeployment{ + Mode: apps.AppDeploymentModeSnapshot, + SourceCodePath: app.SourceCodePath, + }, + }) + + if err != nil { + return err + } + + _, err = wait.OnProgress(func(ad *apps.AppDeployment) { + if ad.Status == nil { + return + } + logProgress(ctx, ad.Status.Message) + }).Get() + + if err != nil { + return err + } + + return nil +} + +func (a *appRunner) Cancel(ctx context.Context) error { + // We should cancel the app by stopping it. + app := a.app + b := a.bundle + if app == nil { + return fmt.Errorf("app is not defined") + } + + w := b.WorkspaceClient() + + logProgress(ctx, fmt.Sprintf("Stopping app %s", app.Name)) + wait, err := w.Apps.Stop(ctx, apps.StopAppRequest{Name: app.Name}) + if err != nil { + return err + } + + _, err = wait.OnProgress(func(p *apps.App) { + if p.AppStatus == nil { + return + } + logProgress(ctx, p.AppStatus.Message) + }).Get() + + logProgress(ctx, "App is stopped!") + return err +} + +func (a *appRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { + // We should restart the app by just running it again meaning a new app deployment will be done. + return a.Run(ctx, opts) +} + +func (a *appRunner) ParseArgs(args []string, opts *Options) error { + if len(args) == 0 { + return nil + } + + return fmt.Errorf("received %d unexpected positional arguments", len(args)) +} + +func (a *appRunner) CompleteArgs(args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return nil, cobra.ShellCompDirectiveNoFileComp +} + +func configToYaml(app *resources.App) (*bytes.Buffer, error) { + buf := bytes.NewBuffer(nil) + enc := yaml.NewEncoder(buf) + enc.SetIndent(2) + + err := enc.Encode(app.Config) + defer enc.Close() + + if err != nil { + return nil, fmt.Errorf("failed to encode app config to yaml: %w", err) + } + + return buf, nil +} diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go new file mode 100644 index 00000000..731219f1 --- /dev/null +++ b/bundle/run/app_test.go @@ -0,0 +1,208 @@ +package run + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/apps" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type testAppRunner struct { + m *mocks.MockWorkspaceClient + b *bundle.Bundle + mockFiler *mockfiler.MockFiler + ctx context.Context +} + +func (ta *testAppRunner) run(t *testing.T) { + r := appRunner{ + key: "my_app", + bundle: ta.b, + app: ta.b.Config.Resources.Apps["my_app"], + filerFactory: func(b *bundle.Bundle) (filer.Filer, error) { + return ta.mockFiler, nil + }, + } + + _, err := r.Run(ta.ctx, &Options{}) + require.NoError(t, err) +} + +func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWorkspaceClient) { + root := t.TempDir() + err := os.MkdirAll(filepath.Join(root, "my_app"), 0700) + require.NoError(t, err) + + b := &bundle.Bundle{ + BundleRootPath: root, + SyncRoot: vfs.MustNew(root), + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/foo@bar.com/", + }, + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "my_app": { + App: &apps.App{ + Name: "my_app", + }, + SourceCodePath: "./my_app", + Config: map[string]interface{}{ + "command": []string{"echo", "hello"}, + "env": []map[string]string{ + {"name": "MY_APP", "value": "my value"}, + }, + }, + }, + }, + }, + }, + } + + mwc := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(mwc.WorkspaceClient) + bundletest.SetLocation(b, "resources.apps.my_app", []dyn.Location{{File: "./databricks.yml"}}) + + ctx := context.Background() + ctx = cmdio.InContext(ctx, cmdio.NewIO(flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "...")) + ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend)) + + diags := bundle.Apply(ctx, b, bundle.Seq( + mutator.DefineDefaultWorkspacePaths(), + mutator.TranslatePaths(), + )) + require.Empty(t, diags) + + return ctx, b, mwc +} + +func setupTestApp(t *testing.T, initialAppState apps.ApplicationState) *testAppRunner { + ctx, b, mwc := setupBundle(t) + + appApi := mwc.GetMockAppsAPI() + appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{ + Name: "my_app", + }).Return(&apps.App{ + Name: "my_app", + AppStatus: &apps.ApplicationStatus{ + State: initialAppState, + }, + }, nil) + + wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{ + Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) { + return nil, nil + }, + } + appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{ + AppName: "my_app", + AppDeployment: &apps.AppDeployment{ + Mode: apps.AppDeploymentModeSnapshot, + SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app", + }, + }).Return(wait, nil) + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command: + - echo + - hello +env: + - name: MY_APP + value: my value +`), filer.OverwriteIfExists).Return(nil) + + return &testAppRunner{ + m: mwc, + b: b, + mockFiler: mockFiler, + ctx: ctx, + } +} + +func TestAppRunStartedApp(t *testing.T) { + r := setupTestApp(t, apps.ApplicationStateRunning) + r.run(t) +} + +func TestAppRunStoppedApp(t *testing.T) { + r := setupTestApp(t, apps.ApplicationStateCrashed) + + appsApi := r.m.GetMockAppsAPI() + appsApi.EXPECT().Start(mock.Anything, apps.StartAppRequest{ + Name: "my_app", + }).Return(&apps.WaitGetAppActive[apps.App]{ + Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) { + return &apps.App{ + Name: "my_app", + AppStatus: &apps.ApplicationStatus{ + State: apps.ApplicationStateRunning, + }, + ActiveDeployment: &apps.AppDeployment{ + SourceCodePath: "/foo/bar", + DeploymentId: "123", + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateInProgress, + }, + }, + PendingDeployment: &apps.AppDeployment{ + SourceCodePath: "/foo/bar", + DeploymentId: "456", + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateInProgress, + }, + }, + }, nil + }, + }, nil) + + appsApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "123", mock.Anything, mock.Anything).Return(nil, nil) + appsApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "456", mock.Anything, mock.Anything).Return(nil, nil) + r.run(t) +} + +func TestStopApp(t *testing.T) { + ctx, b, mwc := setupBundle(t) + appsApi := mwc.GetMockAppsAPI() + appsApi.EXPECT().Stop(mock.Anything, apps.StopAppRequest{ + Name: "my_app", + }).Return(&apps.WaitGetAppStopped[apps.App]{ + Poll: func(_ time.Duration, _ func(*apps.App)) (*apps.App, error) { + return &apps.App{ + Name: "my_app", + AppStatus: &apps.ApplicationStatus{ + State: apps.ApplicationStateUnavailable, + }, + }, nil + }, + }, nil) + + r := appRunner{ + key: "my_app", + bundle: b, + app: b.Config.Resources.Apps["my_app"], + filerFactory: func(b *bundle.Bundle) (filer.Filer, error) { + return nil, nil + }, + } + + err := r.Cancel(ctx) + require.NoError(t, err) +} diff --git a/bundle/run/runner.go b/bundle/run/runner.go index 4c907d06..e4706ca5 100644 --- a/bundle/run/runner.go +++ b/bundle/run/runner.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" refs "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run/output" + "github.com/databricks/cli/libs/filer" ) type key string @@ -42,7 +43,7 @@ type Runner interface { // IsRunnable returns a filter that only allows runnable resources. func IsRunnable(ref refs.Reference) bool { switch ref.Resource.(type) { - case *resources.Job, *resources.Pipeline: + case *resources.Job, *resources.Pipeline, *resources.App: return true default: return false @@ -56,6 +57,15 @@ func ToRunner(b *bundle.Bundle, ref refs.Reference) (Runner, error) { return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil case *resources.Pipeline: return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil + case *resources.App: + return &appRunner{ + key: key(ref.KeyWithType), + bundle: b, + app: resource, + filerFactory: func(b *bundle.Bundle) (filer.Filer, error) { + return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath) + }, + }, nil default: return nil, fmt.Errorf("unsupported resource type: %T", resource) }