Retry app deployment if there is an active deployment in progress (#2153)

## Changes
If before running an app, the app was stopped with an active deployment,
then Apps backend start it and does the auto-deploy of the last active
deployment. In such cases StartApp API won't return any active or
pending deployments in its response but doing the deploy immediately
after the start might result in the error `Cannot deploy app *** as
there is an active deployment in progress`.

From DABs side, we have to do a new deployment on every `bundle run`
(command which start the app and does deployment) because local files in
bundle might have been changed and users expect to have the app running
with new code.

Thus this PR works around the error by catching “deployment in progress”
error, getting any active / pending deployments, waits for them to
finish and start the new deployment again. If 2nd attempts fails, the
whole command fails.

## Tests
Added unit test.

bundle/run/app.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/run/output"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/spf13/cobra"
 )
@@ -111,11 +112,21 @@ func (a *appRunner) start(ctx context.Context) error {
 	// active and pending deployments fields (if any). If there are active or pending deployments,
 	// we need to wait for them to complete before we can do the new deployment.
 	// Otherwise, the new deployment will fail.
-	// Thus, we first wait for the active deployment to complete.
+	err = waitForDeploymentToComplete(ctx, w, startedApp)
-	if startedApp.ActiveDeployment != nil &&
+	if err != nil {
-		startedApp.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
+		return err
+	}
+
+	logProgress(ctx, "App is started!")
+	return nil
+}
+
+func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error {
+	// We first wait for the active deployment to complete.
+	if app.ActiveDeployment != nil &&
+		app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress {
 		logProgress(ctx, "Waiting for the active deployment to complete...")
-		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
+		_, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil)
 		if err != nil {
 			return err
 		}
@@ -123,17 +134,16 @@ func (a *appRunner) start(ctx context.Context) error {
 	}
 
 	// Then, we wait for the pending deployment to complete.
-	if startedApp.PendingDeployment != nil &&
+	if app.PendingDeployment != nil &&
-		startedApp.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
+		app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress {
 		logProgress(ctx, "Waiting for the pending deployment to complete...")
-		_, err = w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, startedApp.PendingDeployment.DeploymentId, 20*time.Minute, nil)
+		_, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil)
 		if err != nil {
 			return err
 		}
 		logProgress(ctx, "Pending deployment is completed!")
 	}
 
-	logProgress(ctx, "App is started!")
 	return nil
 }
 
@@ -142,16 +152,38 @@ func (a *appRunner) deploy(ctx context.Context) error {
 	b := a.bundle
 	w := b.WorkspaceClient()
 
+	sourceCodePath := app.SourceCodePath
 	wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
 		AppName: app.Name,
 		AppDeployment: &apps.AppDeployment{
 			Mode:           apps.AppDeploymentModeSnapshot,
-			SourceCodePath: app.SourceCodePath,
+			SourceCodePath: sourceCodePath,
 		},
 	})
 	// If deploy returns an error, then there's an active deployment in progress, wait for it to complete.
+	// For this we first need to get an app and its acrive and pending deployments and then wait for them.
 	if err != nil {
-		return err
+		app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name})
+		if err != nil {
+			return fmt.Errorf("failed to get app %s: %w", app.Name, err)
+		}
+
+		err = waitForDeploymentToComplete(ctx, w, app)
+		if err != nil {
+			return err
+		}
+
+		// Now we can try to deploy the app again
+		wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{
+			AppName: app.Name,
+			AppDeployment: &apps.AppDeployment{
+				Mode:           apps.AppDeploymentModeSnapshot,
+				SourceCodePath: sourceCodePath,
+			},
+		})
+		if err != nil {
+			return err
+		}
 	}
 
 	_, err = wait.OnProgress(func(ad *apps.AppDeployment) {

bundle/run/app_test.go

@@ -3,6 +3,7 @@ package run
 import (
 	"bytes"
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"testing"
@@ -189,6 +190,69 @@ func TestAppRunWithAnActiveDeploymentInProgress(t *testing.T) {
 	r.run(t)
 }
 
+func TestAppDeployWithDeploymentInProgress(t *testing.T) {
+	ctx, b, mwc := setupBundle(t)
+
+	appApi := mwc.GetMockAppsAPI()
+	appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
+		Name: "my_app",
+	}).Return(&apps.App{
+		Name: "my_app",
+		AppStatus: &apps.ApplicationStatus{
+			State: apps.ApplicationStateRunning,
+		},
+		ComputeStatus: &apps.ComputeStatus{
+			State: apps.ComputeStateActive,
+		},
+	}, nil).Once()
+
+	wait := &apps.WaitGetDeploymentAppSucceeded[apps.AppDeployment]{
+		Poll: func(_ time.Duration, _ func(*apps.AppDeployment)) (*apps.AppDeployment, error) {
+			return nil, nil
+		},
+	}
+
+	// First deployment fails
+	appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
+		AppName: "my_app",
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
+		},
+	}).Return(nil, errors.New("deployment in progress")).Once()
+
+	// After first deployment fails, we should get the app and wait for the deployment to complete
+	appApi.EXPECT().Get(mock.Anything, apps.GetAppRequest{
+		Name: "my_app",
+	}).Return(&apps.App{
+		Name: "my_app",
+		ActiveDeployment: &apps.AppDeployment{
+			DeploymentId: "active_deployment_id",
+			Status: &apps.AppDeploymentStatus{
+				State: apps.AppDeploymentStateInProgress,
+			},
+		},
+	}, nil).Once()
+
+	appApi.EXPECT().WaitGetDeploymentAppSucceeded(mock.Anything, "my_app", "active_deployment_id", mock.Anything, mock.Anything).Return(nil, nil)
+
+	// Second one should succeeed
+	appApi.EXPECT().Deploy(mock.Anything, apps.CreateAppDeploymentRequest{
+		AppName: "my_app",
+		AppDeployment: &apps.AppDeployment{
+			Mode:           apps.AppDeploymentModeSnapshot,
+			SourceCodePath: "/Workspace/Users/foo@bar.com/files/my_app",
+		},
+	}).Return(wait, nil).Once()
+
+	r := &testAppRunner{
+		m:   mwc,
+		b:   b,
+		ctx: ctx,
+	}
+	r.run(t)
+}
+
 func TestStopApp(t *testing.T) {
 	ctx, b, mwc := setupBundle(t)
 	appsApi := mwc.GetMockAppsAPI()