Add an error if state files grow bigger than the export limit (#1795)

## Changes
Currently API limits on exporting files from workspaces are set at 10
MBs while uploading to is 500 MBs. We want to prevent users running into
deadlock when they won't be able to pull state file anymore so we
prevent from uploading large state files (over 10 MBs) to Databricks
workspace.

bundle/deploy/state_push.go

@@ -10,6 +10,8 @@ import (
 	"github.com/databricks/cli/libs/log"
 )
 
+const MaxStateFileSize = 10 * 1024 * 1024 // 10MB
+
 type statePush struct {
 	filerFactory FilerFactory
 }
@@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	}
 	defer local.Close()
 
+	if !b.Config.Bundle.Force {
+		state, err := local.Stat()
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+		if state.Size() > MaxStateFileSize {
+			return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize)
+		}
+	}
+
 	log.Infof(ctx, "Writing local deployment state file to remote state directory")
 	err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists)
 	if err != nil {

bundle/deploy/terraform/state_push.go

@@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	}
 	defer local.Close()
 
+	if !b.Config.Bundle.Force {
+		state, err := local.Stat()
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+		if state.Size() > deploy.MaxStateFileSize {
+			return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize)
+		}
+	}
+
 	// Upload state file from local cache directory to filer.
 	cmdio.LogString(ctx, "Updating deployment state...")
 	log.Infof(ctx, "Writing local state file to remote state directory")

bundle/deploy/terraform/state_push_test.go

@@ -3,6 +3,7 @@ package terraform
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"testing"
 
@@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) {
 	diags := bundle.Apply(ctx, b, m)
 	assert.NoError(t, diags.Error())
 }
+
+func TestStatePushLargeState(t *testing.T) {
+	mock := mockfiler.NewMockFiler(t)
+	m := &statePush{
+		identityFiler(mock),
+	}
+
+	ctx := context.Background()
+	b := statePushTestBundle(t)
+
+	largeState := map[string]any{}
+	for i := 0; i < 1000000; i++ {
+		largeState[fmt.Sprintf("field_%d", i)] = i
+	}
+
+	// Write a stale local state file.
+	writeLocalState(t, ctx, b, largeState)
+	diags := bundle.Apply(ctx, b, m)
+	assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag")
+
+	// Force the write.
+	b = statePushTestBundle(t)
+	b.Config.Bundle.Force = true
+	diags = bundle.Apply(ctx, b, m)
+	assert.NoError(t, diags.Error())
+}