Add an error if state files grow bigger than the export limit (#1795)

## Changes
Currently API limits on exporting files from workspaces are set at 10
MBs while uploading to is 500 MBs. We want to prevent users running into
deadlock when they won't be able to pull state file anymore so we
prevent from uploading large state files (over 10 MBs) to Databricks
workspace.
This commit is contained in:
Andrew Nester 2024-10-02 15:53:24 +02:00 committed by GitHub
parent c28d64f2dc
commit 044a00c7f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 51 additions and 0 deletions

View File

@ -10,6 +10,8 @@ import (
"github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/log"
) )
const MaxStateFileSize = 10 * 1024 * 1024 // 10MB
type statePush struct { type statePush struct {
filerFactory FilerFactory filerFactory FilerFactory
} }
@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
} }
defer local.Close() defer local.Close()
if !b.Config.Bundle.Force {
state, err := local.Stat()
if err != nil {
return diag.FromErr(err)
}
if state.Size() > MaxStateFileSize {
return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize)
}
}
log.Infof(ctx, "Writing local deployment state file to remote state directory") log.Infof(ctx, "Writing local deployment state file to remote state directory")
err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists)
if err != nil { if err != nil {

View File

@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
} }
defer local.Close() defer local.Close()
if !b.Config.Bundle.Force {
state, err := local.Stat()
if err != nil {
return diag.FromErr(err)
}
if state.Size() > deploy.MaxStateFileSize {
return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize)
}
}
// Upload state file from local cache directory to filer. // Upload state file from local cache directory to filer.
cmdio.LogString(ctx, "Updating deployment state...") cmdio.LogString(ctx, "Updating deployment state...")
log.Infof(ctx, "Writing local state file to remote state directory") log.Infof(ctx, "Writing local state file to remote state directory")

View File

@ -3,6 +3,7 @@ package terraform
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"io" "io"
"testing" "testing"
@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) {
diags := bundle.Apply(ctx, b, m) diags := bundle.Apply(ctx, b, m)
assert.NoError(t, diags.Error()) assert.NoError(t, diags.Error())
} }
func TestStatePushLargeState(t *testing.T) {
mock := mockfiler.NewMockFiler(t)
m := &statePush{
identityFiler(mock),
}
ctx := context.Background()
b := statePushTestBundle(t)
largeState := map[string]any{}
for i := 0; i < 1000000; i++ {
largeState[fmt.Sprintf("field_%d", i)] = i
}
// Write a stale local state file.
writeLocalState(t, ctx, b, largeState)
diags := bundle.Apply(ctx, b, m)
assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag")
// Force the write.
b = statePushTestBundle(t)
b.Config.Bundle.Force = true
diags = bundle.Apply(ctx, b, m)
assert.NoError(t, diags.Error())
}