steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't fail while parsing outdated terraform state (#1404) 

`terraform show -json` (`terraform.Show()`) fails if the state file
contains resources with fields that non longer conform to the provider
schemas.

This can happen when you deploy a bundle with one version of the CLI,
then updated the CLI to a version that uses different databricks
terraform provider, and try to run `bundle run` or `bundle summary`.
Those commands don't recreate local terraform state (only `terraform
apply` or `plan` do) and terraform itself fails while parsing it.
[Terraform
docs](https://developer.hashicorp.com/terraform/language/state#format)
point out that it's best to use `terraform show` after successful
`apply` or `plan`.

Here we parse the state ourselves. The state file format is internal to
terraform, but it's more stable than our resource schemas. We only parse
a subset of fields from the state, and only update ID and ModifiedStatus
of bundle resources in the `terraform.Load` mutator.
This commit is contained in:
Ilia Babanov 2024-05-01 10:22:35 +02:00 committed by GitHub
parent 781688c9cb
commit 153141d3ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 423 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
bundle/deploy/check_running_resources.go
View File

@ -6,12 +6,11 @@ import (
"strconv"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"golang.org/x/sync/errgroup"
)
@ -35,27 +34,11 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia
if !b.Config.Bundle.Deployment.FailOnActiveRuns {
return nil
}
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
err := tf.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return diag.Errorf("terraform init: %v", err)
}
state, err := b.Terraform.Show(ctx)
w := b.WorkspaceClient()
err := checkAnyResourceRunning(ctx, w, &b.Config.Resources)
if err != nil {
return diag.FromErr(err)
}
err = checkAnyResourceRunning(ctx, b.WorkspaceClient(), state)
if err != nil {
return diag.Errorf("deployment aborted, err: %v", err)
}
return nil
}
@ -63,54 +46,43 @@ func CheckRunningResource() *checkRunningResources {
return &checkRunningResources{}
}
func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *tfjson.State) error {
if state.Values == nil || state.Values.RootModule == nil {
return nil
}
func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, resources *config.Resources) error {
errs, errCtx := errgroup.WithContext(ctx)
for _, resource := range state.Values.RootModule.Resources {
// Limit to resources.
if resource.Mode != tfjson.ManagedResourceMode {
for _, job := range resources.Jobs {
id := job.ID
if id == "" {
continue
}
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
return nil
})
}
value, ok := resource.AttributeValues["id"]
if !ok {
for _, pipeline := range resources.Pipelines {
id := pipeline.ID
if id == "" {
continue
}
id, ok := value.(string)
if !ok {
continue
}
switch resource.Type {
case "databricks_job":
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
return nil
})
case "databricks_pipeline":
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
return nil
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
}
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
}
return errs.Wait()

64
bundle/deploy/check_running_resources_test.go
View File

@ -5,36 +5,26 @@ import (
"errors"
"testing"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
mock := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{}
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &config.Resources{})
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithJob(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Jobs: map[string]*resources.Job{
"job1": {ID: "123"},
},
}
@ -46,7 +36,7 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
{RunId: 1234},
}, nil).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "job 123 is running")
jobsApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{
@ -54,25 +44,15 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
ActiveOnly: true,
}).Return([]jobs.BaseRun{}, nil).Once()
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {ID: "123"},
},
}
@ -84,7 +64,7 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
State: pipelines.PipelineStateRunning,
}, nil).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "pipeline 123 is running")
pipelineApi.EXPECT().Get(mock.Anything, pipelines.GetPipelineRequest{
@ -93,25 +73,15 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
PipelineId: "123",
State: pipelines.PipelineStateIdle,
}, nil).Once()
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {ID: "123"},
},
}
@ -120,6 +90,6 @@ func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
PipelineId: "123",
}).Return(nil, errors.New("API failure")).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}

88
bundle/deploy/terraform/convert.go
View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
@ -19,15 +18,6 @@ func conv(from any, to any) {
json.Unmarshal(buf, &to)
}
func convRemoteToLocal(remote any, local any) resources.ModifiedStatus {
var modifiedStatus resources.ModifiedStatus
if reflect.ValueOf(local).Elem().IsNil() {
modifiedStatus = resources.ModifiedStatusDeleted
}
conv(remote, local)
return modifiedStatus
}
func convPermissions(acl []resources.Permission) *schema.ResourcePermissions {
if len(acl) == 0 {
return nil
@ -248,7 +238,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
tfroot.Provider = schema.NewProviders()
// Convert each resource in the bundle to the equivalent Terraform representation.
resources, err := dyn.Get(root, "resources")
dynResources, err := dyn.Get(root, "resources")
if err != nil {
// If the resources key is missing, return an empty root.
if dyn.IsNoSuchKeyError(err) {
@ -260,11 +250,20 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
tfroot.Resource = schema.NewResources()
numResources := 0
_, err = dyn.Walk(resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
_, err = dyn.Walk(dynResources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
if len(p) < 2 {
return v, nil
}
// Skip resources that have been deleted locally.
modifiedStatus, err := dyn.Get(v, "modified_status")
if err == nil {
modifiedStatusStr, ok := modifiedStatus.AsString()
if ok && modifiedStatusStr == resources.ModifiedStatusDeleted {
return v, dyn.ErrSkip
}
}
typ := p[0].Key()
key := p[1].Key()
@ -275,7 +274,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
}
// Convert resource to Terraform representation.
err := c.Convert(ctx, key, v, tfroot.Resource)
err = c.Convert(ctx, key, v, tfroot.Resource)
if err != nil {
return dyn.InvalidValue, err
}
@ -299,75 +298,72 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
return tfroot, nil
}
func TerraformToBundle(state *tfjson.State, config *config.Root) error {
if state.Values != nil && state.Values.RootModule != nil {
for _, resource := range state.Values.RootModule.Resources {
// Limit to resources.
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
func TerraformToBundle(state *resourcesState, config *config.Root) error {
for _, resource := range state.Resources {
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
for _, instance := range resource.Instances {
switch resource.Type {
case "databricks_job":
var tmp schema.ResourceJob
conv(resource.AttributeValues, &tmp)
if config.Resources.Jobs == nil {
config.Resources.Jobs = make(map[string]*resources.Job)
}
cur := config.Resources.Jobs[resource.Name]
// TODO: make sure we can unmarshall tf state properly and don't swallow errors
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.Job{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Jobs[resource.Name] = cur
case "databricks_pipeline":
var tmp schema.ResourcePipeline
conv(resource.AttributeValues, &tmp)
if config.Resources.Pipelines == nil {
config.Resources.Pipelines = make(map[string]*resources.Pipeline)
}
cur := config.Resources.Pipelines[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.Pipeline{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Pipelines[resource.Name] = cur
case "databricks_mlflow_model":
var tmp schema.ResourceMlflowModel
conv(resource.AttributeValues, &tmp)
if config.Resources.Models == nil {
config.Resources.Models = make(map[string]*resources.MlflowModel)
}
cur := config.Resources.Models[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.MlflowModel{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Models[resource.Name] = cur
case "databricks_mlflow_experiment":
var tmp schema.ResourceMlflowExperiment
conv(resource.AttributeValues, &tmp)
if config.Resources.Experiments == nil {
config.Resources.Experiments = make(map[string]*resources.MlflowExperiment)
}
cur := config.Resources.Experiments[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.MlflowExperiment{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Experiments[resource.Name] = cur
case "databricks_model_serving":
var tmp schema.ResourceModelServing
conv(resource.AttributeValues, &tmp)
if config.Resources.ModelServingEndpoints == nil {
config.Resources.ModelServingEndpoints = make(map[string]*resources.ModelServingEndpoint)
}
cur := config.Resources.ModelServingEndpoints[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.ModelServingEndpoint{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.ModelServingEndpoints[resource.Name] = cur
case "databricks_registered_model":
var tmp schema.ResourceRegisteredModel
conv(resource.AttributeValues, &tmp)
if config.Resources.RegisteredModels == nil {
config.Resources.RegisteredModels = make(map[string]*resources.RegisteredModel)
}
cur := config.Resources.RegisteredModels[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.RegisteredModel{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.RegisteredModels[resource.Name] = cur
case "databricks_permissions":
case "databricks_grants":

293
bundle/deploy/terraform/convert_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/serving"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -548,50 +547,86 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) {
bundleToTerraformEquivalenceTest(t, &config)
}
func TestBundleToTerraformDeletedResources(t *testing.T) {
var job1 = resources.Job{
JobSettings: &jobs.JobSettings{},
}
var job2 = resources.Job{
ModifiedStatus: resources.ModifiedStatusDeleted,
JobSettings: &jobs.JobSettings{},
}
var config = config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"my_job1": &job1,
"my_job2": &job2,
},
},
}
vin, err := convert.FromTyped(config, dyn.NilValue)
require.NoError(t, err)
out, err := BundleToTerraformWithDynValue(context.Background(), vin)
require.NoError(t, err)
_, ok := out.Resource.Job["my_job1"]
assert.True(t, ok)
_, ok = out.Resource.Job["my_job2"]
assert.False(t, ok)
}
func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
var config = config.Root{
Resources: config.Resources{},
}
var tfState = tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
var tfState = resourcesState{
Resources: []stateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
},
@ -667,8 +702,8 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
},
},
}
var tfState = tfjson.State{
Values: nil,
var tfState = resourcesState{
Resources: nil,
}
err := TerraformToBundle(&tfState, &config)
assert.NoError(t, err)
@ -771,82 +806,102 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
},
},
}
var tfState = tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
var tfState = resourcesState{
Resources: []stateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
},

16
bundle/deploy/terraform/load.go
View File

@ -8,7 +8,6 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
)
type loadMode int
@ -34,7 +33,7 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
return diag.Errorf("terraform init: %v", err)
}
state, err := b.Terraform.Show(ctx)
state, err := ParseResourcesState(ctx, b)
if err != nil {
return diag.FromErr(err)
}
@ -53,16 +52,13 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
return nil
}
func (l *load) validateState(state *tfjson.State) error {
if state.Values == nil {
if slices.Contains(l.modes, ErrorOnEmptyState) {
return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
}
return nil
func (l *load) validateState(state *resourcesState) error {
if state.Version != SupportedStateVersion {
return fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version)
}
if state.Values.RootModule == nil {
return fmt.Errorf("malformed terraform state: RootModule not set")
if len(state.Resources) == 0 && slices.Contains(l.modes, ErrorOnEmptyState) {
return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
}
return nil

55
bundle/deploy/terraform/util.go
View File

@ -1,14 +1,46 @@
package terraform
import (
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"github.com/databricks/cli/bundle"
tfjson "github.com/hashicorp/terraform-json"
)
type state struct {
// Partial representation of the Terraform state file format.
// We are only interested global version and serial numbers,
// plus resource types, names, modes, and ids.
type resourcesState struct {
Version int `json:"version"`
Resources []stateResource `json:"resources"`
}
const SupportedStateVersion = 4
type serialState struct {
Serial int `json:"serial"`
}
type stateResource struct {
Type string `json:"type"`
Name string `json:"name"`
Mode tfjson.ResourceMode `json:"mode"`
Instances []stateResourceInstance `json:"instances"`
}
type stateResourceInstance struct {
Attributes stateInstanceAttributes `json:"attributes"`
}
type stateInstanceAttributes struct {
ID string `json:"id"`
}
func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
localState, err := loadState(local)
if err != nil {
@ -23,12 +55,12 @@ func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
return localState.Serial < remoteState.Serial
}
func loadState(input io.Reader) (*state, error) {
func loadState(input io.Reader) (*serialState, error) {
content, err := io.ReadAll(input)
if err != nil {
return nil, err
}
var s state
var s serialState
err = json.Unmarshal(content, &s)
if err != nil {
return nil, err
@ -36,3 +68,20 @@ func loadState(input io.Reader) (*state, error) {
return &s, nil
}
func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) {
cacheDir, err := Dir(ctx, b)
if err != nil {
return nil, err
}
rawState, err := os.ReadFile(filepath.Join(cacheDir, TerraformStateFileName))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return &resourcesState{Version: SupportedStateVersion}, nil
}
return nil, err
}
var state resourcesState
err = json.Unmarshal(rawState, &state)
return &state, err
}

99
bundle/deploy/terraform/util_test.go
View File

@ -1,11 +1,16 @@
package terraform
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"testing/iotest"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/stretchr/testify/assert"
)
@ -38,3 +43,97 @@ func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) {
remote := iotest.ErrReader(fmt.Errorf("Random error"))
assert.False(t, IsLocalStateStale(local, remote))
}
func TestParseResourcesStateWithNoFile(t *testing.T) {
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},
},
},
}
state, err := ParseResourcesState(context.Background(), b)
assert.NoError(t, err)
assert.Equal(t, &resourcesState{Version: SupportedStateVersion}, state)
}
func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
ctx := context.Background()
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},
},
},
}
cacheDir, err := Dir(ctx, b)
assert.NoError(t, err)
data := []byte(`{
"version": 4,
"unknown_field": "hello",
"resources": [
{
"mode": "managed",
"type": "databricks_pipeline",
"name": "test_pipeline",
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"allow_duplicate_names": false,
"catalog": null,
"channel": "CURRENT",
"cluster": [],
"random_field": "random_value",
"configuration": {
"bundle.sourcePath": "/Workspace//Users/user/.bundle/test/dev/files/src"
},
"continuous": false,
"development": true,
"edition": "ADVANCED",
"filters": [],
"id": "123",
"library": [],
"name": "test_pipeline",
"notification": [],
"photon": false,
"serverless": false,
"storage": "dbfs:/123456",
"target": "test_dev",
"timeouts": null,
"url": "https://test.com"
},
"sensitive_attributes": []
}
]
}
]
}`)
err = os.WriteFile(filepath.Join(cacheDir, TerraformStateFileName), data, os.ModePerm)
assert.NoError(t, err)
state, err := ParseResourcesState(ctx, b)
assert.NoError(t, err)
expected := &resourcesState{
Version: 4,
Resources: []stateResource{
{
Mode: "managed",
Type: "databricks_pipeline",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "123"}},
},
},
},
}
assert.Equal(t, expected, state)
}

1
bundle/phases/deploy.go
View File

@ -36,6 +36,7 @@ func Deploy() bundle.Mutator {
permissions.ApplyWorkspaceRootPermissions(),
terraform.Interpolate(),
terraform.Write(),
terraform.Load(),
deploy.CheckRunningResource(),
bundle.Defer(
terraform.Apply(),