diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index bf29106a..bd43ed0a 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -8,8 +8,9 @@ import ( ) type Job struct { - ID string `json:"id,omitempty" bundle:"readonly"` - Permissions []Permission `json:"permissions,omitempty"` + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` paths.Paths diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index e4a9a8a8..0f53096a 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -7,7 +7,9 @@ import ( ) type MlflowExperiment struct { - Permissions []Permission `json:"permissions,omitempty"` + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` paths.Paths diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 51fb0e08..59893aa4 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -7,7 +7,9 @@ import ( ) type MlflowModel struct { - Permissions []Permission `json:"permissions,omitempty"` + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` paths.Paths diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 88a55ac8..d1d57baf 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -13,7 +13,7 @@ type ModelServingEndpoint struct { // This represents the id (ie serving_endpoint_id) that can be used // as a reference in other resources. This value is returned by terraform. - ID string + ID string `json:"id,omitempty" bundle:"readonly"` // Path to config file where the resource is defined. All bundle resources // include this for interpolation purposes. @@ -22,6 +22,8 @@ type ModelServingEndpoint struct { // This is a resource agnostic implementation of permissions for ACLs. // Implementation could be different based on the resource type. Permissions []Permission `json:"permissions,omitempty"` + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` } func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources/modified_status.go b/bundle/config/resources/modified_status.go new file mode 100644 index 00000000..d135c196 --- /dev/null +++ b/bundle/config/resources/modified_status.go @@ -0,0 +1,15 @@ +package resources + +// ModifiedStatus is an enum of the possible statuses of a resource from the local perspective. +// CREATED - new resources that have been added to the local bundle configuration and don't yet exist remotely. +// DELETED - existing resources that have been removed from the local bundle but still exist remotely. +// UPDATED - existing resources that have been modified. +// An empty status means that the resource is unchanged. +// We use these statuses to build git-status-like UI of the resources in the Databricks VSCode extension. +type ModifiedStatus = string + +const ( + ModifiedStatusCreated ModifiedStatus = "created" + ModifiedStatusUpdated ModifiedStatus = "updated" + ModifiedStatusDeleted ModifiedStatus = "deleted" +) diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 5c741f8a..43450dc4 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -10,8 +10,9 @@ import ( ) type Pipeline struct { - ID string `json:"id,omitempty" bundle:"readonly"` - Permissions []Permission `json:"permissions,omitempty"` + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` paths.Paths diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index 32a451a2..7b4b70d1 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -14,7 +14,7 @@ type RegisteredModel struct { // This represents the id which is the full name of the model // (catalog_name.schema_name.model_name) that can be used // as a reference in other resources. This value is returned by terraform. - ID string + ID string `json:"id,omitempty" bundle:"readonly"` // Path to config file where the resource is defined. All bundle resources // include this for interpolation purposes. @@ -23,6 +23,8 @@ type RegisteredModel struct { // This represents the input args for terraform, and will get converted // to a HCL representation for CRUD *catalog.CreateRegisteredModelRequest + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` } func (s *RegisteredModel) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 8d51a375..6723caee 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -3,6 +3,7 @@ package terraform import ( "encoding/json" "fmt" + "reflect" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" @@ -15,6 +16,15 @@ func conv(from any, to any) { json.Unmarshal(buf, &to) } +func convRemoteToLocal(remote any, local any) resources.ModifiedStatus { + var modifiedStatus resources.ModifiedStatus + if reflect.ValueOf(local).Elem().IsNil() { + modifiedStatus = resources.ModifiedStatusDeleted + } + conv(remote, local) + return modifiedStatus +} + func convPermissions(acl []resources.Permission) *schema.ResourcePermissions { if len(acl) == 0 { return nil @@ -219,59 +229,112 @@ func BundleToTerraform(config *config.Root) *schema.Root { } func TerraformToBundle(state *tfjson.State, config *config.Root) error { - // This is a no-op if the state is empty. - if state.Values == nil || state.Values.RootModule == nil { - return nil + if state.Values != nil && state.Values.RootModule != nil { + for _, resource := range state.Values.RootModule.Resources { + // Limit to resources. + if resource.Mode != tfjson.ManagedResourceMode { + continue + } + + switch resource.Type { + case "databricks_job": + var tmp schema.ResourceJob + conv(resource.AttributeValues, &tmp) + if config.Resources.Jobs == nil { + config.Resources.Jobs = make(map[string]*resources.Job) + } + cur := config.Resources.Jobs[resource.Name] + // TODO: make sure we can unmarshall tf state properly and don't swallow errors + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.Jobs[resource.Name] = cur + case "databricks_pipeline": + var tmp schema.ResourcePipeline + conv(resource.AttributeValues, &tmp) + if config.Resources.Pipelines == nil { + config.Resources.Pipelines = make(map[string]*resources.Pipeline) + } + cur := config.Resources.Pipelines[resource.Name] + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.Pipelines[resource.Name] = cur + case "databricks_mlflow_model": + var tmp schema.ResourceMlflowModel + conv(resource.AttributeValues, &tmp) + if config.Resources.Models == nil { + config.Resources.Models = make(map[string]*resources.MlflowModel) + } + cur := config.Resources.Models[resource.Name] + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.Models[resource.Name] = cur + case "databricks_mlflow_experiment": + var tmp schema.ResourceMlflowExperiment + conv(resource.AttributeValues, &tmp) + if config.Resources.Experiments == nil { + config.Resources.Experiments = make(map[string]*resources.MlflowExperiment) + } + cur := config.Resources.Experiments[resource.Name] + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.Experiments[resource.Name] = cur + case "databricks_model_serving": + var tmp schema.ResourceModelServing + conv(resource.AttributeValues, &tmp) + if config.Resources.ModelServingEndpoints == nil { + config.Resources.ModelServingEndpoints = make(map[string]*resources.ModelServingEndpoint) + } + cur := config.Resources.ModelServingEndpoints[resource.Name] + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.ModelServingEndpoints[resource.Name] = cur + case "databricks_registered_model": + var tmp schema.ResourceRegisteredModel + conv(resource.AttributeValues, &tmp) + if config.Resources.RegisteredModels == nil { + config.Resources.RegisteredModels = make(map[string]*resources.RegisteredModel) + } + cur := config.Resources.RegisteredModels[resource.Name] + modifiedStatus := convRemoteToLocal(tmp, &cur) + cur.ModifiedStatus = modifiedStatus + config.Resources.RegisteredModels[resource.Name] = cur + case "databricks_permissions": + case "databricks_grants": + // Ignore; no need to pull these back into the configuration. + default: + return fmt.Errorf("missing mapping for %s", resource.Type) + } + } } - for _, resource := range state.Values.RootModule.Resources { - // Limit to resources. - if resource.Mode != tfjson.ManagedResourceMode { - continue + for _, src := range config.Resources.Jobs { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated } - - switch resource.Type { - case "databricks_job": - var tmp schema.ResourceJob - conv(resource.AttributeValues, &tmp) - cur := config.Resources.Jobs[resource.Name] - conv(tmp, &cur) - config.Resources.Jobs[resource.Name] = cur - case "databricks_pipeline": - var tmp schema.ResourcePipeline - conv(resource.AttributeValues, &tmp) - cur := config.Resources.Pipelines[resource.Name] - conv(tmp, &cur) - config.Resources.Pipelines[resource.Name] = cur - case "databricks_mlflow_model": - var tmp schema.ResourceMlflowModel - conv(resource.AttributeValues, &tmp) - cur := config.Resources.Models[resource.Name] - conv(tmp, &cur) - config.Resources.Models[resource.Name] = cur - case "databricks_mlflow_experiment": - var tmp schema.ResourceMlflowExperiment - conv(resource.AttributeValues, &tmp) - cur := config.Resources.Experiments[resource.Name] - conv(tmp, &cur) - config.Resources.Experiments[resource.Name] = cur - case "databricks_model_serving": - var tmp schema.ResourceModelServing - conv(resource.AttributeValues, &tmp) - cur := config.Resources.ModelServingEndpoints[resource.Name] - conv(tmp, &cur) - config.Resources.ModelServingEndpoints[resource.Name] = cur - case "databricks_registered_model": - var tmp schema.ResourceRegisteredModel - conv(resource.AttributeValues, &tmp) - cur := config.Resources.RegisteredModels[resource.Name] - conv(tmp, &cur) - config.Resources.RegisteredModels[resource.Name] = cur - case "databricks_permissions": - case "databricks_grants": - // Ignore; no need to pull these back into the configuration. - default: - return fmt.Errorf("missing mapping for %s", resource.Type) + } + for _, src := range config.Resources.Pipelines { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } + for _, src := range config.Resources.Models { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } + for _, src := range config.Resources.Experiments { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } + for _, src := range config.Resources.ModelServingEndpoints { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } + for _, src := range config.Resources.RegisteredModels { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated } } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 00086c76..bb77f287 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -1,6 +1,7 @@ package terraform import ( + "reflect" "testing" "github.com/databricks/cli/bundle/config" @@ -11,11 +12,12 @@ import ( "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" + tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestConvertJob(t *testing.T) { +func TestBundleToTerraformJob(t *testing.T) { var src = resources.Job{ JobSettings: &jobs.JobSettings{ Name: "my job", @@ -62,7 +64,7 @@ func TestConvertJob(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertJobPermissions(t *testing.T) { +func TestBundleToTerraformJobPermissions(t *testing.T) { var src = resources.Job{ Permissions: []resources.Permission{ { @@ -89,7 +91,7 @@ func TestConvertJobPermissions(t *testing.T) { assert.Equal(t, "CAN_VIEW", p.PermissionLevel) } -func TestConvertJobTaskLibraries(t *testing.T) { +func TestBundleToTerraformJobTaskLibraries(t *testing.T) { var src = resources.Job{ JobSettings: &jobs.JobSettings{ Name: "my job", @@ -123,7 +125,7 @@ func TestConvertJobTaskLibraries(t *testing.T) { assert.Equal(t, "mlflow", out.Resource.Job["my_job"].Task[0].Library[0].Pypi.Package) } -func TestConvertPipeline(t *testing.T) { +func TestBundleToTerraformPipeline(t *testing.T) { var src = resources.Pipeline{ PipelineSpec: &pipelines.PipelineSpec{ Name: "my pipeline", @@ -182,7 +184,7 @@ func TestConvertPipeline(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertPipelinePermissions(t *testing.T) { +func TestBundleToTerraformPipelinePermissions(t *testing.T) { var src = resources.Pipeline{ Permissions: []resources.Permission{ { @@ -209,7 +211,7 @@ func TestConvertPipelinePermissions(t *testing.T) { assert.Equal(t, "CAN_VIEW", p.PermissionLevel) } -func TestConvertModel(t *testing.T) { +func TestBundleToTerraformModel(t *testing.T) { var src = resources.MlflowModel{ Model: &ml.Model{ Name: "name", @@ -246,7 +248,7 @@ func TestConvertModel(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertModelPermissions(t *testing.T) { +func TestBundleToTerraformModelPermissions(t *testing.T) { var src = resources.MlflowModel{ Permissions: []resources.Permission{ { @@ -273,7 +275,7 @@ func TestConvertModelPermissions(t *testing.T) { assert.Equal(t, "CAN_READ", p.PermissionLevel) } -func TestConvertExperiment(t *testing.T) { +func TestBundleToTerraformExperiment(t *testing.T) { var src = resources.MlflowExperiment{ Experiment: &ml.Experiment{ Name: "name", @@ -293,7 +295,7 @@ func TestConvertExperiment(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertExperimentPermissions(t *testing.T) { +func TestBundleToTerraformExperimentPermissions(t *testing.T) { var src = resources.MlflowExperiment{ Permissions: []resources.Permission{ { @@ -321,7 +323,7 @@ func TestConvertExperimentPermissions(t *testing.T) { } -func TestConvertModelServing(t *testing.T) { +func TestBundleToTerraformModelServing(t *testing.T) { var src = resources.ModelServingEndpoint{ CreateServingEndpoint: &serving.CreateServingEndpoint{ Name: "name", @@ -366,7 +368,7 @@ func TestConvertModelServing(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertModelServingPermissions(t *testing.T) { +func TestBundleToTerraformModelServingPermissions(t *testing.T) { var src = resources.ModelServingEndpoint{ Permissions: []resources.Permission{ { @@ -394,7 +396,7 @@ func TestConvertModelServingPermissions(t *testing.T) { } -func TestConvertRegisteredModel(t *testing.T) { +func TestBundleToTerraformRegisteredModel(t *testing.T) { var src = resources.RegisteredModel{ CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ Name: "name", @@ -421,7 +423,7 @@ func TestConvertRegisteredModel(t *testing.T) { assert.Nil(t, out.Data) } -func TestConvertRegisteredModelGrants(t *testing.T) { +func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { var src = resources.RegisteredModel{ Grants: []resources.Grant{ { @@ -446,5 +448,370 @@ func TestConvertRegisteredModelGrants(t *testing.T) { p := out.Resource.Grants["registered_model_my_registered_model"].Grant[0] assert.Equal(t, "jane@doe.com", p.Principal) assert.Equal(t, "EXECUTE", p.Privileges[0]) - +} + +func TestTerraformToBundleEmptyLocalResources(t *testing.T) { + var config = config.Root{ + Resources: config.Resources{}, + } + var tfState = tfjson.State{ + Values: &tfjson.StateValues{ + RootModule: &tfjson.StateModule{ + Resources: []*tfjson.StateResource{ + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + }, + }, + }, + } + err := TerraformToBundle(&tfState, &config) + assert.NoError(t, err) + + assert.Equal(t, "1", config.Resources.Jobs["test_job"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Jobs["test_job"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Pipelines["test_pipeline"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Pipelines["test_pipeline"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Models["test_mlflow_model"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Models["test_mlflow_model"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Experiments["test_mlflow_experiment"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Experiments["test_mlflow_experiment"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.ModelServingEndpoints["test_model_serving"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ModelServingEndpoints["test_model_serving"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.RegisteredModels["test_registered_model"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus) + + AssertFullResourceCoverage(t, &config) +} + +func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { + var config = config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": { + JobSettings: &jobs.JobSettings{ + Name: "test_job", + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "test_pipeline": { + PipelineSpec: &pipelines.PipelineSpec{ + Name: "test_pipeline", + }, + }, + }, + Models: map[string]*resources.MlflowModel{ + "test_mlflow_model": { + Model: &ml.Model{ + Name: "test_mlflow_model", + }, + }, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "test_mlflow_experiment": { + Experiment: &ml.Experiment{ + Name: "test_mlflow_experiment", + }, + }, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "test_model_serving": { + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "test_model_serving", + }, + }, + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "test_registered_model": { + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "test_registered_model", + }, + }, + }, + }, + } + var tfState = tfjson.State{ + Values: nil, + } + err := TerraformToBundle(&tfState, &config) + assert.NoError(t, err) + + assert.Equal(t, "", config.Resources.Jobs["test_job"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Jobs["test_job"].ModifiedStatus) + + assert.Equal(t, "", config.Resources.Pipelines["test_pipeline"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Pipelines["test_pipeline"].ModifiedStatus) + + assert.Equal(t, "", config.Resources.Models["test_mlflow_model"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Models["test_mlflow_model"].ModifiedStatus) + + assert.Equal(t, "", config.Resources.Experiments["test_mlflow_experiment"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Experiments["test_mlflow_experiment"].ModifiedStatus) + + assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServingEndpoints["test_model_serving"].ModifiedStatus) + + assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus) + + AssertFullResourceCoverage(t, &config) +} + +func TestTerraformToBundleModifiedResources(t *testing.T) { + var config = config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": { + JobSettings: &jobs.JobSettings{ + Name: "test_job", + }, + }, + "test_job_new": { + JobSettings: &jobs.JobSettings{ + Name: "test_job_new", + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "test_pipeline": { + PipelineSpec: &pipelines.PipelineSpec{ + Name: "test_pipeline", + }, + }, + "test_pipeline_new": { + PipelineSpec: &pipelines.PipelineSpec{ + Name: "test_pipeline_new", + }, + }, + }, + Models: map[string]*resources.MlflowModel{ + "test_mlflow_model": { + Model: &ml.Model{ + Name: "test_mlflow_model", + }, + }, + "test_mlflow_model_new": { + Model: &ml.Model{ + Name: "test_mlflow_model_new", + }, + }, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "test_mlflow_experiment": { + Experiment: &ml.Experiment{ + Name: "test_mlflow_experiment", + }, + }, + "test_mlflow_experiment_new": { + Experiment: &ml.Experiment{ + Name: "test_mlflow_experiment_new", + }, + }, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "test_model_serving": { + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "test_model_serving", + }, + }, + "test_model_serving_new": { + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "test_model_serving_new", + }, + }, + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "test_registered_model": { + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "test_registered_model", + }, + }, + "test_registered_model_new": { + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "test_registered_model_new", + }, + }, + }, + }, + } + var tfState = tfjson.State{ + Values: &tfjson.StateValues{ + RootModule: &tfjson.StateModule{ + Resources: []*tfjson.StateResource{ + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model", + AttributeValues: map[string]interface{}{"id": "1"}, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model_old", + AttributeValues: map[string]interface{}{"id": "2"}, + }, + }, + }, + }, + } + err := TerraformToBundle(&tfState, &config) + assert.NoError(t, err) + + assert.Equal(t, "1", config.Resources.Jobs["test_job"].ID) + assert.Equal(t, "", config.Resources.Jobs["test_job"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Jobs["test_job_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Jobs["test_job_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Jobs["test_job_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Jobs["test_job_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Pipelines["test_pipeline"].ID) + assert.Equal(t, "", config.Resources.Pipelines["test_pipeline"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Pipelines["test_pipeline_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Pipelines["test_pipeline_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Pipelines["test_pipeline_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Pipelines["test_pipeline_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Models["test_mlflow_model"].ID) + assert.Equal(t, "", config.Resources.Models["test_mlflow_model"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Models["test_mlflow_model_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Models["test_mlflow_model_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Models["test_mlflow_model_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Models["test_mlflow_model_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.RegisteredModels["test_registered_model"].ID) + assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.RegisteredModels["test_registered_model_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.RegisteredModels["test_registered_model_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.RegisteredModels["test_registered_model_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Experiments["test_mlflow_experiment"].ID) + assert.Equal(t, "", config.Resources.Experiments["test_mlflow_experiment"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Experiments["test_mlflow_experiment_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Experiments["test_mlflow_experiment_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Experiments["test_mlflow_experiment_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Experiments["test_mlflow_experiment_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.ModelServingEndpoints["test_model_serving"].ID) + assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.ModelServingEndpoints["test_model_serving_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ModelServingEndpoints["test_model_serving_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServingEndpoints["test_model_serving_new"].ModifiedStatus) + + AssertFullResourceCoverage(t, &config) +} + +func AssertFullResourceCoverage(t *testing.T, config *config.Root) { + resources := reflect.ValueOf(config.Resources) + for i := 0; i < resources.NumField(); i++ { + field := resources.Field(i) + if field.Kind() == reflect.Map { + assert.True( + t, + !field.IsNil() && field.Len() > 0, + "TerraformToBundle should support '%s' (please add it to convert.go and extend the test suite)", + resources.Type().Field(i).Name, + ) + } + } } diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 3aa6945b..a82311d8 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -22,6 +22,7 @@ func New() *cobra.Command { cmd.AddCommand(newTestCommand()) cmd.AddCommand(newValidateCommand()) cmd.AddCommand(newInitCommand()) + cmd.AddCommand(newSummaryCommand()) cmd.AddCommand(newGenerateCommand()) return cmd } diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go new file mode 100644 index 00000000..efa3c679 --- /dev/null +++ b/cmd/bundle/summary.go @@ -0,0 +1,76 @@ +package bundle + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" + "github.com/spf13/cobra" +) + +func newSummaryCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "summary", + Short: "Describe the bundle resources and their deployment states", + + PreRunE: ConfigureBundleWithVariables, + + // This command is currently intended for the Databricks VSCode extension only + Hidden: true, + } + + var forcePull bool + cmd.Flags().BoolVar(&forcePull, "force-pull", false, "Skip local cache and load the state from the remote workspace") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + b := bundle.Get(cmd.Context()) + + err := bundle.Apply(cmd.Context(), b, phases.Initialize()) + if err != nil { + return err + } + + cacheDir, err := terraform.Dir(cmd.Context(), b) + if err != nil { + return err + } + _, err = os.Stat(filepath.Join(cacheDir, terraform.TerraformStateFileName)) + noCache := errors.Is(err, os.ErrNotExist) + + if forcePull || noCache { + err = bundle.Apply(cmd.Context(), b, terraform.StatePull()) + if err != nil { + return err + } + } + + err = bundle.Apply(cmd.Context(), b, terraform.Load()) + if err != nil { + return err + } + + switch root.OutputType(cmd) { + case flags.OutputText: + return fmt.Errorf("%w, only json output is supported", errors.ErrUnsupported) + case flags.OutputJSON: + buf, err := json.MarshalIndent(b.Config, "", " ") + if err != nil { + return err + } + cmd.OutOrStdout().Write(buf) + default: + return fmt.Errorf("unknown output type %s", root.OutputType(cmd)) + } + + return nil + } + + return cmd +}