From f3db42e622867843d86c0b2af7f80f98ab62ef8d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 13 Nov 2023 12:29:40 +0100 Subject: [PATCH] Added support for top-level permissions (#928) ## Changes Now it's possible to define top level `permissions` section in bundle configuration and permissions defined there will be applied to all resources defined in the bundle. Supported top-level permission levels: CAN_MANAGE, CAN_VIEW, CAN_RUN. Permissions are applied to: Jobs, DLT Pipelines, ML Models, ML Experiments and Model Service Endpoints ``` bundle: name: permissions workspace: host: *** permissions: - level: CAN_VIEW group_name: test-group - level: CAN_MANAGE user_name: user@company.com - level: CAN_RUN service_principal_name: 123456-abcdef ``` ## Tests Added corresponding unit tests + ran `bundle validate` and `bundle deploy` manually --- bundle/config/root.go | 12 ++ bundle/config/target.go | 7 +- bundle/permissions/mutator.go | 136 +++++++++++++++++ bundle/permissions/mutator_test.go | 141 ++++++++++++++++++ bundle/permissions/utils.go | 81 ++++++++++ bundle/permissions/workspace_root.go | 78 ++++++++++ bundle/permissions/workspace_root_test.go | 129 ++++++++++++++++ bundle/phases/deploy.go | 2 + bundle/phases/initialize.go | 2 + .../tests/bundle_permissions/databricks.yml | 35 +++++ bundle/tests/bundle_permissions_test.go | 56 +++++++ 11 files changed, 678 insertions(+), 1 deletion(-) create mode 100644 bundle/permissions/mutator.go create mode 100644 bundle/permissions/mutator_test.go create mode 100644 bundle/permissions/utils.go create mode 100644 bundle/permissions/workspace_root.go create mode 100644 bundle/permissions/workspace_root_test.go create mode 100644 bundle/tests/bundle_permissions/databricks.yml create mode 100644 bundle/tests/bundle_permissions_test.go diff --git a/bundle/config/root.go b/bundle/config/root.go index 31867c6c..1fb5773b 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/ghodss/yaml" @@ -56,6 +57,10 @@ type Root struct { RunAs *jobs.JobRunAs `json:"run_as,omitempty"` Experimental *Experimental `json:"experimental,omitempty"` + + // Permissions section allows to define permissions which will be + // applied to all resources defined in bundle + Permissions []resources.Permission `json:"permissions,omitempty"` } // Load loads the bundle configuration file at the specified path. @@ -237,5 +242,12 @@ func (r *Root) MergeTargetOverrides(target *Target) error { } } + if target.Permissions != nil { + err = mergo.Merge(&r.Permissions, target.Permissions, mergo.WithAppendSlice) + if err != nil { + return err + } + } + return nil } diff --git a/bundle/config/target.go b/bundle/config/target.go index fc776c7b..1264430e 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -1,6 +1,9 @@ package config -import "github.com/databricks/databricks-sdk-go/service/jobs" +import ( + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" +) type Mode string @@ -37,6 +40,8 @@ type Target struct { RunAs *jobs.JobRunAs `json:"run_as,omitempty"` Sync *Sync `json:"sync,omitempty"` + + Permissions []resources.Permission `json:"permissions,omitempty"` } const ( diff --git a/bundle/permissions/mutator.go b/bundle/permissions/mutator.go new file mode 100644 index 00000000..025556f3 --- /dev/null +++ b/bundle/permissions/mutator.go @@ -0,0 +1,136 @@ +package permissions + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" +) + +const CAN_MANAGE = "CAN_MANAGE" +const CAN_VIEW = "CAN_VIEW" +const CAN_RUN = "CAN_RUN" + +var allowedLevels = []string{CAN_MANAGE, CAN_VIEW, CAN_RUN} +var levelsMap = map[string](map[string]string){ + "jobs": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_VIEW", + CAN_RUN: "CAN_MANAGE_RUN", + }, + "pipelines": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_VIEW", + CAN_RUN: "CAN_RUN", + }, + "mlflow_experiments": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_READ", + }, + "mlflow_models": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_READ", + }, + "model_serving_endpoints": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_VIEW", + CAN_RUN: "CAN_QUERY", + }, +} + +type bundlePermissions struct{} + +func ApplyBundlePermissions() bundle.Mutator { + return &bundlePermissions{} +} + +func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) error { + err := validate(b) + if err != nil { + return err + } + + applyForJobs(ctx, b) + applyForPipelines(ctx, b) + applyForMlModels(ctx, b) + applyForMlExperiments(ctx, b) + applyForModelServiceEndpoints(ctx, b) + + return nil +} + +func validate(b *bundle.Bundle) error { + for _, p := range b.Config.Permissions { + if !slices.Contains(allowedLevels, p.Level) { + return fmt.Errorf("invalid permission level: %s, allowed values: [%s]", p.Level, strings.Join(allowedLevels, ", ")) + } + } + + return nil +} + +func applyForJobs(ctx context.Context, b *bundle.Bundle) { + for _, job := range b.Config.Resources.Jobs { + job.Permissions = append(job.Permissions, convert( + ctx, + b.Config.Permissions, + job.Permissions, + job.Name, + levelsMap["jobs"], + )...) + } +} + +func applyForPipelines(ctx context.Context, b *bundle.Bundle) { + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Permissions = append(pipeline.Permissions, convert( + ctx, + b.Config.Permissions, + pipeline.Permissions, + pipeline.Name, + levelsMap["pipelines"], + )...) + } +} + +func applyForMlExperiments(ctx context.Context, b *bundle.Bundle) { + for _, experiment := range b.Config.Resources.Experiments { + experiment.Permissions = append(experiment.Permissions, convert( + ctx, + b.Config.Permissions, + experiment.Permissions, + experiment.Name, + levelsMap["mlflow_experiments"], + )...) + } +} + +func applyForMlModels(ctx context.Context, b *bundle.Bundle) { + for _, model := range b.Config.Resources.Models { + model.Permissions = append(model.Permissions, convert( + ctx, + b.Config.Permissions, + model.Permissions, + model.Name, + levelsMap["mlflow_models"], + )...) + } +} + +func applyForModelServiceEndpoints(ctx context.Context, b *bundle.Bundle) { + for _, model := range b.Config.Resources.ModelServingEndpoints { + model.Permissions = append(model.Permissions, convert( + ctx, + b.Config.Permissions, + model.Permissions, + model.Name, + levelsMap["model_serving_endpoints"], + )...) + } +} + +func (m *bundlePermissions) Name() string { + return "ApplyBundlePermissions" +} diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go new file mode 100644 index 00000000..d9bf3efe --- /dev/null +++ b/bundle/permissions/mutator_test.go @@ -0,0 +1,141 @@ +package permissions + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/require" +) + +func TestApplyBundlePermissions(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Users/foo@bar.com", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "TestUser"}, + {Level: CAN_VIEW, GroupName: "TestGroup"}, + {Level: CAN_RUN, ServicePrincipalName: "TestServicePrincipal"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": {JobSettings: &jobs.JobSettings{}}, + "job_2": {JobSettings: &jobs.JobSettings{}}, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + }, + Models: map[string]*resources.MlflowModel{ + "model_1": {Model: &ml.Model{}}, + "model_2": {Model: &ml.Model{}}, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "experiment_1": {Experiment: &ml.Experiment{}}, + "experiment_2": {Experiment: &ml.Experiment{}}, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint_1": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + "endpoint_2": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + }, + }, + }, + } + + err := bundle.Apply(context.Background(), b, ApplyBundlePermissions()) + require.NoError(t, err) + + require.Len(t, b.Config.Resources.Jobs["job_1"].Permissions, 3) + require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_MANAGE_RUN", ServicePrincipalName: "TestServicePrincipal"}) + + require.Len(t, b.Config.Resources.Jobs["job_2"].Permissions, 3) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_MANAGE_RUN", ServicePrincipalName: "TestServicePrincipal"}) + + require.Len(t, b.Config.Resources.Pipelines["pipeline_1"].Permissions, 3) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_1"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_1"].Permissions, resources.Permission{Level: "CAN_RUN", ServicePrincipalName: "TestServicePrincipal"}) + + require.Len(t, b.Config.Resources.Pipelines["pipeline_2"].Permissions, 3) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_2"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.Pipelines["pipeline_2"].Permissions, resources.Permission{Level: "CAN_RUN", ServicePrincipalName: "TestServicePrincipal"}) + + require.Len(t, b.Config.Resources.Models["model_1"].Permissions, 2) + require.Contains(t, b.Config.Resources.Models["model_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Models["model_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.Models["model_2"].Permissions, 2) + require.Contains(t, b.Config.Resources.Models["model_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Models["model_2"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.Experiments["experiment_1"].Permissions, 2) + require.Contains(t, b.Config.Resources.Experiments["experiment_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Experiments["experiment_1"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.Experiments["experiment_2"].Permissions, 2) + require.Contains(t, b.Config.Resources.Experiments["experiment_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Experiments["experiment_2"].Permissions, resources.Permission{Level: "CAN_READ", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint_1"].Permissions, 3) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_1"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_1"].Permissions, resources.Permission{Level: "CAN_QUERY", ServicePrincipalName: "TestServicePrincipal"}) + + require.Len(t, b.Config.Resources.ModelServingEndpoints["endpoint_2"].Permissions, 3) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_2"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.ModelServingEndpoints["endpoint_2"].Permissions, resources.Permission{Level: "CAN_QUERY", ServicePrincipalName: "TestServicePrincipal"}) +} + +func TestWarningOnOverlapPermission(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Users/foo@bar.com", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "TestUser"}, + {Level: CAN_VIEW, GroupName: "TestGroup"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": { + Permissions: []resources.Permission{ + {Level: CAN_VIEW, UserName: "TestUser"}, + }, + JobSettings: &jobs.JobSettings{}, + }, + "job_2": { + Permissions: []resources.Permission{ + {Level: CAN_VIEW, UserName: "TestUser2"}, + }, + JobSettings: &jobs.JobSettings{}, + }, + }, + }, + }, + } + + err := bundle.Apply(context.Background(), b, ApplyBundlePermissions()) + require.NoError(t, err) + + require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_VIEW", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Jobs["job_1"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_VIEW", UserName: "TestUser2"}) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.Jobs["job_2"].Permissions, resources.Permission{Level: "CAN_VIEW", GroupName: "TestGroup"}) + +} diff --git a/bundle/permissions/utils.go b/bundle/permissions/utils.go new file mode 100644 index 00000000..9072cd25 --- /dev/null +++ b/bundle/permissions/utils.go @@ -0,0 +1,81 @@ +package permissions + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +func convert( + ctx context.Context, + bundlePermissions []resources.Permission, + resourcePermissions []resources.Permission, + resourceName string, + lm map[string]string, +) []resources.Permission { + permissions := make([]resources.Permission, 0) + for _, p := range bundlePermissions { + level, ok := lm[p.Level] + // If there is no bundle permission level defined in the map, it means + // it's not applicable for the resource, therefore skipping + if !ok { + continue + } + + if notifyForPermissionOverlap(ctx, p, resourcePermissions, resourceName) { + continue + } + + permissions = append(permissions, resources.Permission{ + Level: level, + UserName: p.UserName, + GroupName: p.GroupName, + ServicePrincipalName: p.ServicePrincipalName, + }) + } + + return permissions +} + +func isPermissionOverlap( + permission resources.Permission, + resourcePermissions []resources.Permission, + resourceName string, +) (bool, diag.Diagnostics) { + var diagnostics diag.Diagnostics + for _, rp := range resourcePermissions { + if rp.GroupName != "" && rp.GroupName == permission.GroupName { + diagnostics = diagnostics.Extend( + diag.Warningf("'%s' already has permissions set for '%s' group", resourceName, rp.GroupName), + ) + } + + if rp.UserName != "" && rp.UserName == permission.UserName { + diagnostics = diagnostics.Extend( + diag.Warningf("'%s' already has permissions set for '%s' user name", resourceName, rp.UserName), + ) + } + + if rp.ServicePrincipalName != "" && rp.ServicePrincipalName == permission.ServicePrincipalName { + diagnostics = diagnostics.Extend( + diag.Warningf("'%s' already has permissions set for '%s' service principal name", resourceName, rp.ServicePrincipalName), + ) + } + } + + return len(diagnostics) > 0, diagnostics +} + +func notifyForPermissionOverlap( + ctx context.Context, + permission resources.Permission, + resourcePermissions []resources.Permission, + resourceName string, +) bool { + isOverlap, _ := isPermissionOverlap(permission, resourcePermissions, resourceName) + // TODO: When we start to collect all diagnostics at the top level and visualize jointly, + // use diagnostics returned from isPermissionOverlap to display warnings + + return isOverlap +} diff --git a/bundle/permissions/workspace_root.go b/bundle/permissions/workspace_root.go new file mode 100644 index 00000000..a8eb9e27 --- /dev/null +++ b/bundle/permissions/workspace_root.go @@ -0,0 +1,78 @@ +package permissions + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type workspaceRootPermissions struct { +} + +func ApplyWorkspaceRootPermissions() bundle.Mutator { + return &workspaceRootPermissions{} +} + +// Apply implements bundle.Mutator. +func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) error { + err := giveAccessForWorkspaceRoot(ctx, b) + if err != nil { + return err + } + + return nil +} + +func (*workspaceRootPermissions) Name() string { + return "ApplyWorkspaceRootPermissions" +} + +func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error { + permissions := make([]workspace.WorkspaceObjectAccessControlRequest, 0) + + for _, p := range b.Config.Permissions { + level, err := getWorkspaceObjectPermissionLevel(p.Level) + if err != nil { + return err + } + + permissions = append(permissions, workspace.WorkspaceObjectAccessControlRequest{ + GroupName: p.GroupName, + UserName: p.UserName, + ServicePrincipalName: p.ServicePrincipalName, + PermissionLevel: level, + }) + } + + if len(permissions) == 0 { + return nil + } + + w := b.WorkspaceClient().Workspace + obj, err := w.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + if err != nil { + return err + } + + _, err = w.UpdatePermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{ + WorkspaceObjectId: fmt.Sprint(obj.ObjectId), + WorkspaceObjectType: "directories", + AccessControlList: permissions, + }) + return err +} + +func getWorkspaceObjectPermissionLevel(bundlePermission string) (workspace.WorkspaceObjectPermissionLevel, error) { + switch bundlePermission { + case CAN_MANAGE: + return workspace.WorkspaceObjectPermissionLevelCanManage, nil + case CAN_RUN: + return workspace.WorkspaceObjectPermissionLevelCanRun, nil + case CAN_VIEW: + return workspace.WorkspaceObjectPermissionLevelCanRead, nil + default: + return "", fmt.Errorf("unsupported bundle permission level %s", bundlePermission) + } +} diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go new file mode 100644 index 00000000..21cc4176 --- /dev/null +++ b/bundle/permissions/workspace_root_test.go @@ -0,0 +1,129 @@ +package permissions + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/require" +) + +type MockWorkspaceClient struct { + t *testing.T +} + +// Delete implements workspace.WorkspaceService. +func (MockWorkspaceClient) Delete(ctx context.Context, request workspace.Delete) error { + panic("unimplemented") +} + +// Export implements workspace.WorkspaceService. +func (MockWorkspaceClient) Export(ctx context.Context, request workspace.ExportRequest) (*workspace.ExportResponse, error) { + panic("unimplemented") +} + +// GetPermissionLevels implements workspace.WorkspaceService. +func (MockWorkspaceClient) GetPermissionLevels(ctx context.Context, request workspace.GetWorkspaceObjectPermissionLevelsRequest) (*workspace.GetWorkspaceObjectPermissionLevelsResponse, error) { + panic("unimplemented") +} + +// GetPermissions implements workspace.WorkspaceService. +func (MockWorkspaceClient) GetPermissions(ctx context.Context, request workspace.GetWorkspaceObjectPermissionsRequest) (*workspace.WorkspaceObjectPermissions, error) { + panic("unimplemented") +} + +// GetStatus implements workspace.WorkspaceService. +func (MockWorkspaceClient) GetStatus(ctx context.Context, request workspace.GetStatusRequest) (*workspace.ObjectInfo, error) { + return &workspace.ObjectInfo{ + ObjectId: 1234, ObjectType: "directories", Path: "/Users/foo@bar.com", + }, nil +} + +// Import implements workspace.WorkspaceService. +func (MockWorkspaceClient) Import(ctx context.Context, request workspace.Import) error { + panic("unimplemented") +} + +// List implements workspace.WorkspaceService. +func (MockWorkspaceClient) List(ctx context.Context, request workspace.ListWorkspaceRequest) (*workspace.ListResponse, error) { + panic("unimplemented") +} + +// Mkdirs implements workspace.WorkspaceService. +func (MockWorkspaceClient) Mkdirs(ctx context.Context, request workspace.Mkdirs) error { + panic("unimplemented") +} + +// SetPermissions implements workspace.WorkspaceService. +func (MockWorkspaceClient) SetPermissions(ctx context.Context, request workspace.WorkspaceObjectPermissionsRequest) (*workspace.WorkspaceObjectPermissions, error) { + panic("unimplemented") +} + +// UpdatePermissions implements workspace.WorkspaceService. +func (m MockWorkspaceClient) UpdatePermissions(ctx context.Context, request workspace.WorkspaceObjectPermissionsRequest) (*workspace.WorkspaceObjectPermissions, error) { + require.Equal(m.t, "1234", request.WorkspaceObjectId) + require.Equal(m.t, "directories", request.WorkspaceObjectType) + require.Contains(m.t, request.AccessControlList, workspace.WorkspaceObjectAccessControlRequest{ + UserName: "TestUser", + PermissionLevel: "CAN_MANAGE", + }) + require.Contains(m.t, request.AccessControlList, workspace.WorkspaceObjectAccessControlRequest{ + GroupName: "TestGroup", + PermissionLevel: "CAN_READ", + }) + require.Contains(m.t, request.AccessControlList, workspace.WorkspaceObjectAccessControlRequest{ + ServicePrincipalName: "TestServicePrincipal", + PermissionLevel: "CAN_RUN", + }) + + return nil, nil +} + +func TestApplyWorkspaceRootPermissions(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Users/foo@bar.com", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "TestUser"}, + {Level: CAN_VIEW, GroupName: "TestGroup"}, + {Level: CAN_RUN, ServicePrincipalName: "TestServicePrincipal"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": {JobSettings: &jobs.JobSettings{}}, + "job_2": {JobSettings: &jobs.JobSettings{}}, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + }, + Models: map[string]*resources.MlflowModel{ + "model_1": {Model: &ml.Model{}}, + "model_2": {Model: &ml.Model{}}, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "experiment_1": {Experiment: &ml.Experiment{}}, + "experiment_2": {Experiment: &ml.Experiment{}}, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint_1": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + "endpoint_2": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + }, + }, + }, + } + + b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceClient{t}) + + err := bundle.Apply(context.Background(), b, ApplyWorkspaceRootPermissions()) + require.NoError(t, err) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 805bae80..6f0d3a6c 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" ) @@ -27,6 +28,7 @@ func Deploy() bundle.Mutator { artifacts.UploadAll(), python.TransformWheelTask(), files.Upload(), + permissions.ApplyWorkspaceRootPermissions(), terraform.Interpolate(), terraform.Write(), terraform.StatePull(), diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index e03a6336..fb9e7b24 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" ) @@ -34,6 +35,7 @@ func Initialize() bundle.Mutator { mutator.ExpandPipelineGlobPaths(), mutator.TranslatePaths(), python.WrapperWarning(), + permissions.ApplyBundlePermissions(), terraform.Initialize(), scripts.Execute(config.ScriptPostInit), }, diff --git a/bundle/tests/bundle_permissions/databricks.yml b/bundle/tests/bundle_permissions/databricks.yml new file mode 100644 index 00000000..78f3d3d7 --- /dev/null +++ b/bundle/tests/bundle_permissions/databricks.yml @@ -0,0 +1,35 @@ +bundle: + name: bundle_permissions + +permissions: + - level: CAN_RUN + user_name: test@company.com + +targets: + development: + permissions: + - level: CAN_MANAGE + group_name: devs + - level: CAN_VIEW + service_principal_name: 1234-abcd + - level: CAN_RUN + user_name: bot@company.com + +resources: + pipelines: + nyc_taxi_pipeline: + target: nyc_taxi_production + development: false + photon: true + + jobs: + pipeline_schedule: + name: Daily refresh of production pipeline + + schedule: + quartz_cron_expression: 6 6 11 * * ? + timezone_id: UTC + + tasks: + - pipeline_task: + pipeline_id: "to be interpolated" diff --git a/bundle/tests/bundle_permissions_test.go b/bundle/tests/bundle_permissions_test.go new file mode 100644 index 00000000..3ea9dc2e --- /dev/null +++ b/bundle/tests/bundle_permissions_test.go @@ -0,0 +1,56 @@ +package config_tests + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBundlePermissions(t *testing.T) { + b := load(t, "./bundle_permissions") + assert.Contains(t, b.Config.Permissions, resources.Permission{Level: "CAN_RUN", UserName: "test@company.com"}) + assert.NotContains(t, b.Config.Permissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.NotContains(t, b.Config.Permissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.NotContains(t, b.Config.Permissions, resources.Permission{Level: "CAN_RUN", UserName: "bot@company.com"}) + + err := bundle.Apply(context.Background(), b, permissions.ApplyBundlePermissions()) + require.NoError(t, err) + pipelinePermissions := b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Permissions + assert.Contains(t, pipelinePermissions, resources.Permission{Level: "CAN_RUN", UserName: "test@company.com"}) + assert.NotContains(t, pipelinePermissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.NotContains(t, pipelinePermissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.NotContains(t, pipelinePermissions, resources.Permission{Level: "CAN_RUN", UserName: "bot@company.com"}) + + jobsPermissions := b.Config.Resources.Jobs["pipeline_schedule"].Permissions + assert.Contains(t, jobsPermissions, resources.Permission{Level: "CAN_MANAGE_RUN", UserName: "test@company.com"}) + assert.NotContains(t, jobsPermissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.NotContains(t, jobsPermissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.NotContains(t, jobsPermissions, resources.Permission{Level: "CAN_RUN", UserName: "bot@company.com"}) +} + +func TestBundlePermissionsDevTarget(t *testing.T) { + b := loadTarget(t, "./bundle_permissions", "development") + assert.Contains(t, b.Config.Permissions, resources.Permission{Level: "CAN_RUN", UserName: "test@company.com"}) + assert.Contains(t, b.Config.Permissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.Contains(t, b.Config.Permissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.Contains(t, b.Config.Permissions, resources.Permission{Level: "CAN_RUN", UserName: "bot@company.com"}) + + err := bundle.Apply(context.Background(), b, permissions.ApplyBundlePermissions()) + require.NoError(t, err) + pipelinePermissions := b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Permissions + assert.Contains(t, pipelinePermissions, resources.Permission{Level: "CAN_RUN", UserName: "test@company.com"}) + assert.Contains(t, pipelinePermissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.Contains(t, pipelinePermissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.Contains(t, pipelinePermissions, resources.Permission{Level: "CAN_RUN", UserName: "bot@company.com"}) + + jobsPermissions := b.Config.Resources.Jobs["pipeline_schedule"].Permissions + assert.Contains(t, jobsPermissions, resources.Permission{Level: "CAN_MANAGE_RUN", UserName: "test@company.com"}) + assert.Contains(t, jobsPermissions, resources.Permission{Level: "CAN_MANAGE", GroupName: "devs"}) + assert.Contains(t, jobsPermissions, resources.Permission{Level: "CAN_VIEW", ServicePrincipalName: "1234-abcd"}) + assert.Contains(t, jobsPermissions, resources.Permission{Level: "CAN_MANAGE_RUN", UserName: "bot@company.com"}) +}