Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls

This commit is contained in:
Lennart Kats 2024-10-11 14:42:16 +02:00
commit ef2400fb76
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
84 changed files with 1852 additions and 205 deletions

View File

@ -1 +1 @@
6f6b1371e640f2dfeba72d365ac566368656f6b6
0c86ea6dbd9a730c24ff0d4e509603e476955ac5

View File

@ -1,5 +1,33 @@
# Version changelog
## [Release] Release v0.230.0
Notable changes for Databricks Asset Bundles:
Workspace paths are automatically prefixed with `/Workspace`. In addition, all usage of path strings such as `/Workspace/${workspace.root_path}/...` in bundle configuration is automatically replaced with `${workspace.root_path}/...` and generates a warning as part of bundle validate.
More details can be found here: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths
Bundles:
* Add an error if state files grow bigger than the export limit ([#1795](https://github.com/databricks/cli/pull/1795)).
* Always prepend bundle remote paths with /Workspace ([#1724](https://github.com/databricks/cli/pull/1724)).
* Add resource path field to bundle workspace configuration ([#1800](https://github.com/databricks/cli/pull/1800)).
* Add validation for files with a `.(resource-name).yml` extension ([#1780](https://github.com/databricks/cli/pull/1780)).
Internal:
* Remove deprecated or readonly fields from the bundle schema ([#1809](https://github.com/databricks/cli/pull/1809)).
API Changes:
* Changed `databricks git-credentials create`, `databricks git-credentials delete`, `databricks git-credentials get`, `databricks git-credentials list`, `databricks git-credentials update` commands .
* Changed `databricks repos create`, `databricks repos delete`, `databricks repos get`, `databricks repos update` command .
OpenAPI commit 0c86ea6dbd9a730c24ff0d4e509603e476955ac5 (2024-10-02)
Dependency updates:
* Upgrade TF provider to 1.53.0 ([#1815](https://github.com/databricks/cli/pull/1815)).
* Bump golang.org/x/term from 0.24.0 to 0.25.0 ([#1811](https://github.com/databricks/cli/pull/1811)).
* Bump golang.org/x/text from 0.18.0 to 0.19.0 ([#1812](https://github.com/databricks/cli/pull/1812)).
* Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 ([#1810](https://github.com/databricks/cli/pull/1810)).
## [Release] Release v0.229.0
Bundles:

View File

@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) {
func TestEntryPoint(t *testing.T) {
b := &bundle.Bundle{
BundleRootPath: "testdata",
BundleRootPath: "testdata/basic",
}
diags := bundle.Apply(context.Background(), b, loader.EntryPoint())
require.NoError(t, diags.Error())

View File

@ -3,12 +3,135 @@ package loader
import (
"context"
"fmt"
"slices"
"sort"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
)
func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics {
for _, resourceDescription := range config.SupportedResources() {
singularName := resourceDescription.SingularName
for _, yamlExt := range []string{"yml", "yaml"} {
ext := fmt.Sprintf(".%s.%s", singularName, yamlExt)
if strings.HasSuffix(filePath, ext) {
return validateSingleResourceDefined(configRoot, ext, singularName)
}
}
}
return nil
}
func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics {
type resource struct {
path dyn.Path
value dyn.Value
typ string
key string
}
resources := []resource{}
supportedResources := config.SupportedResources()
// Gather all resources defined in the resources block.
_, err := dyn.MapByPattern(
configRoot,
dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
// The key for the resource, e.g. "my_job" for jobs.my_job.
k := p[2].Key()
// The type of the resource, e.g. "job" for jobs.my_job.
typ := supportedResources[p[1].Key()].SingularName
resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
return v, nil
})
if err != nil {
return diag.FromErr(err)
}
// Gather all resources defined in a target block.
_, err = dyn.MapByPattern(
configRoot,
dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()),
func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
// The key for the resource, e.g. "my_job" for jobs.my_job.
k := p[4].Key()
// The type of the resource, e.g. "job" for jobs.my_job.
typ := supportedResources[p[3].Key()].SingularName
resources = append(resources, resource{path: p, value: v, typ: typ, key: k})
return v, nil
})
if err != nil {
return diag.FromErr(err)
}
typeMatch := true
seenKeys := map[string]struct{}{}
for _, rr := range resources {
// case: The resource is not of the correct type.
if rr.typ != typ {
typeMatch = false
break
}
seenKeys[rr.key] = struct{}{}
}
// Format matches. There's at most one resource defined in the file.
// The resource is also of the correct type.
if typeMatch && len(seenKeys) <= 1 {
return nil
}
detail := strings.Builder{}
detail.WriteString("The following resources are defined or configured in this file:\n")
lines := []string{}
for _, r := range resources {
lines = append(lines, fmt.Sprintf(" - %s (%s)\n", r.key, r.typ))
}
// Sort the lines to print to make the output deterministic.
sort.Strings(lines)
// Compact the lines before writing them to the message to remove any duplicate lines.
// This is needed because we do not dedup earlier when gathering the resources
// and it's valid to define the same resource in both the resources and targets block.
lines = slices.Compact(lines)
for _, l := range lines {
detail.WriteString(l)
}
locations := []dyn.Location{}
paths := []dyn.Path{}
for _, rr := range resources {
locations = append(locations, rr.value.Locations()...)
paths = append(paths, rr.path)
}
// Sort the locations and paths to make the output deterministic.
sort.Slice(locations, func(i, j int) bool {
return locations[i].String() < locations[j].String()
})
sort.Slice(paths, func(i, j int) bool {
return paths[i].String() < paths[j].String()
})
return diag.Diagnostics{
{
Severity: diag.Recommendation,
Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext),
Detail: detail.String(),
Locations: locations,
Paths: paths,
},
}
}
type processInclude struct {
fullPath string
relPath string
@ -31,6 +154,13 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
if diags.HasError() {
return diags
}
// Add any diagnostics associated with the file format.
diags = append(diags, validateFileFormat(this.Value(), m.relPath)...)
if diags.HasError() {
return diags
}
err := b.Config.Merge(this)
if err != nil {
diags = diags.Extend(diag.FromErr(err))

View File

@ -8,13 +8,15 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/loader"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestProcessInclude(t *testing.T) {
b := &bundle.Bundle{
BundleRootPath: "testdata",
BundleRootPath: "testdata/basic",
Config: config.Root{
Workspace: config.Workspace{
Host: "foo",
@ -33,3 +35,184 @@ func TestProcessInclude(t *testing.T) {
require.NoError(t, diags.Error())
assert.Equal(t, "bar", b.Config.Workspace.Host)
}
func TestProcessIncludeFormatMatch(t *testing.T) {
for _, fileName := range []string{
"one_job.job.yml",
"one_pipeline.pipeline.yaml",
"two_job.yml",
"job_and_pipeline.yml",
"multiple_resources.yml",
} {
t.Run(fileName, func(t *testing.T) {
b := &bundle.Bundle{
BundleRootPath: "testdata/format_match",
Config: config.Root{
Bundle: config.Bundle{
Name: "format_test",
},
},
}
m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
diags := bundle.Apply(context.Background(), b, m)
assert.Empty(t, diags)
})
}
}
func TestProcessIncludeFormatNotMatch(t *testing.T) {
for fileName, expectedDiags := range map[string]diag.Diagnostics{
"single_job.pipeline.yaml": {
{
Severity: diag.Recommendation,
Summary: "define a single pipeline in a file with the .pipeline.yaml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 11, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 4, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.jobs.job1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
},
},
},
"job_and_pipeline.job.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single job in a file with the .job.yml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 11, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 4, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.pipelines.pipeline1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
},
},
},
"job_and_pipeline.experiment.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single experiment in a file with the .experiment.yml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 11, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 4, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.pipelines.pipeline1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
},
},
},
"two_jobs.job.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single job in a file with the .job.yml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 4, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 7, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.jobs.job1"),
dyn.MustPathFromString("resources.jobs.job2"),
},
},
},
"second_job_in_target.job.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single job in a file with the .job.yml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 11, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 4, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.jobs.job1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
},
},
},
"two_jobs_in_target.job.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single job in a file with the .job.yml extension.",
Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n",
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 6, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 8, Column: 11},
},
Paths: []dyn.Path{
dyn.MustPathFromString("targets.target1.resources.jobs.job1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job2"),
},
},
},
"multiple_resources.model_serving_endpoint.yml": {
{
Severity: diag.Recommendation,
Summary: "define a single model serving endpoint in a file with the .model_serving_endpoint.yml extension.",
Detail: `The following resources are defined or configured in this file:
- experiment1 (experiment)
- job1 (job)
- job2 (job)
- job3 (job)
- model1 (model)
- model_serving_endpoint1 (model_serving_endpoint)
- pipeline1 (pipeline)
- pipeline2 (pipeline)
- quality_monitor1 (quality_monitor)
- registered_model1 (registered_model)
- schema1 (schema)
`,
Locations: []dyn.Location{
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 12, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 14, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 18, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 22, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 24, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 28, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 35, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 39, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 43, Column: 11},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 4, Column: 7},
{File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 8, Column: 7},
},
Paths: []dyn.Path{
dyn.MustPathFromString("resources.experiments.experiment1"),
dyn.MustPathFromString("resources.jobs.job1"),
dyn.MustPathFromString("resources.jobs.job2"),
dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"),
dyn.MustPathFromString("resources.models.model1"),
dyn.MustPathFromString("resources.pipelines.pipeline1"),
dyn.MustPathFromString("resources.pipelines.pipeline2"),
dyn.MustPathFromString("resources.schemas.schema1"),
dyn.MustPathFromString("targets.target1.resources.jobs.job3"),
dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"),
dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"),
},
},
},
} {
t.Run(fileName, func(t *testing.T) {
b := &bundle.Bundle{
BundleRootPath: "testdata/format_not_match",
Config: config.Root{
Bundle: config.Bundle{
Name: "format_test",
},
},
}
m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName)
diags := bundle.Apply(context.Background(), b, m)
require.Len(t, diags, 1)
assert.Equal(t, expectedDiags, diags)
})
}
}

View File

@ -0,0 +1,11 @@
resources:
pipelines:
pipeline1:
name: pipeline1
targets:
target1:
resources:
jobs:
job1:
name: job1

View File

@ -0,0 +1,43 @@
resources:
experiments:
experiment1:
name: experiment1
model_serving_endpoints:
model_serving_endpoint1:
name: model_serving_endpoint1
jobs:
job1:
name: job1
job2:
name: job2
models:
model1:
name: model1
pipelines:
pipeline1:
name: pipeline1
pipeline2:
name: pipeline2
schemas:
schema1:
name: schema1
targets:
target1:
resources:
quality_monitors:
quality_monitor1:
baseline_table_name: quality_monitor1
jobs:
job3:
name: job3
registered_models:
registered_model1:
name: registered_model1

View File

@ -0,0 +1,11 @@
resources:
jobs:
job1:
name: job1
targets:
target1:
resources:
jobs:
job1:
description: job1

View File

@ -0,0 +1,4 @@
resources:
pipelines:
pipeline1:
name: pipeline1

View File

@ -0,0 +1,7 @@
resources:
jobs:
job1:
name: job1
job2:
name: job2

View File

@ -0,0 +1,11 @@
resources:
pipelines:
pipeline1:
name: pipeline1
targets:
target1:
resources:
jobs:
job1:
name: job1

View File

@ -0,0 +1,11 @@
resources:
pipelines:
pipeline1:
name: pipeline1
targets:
target1:
resources:
jobs:
job1:
name: job1

View File

@ -0,0 +1,43 @@
resources:
experiments:
experiment1:
name: experiment1
model_serving_endpoints:
model_serving_endpoint1:
name: model_serving_endpoint1
jobs:
job1:
name: job1
job2:
name: job2
models:
model1:
name: model1
pipelines:
pipeline1:
name: pipeline1
pipeline2:
name: pipeline2
schemas:
schema1:
name: schema1
targets:
target1:
resources:
quality_monitors:
quality_monitor1:
baseline_table_name: quality_monitor1
jobs:
job3:
name: job3
registered_models:
registered_model1:
name: registered_model1

View File

@ -0,0 +1,11 @@
resources:
jobs:
job1:
name: job1
targets:
target1:
resources:
jobs:
job2:
name: job2

View File

@ -0,0 +1,11 @@
resources:
jobs:
job1:
name: job1
targets:
target1:
resources:
jobs:
job1:
description: job1

View File

@ -0,0 +1,7 @@
resources:
jobs:
job1:
name: job1
job2:
name: job2

View File

@ -0,0 +1,8 @@
targets:
target1:
resources:
jobs:
job1:
description: job1
job2:
description: job2

View File

@ -5,8 +5,8 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/cli/libs/tags"
)
@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag.
}
b.Config.Workspace.CurrentUser = &config.User{
ShortName: auth.GetShortUserName(me),
ShortName: iamutil.GetShortUserName(me),
User: me,
}

View File

@ -6,9 +6,9 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/cli/libs/log"
)
@ -174,7 +174,7 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di
transformDevelopmentMode(ctx, b)
return diags
case config.Production:
isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName)
isPrincipal := iamutil.IsServicePrincipal(b.Config.Workspace.CurrentUser.User)
return validateProductionMode(ctx, b, isPrincipal)
case "":
// No action

View File

@ -30,50 +30,44 @@ func (m *setRunAs) Name() string {
return "SetRunAs"
}
type errUnsupportedResourceTypeForRunAs struct {
resourceType string
resourceLocation dyn.Location
currentUser string
runAsUser string
func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser string, runAsUser string) diag.Diagnostics {
return diag.Diagnostics{{
Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+
"Current identity: %s. Run as identity: %s.\n"+
"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser),
Locations: []dyn.Location{location},
Severity: diag.Error,
}}
}
func (e errUnsupportedResourceTypeForRunAs) Error() string {
return fmt.Sprintf("%s are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s. Current identity: %s. Run as identity: %s", e.resourceType, e.resourceLocation, e.currentUser, e.runAsUser)
}
func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
diags := diag.Diagnostics{}
type errBothSpAndUserSpecified struct {
spName string
spLoc dyn.Location
userName string
userLoc dyn.Location
}
neitherSpecifiedErr := diag.Diagnostics{{
Summary: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
Locations: []dyn.Location{b.Config.GetLocation("run_as")},
Severity: diag.Error,
}}
func (e errBothSpAndUserSpecified) Error() string {
return fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name %q is specified at %s. A user_name %q is defined at %s", e.spName, e.spLoc, e.userName, e.userLoc)
}
func validateRunAs(b *bundle.Bundle) error {
neitherSpecifiedErr := fmt.Errorf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s", b.Config.GetLocation("run_as"))
// Error if neither service_principal_name nor user_name are specified, but the
// Fail fast if neither service_principal_name nor user_name are specified, but the
// run_as section is present.
if b.Config.Value().Get("run_as").Kind() == dyn.KindNil {
return neitherSpecifiedErr
}
// Error if one or both of service_principal_name and user_name are specified,
// Fail fast if one or both of service_principal_name and user_name are specified,
// but with empty values.
if b.Config.RunAs.ServicePrincipalName == "" && b.Config.RunAs.UserName == "" {
runAs := b.Config.RunAs
if runAs.ServicePrincipalName == "" && runAs.UserName == "" {
return neitherSpecifiedErr
}
// Error if both service_principal_name and user_name are specified
runAs := b.Config.RunAs
if runAs.UserName != "" && runAs.ServicePrincipalName != "" {
return errBothSpAndUserSpecified{
spName: runAs.ServicePrincipalName,
userName: runAs.UserName,
spLoc: b.Config.GetLocation("run_as.service_principal_name"),
userLoc: b.Config.GetLocation("run_as.user_name"),
}
diags = diags.Extend(diag.Diagnostics{{
Summary: "run_as section cannot specify both user_name and service_principal_name",
Locations: []dyn.Location{b.Config.GetLocation("run_as")},
Severity: diag.Error,
}})
}
identity := runAs.ServicePrincipalName
@ -83,40 +77,40 @@ func validateRunAs(b *bundle.Bundle) error {
// All resources are supported if the run_as identity is the same as the current deployment identity.
if identity == b.Config.Workspace.CurrentUser.UserName {
return nil
return diags
}
// DLT pipelines do not support run_as in the API.
if len(b.Config.Resources.Pipelines) > 0 {
return errUnsupportedResourceTypeForRunAs{
resourceType: "pipelines",
resourceLocation: b.Config.GetLocation("resources.pipelines"),
currentUser: b.Config.Workspace.CurrentUser.UserName,
runAsUser: identity,
}
diags = diags.Extend(reportRunAsNotSupported(
"pipelines",
b.Config.GetLocation("resources.pipelines"),
b.Config.Workspace.CurrentUser.UserName,
identity,
))
}
// Model serving endpoints do not support run_as in the API.
if len(b.Config.Resources.ModelServingEndpoints) > 0 {
return errUnsupportedResourceTypeForRunAs{
resourceType: "model_serving_endpoints",
resourceLocation: b.Config.GetLocation("resources.model_serving_endpoints"),
currentUser: b.Config.Workspace.CurrentUser.UserName,
runAsUser: identity,
}
diags = diags.Extend(reportRunAsNotSupported(
"model_serving_endpoints",
b.Config.GetLocation("resources.model_serving_endpoints"),
b.Config.Workspace.CurrentUser.UserName,
identity,
))
}
// Monitors do not support run_as in the API.
if len(b.Config.Resources.QualityMonitors) > 0 {
return errUnsupportedResourceTypeForRunAs{
resourceType: "quality_monitors",
resourceLocation: b.Config.GetLocation("resources.quality_monitors"),
currentUser: b.Config.Workspace.CurrentUser.UserName,
runAsUser: identity,
}
diags = diags.Extend(reportRunAsNotSupported(
"quality_monitors",
b.Config.GetLocation("resources.quality_monitors"),
b.Config.Workspace.CurrentUser.UserName,
identity,
))
}
return nil
return diags
}
func setRunAsForJobs(b *bundle.Bundle) {
@ -187,8 +181,9 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
}
// Assert the run_as configuration is valid in the context of the bundle
if err := validateRunAs(b); err != nil {
return diag.FromErr(err)
diags := validateRunAs(b)
if diags.HasError() {
return diags
}
setRunAsForJobs(b)

View File

@ -188,11 +188,8 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
Config: *r,
}
diags := bundle.Apply(context.Background(), b, SetRunAs())
assert.Equal(t, diags.Error().Error(), errUnsupportedResourceTypeForRunAs{
resourceType: rt,
resourceLocation: dyn.Location{},
currentUser: "alice",
runAsUser: "bob",
}.Error(), "expected run_as with a different identity than the current deployment user to not supported for resources of type: %s", rt)
assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+
"Current identity: alice. Run as identity: bob.\n"+
"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)
}
}

View File

@ -128,3 +128,22 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
return found[0], nil
}
type ResourceDescription struct {
SingularName string
}
// The keys of the map corresponds to the resource key in the bundle configuration.
func SupportedResources() map[string]ResourceDescription {
return map[string]ResourceDescription{
"jobs": {SingularName: "job"},
"pipelines": {SingularName: "pipeline"},
"models": {SingularName: "model"},
"experiments": {SingularName: "experiment"},
"model_serving_endpoints": {SingularName: "model_serving_endpoint"},
"registered_models": {SingularName: "registered_model"},
"quality_monitors": {SingularName: "quality_monitor"},
"schemas": {SingularName: "schema"},
"clusters": {SingularName: "cluster"},
}
}

View File

@ -81,3 +81,18 @@ func TestResourcesAllResourcesCompleteness(t *testing.T) {
assert.True(t, exists, "Field %s is missing in AllResources map", field.Name)
}
}
func TestSupportedResources(t *testing.T) {
expected := map[string]ResourceDescription{}
typ := reflect.TypeOf(Resources{})
for i := 0; i < typ.NumField(); i++ {
field := typ.Field(i)
jsonTags := strings.Split(field.Tag.Get("json"), ",")
singularName := strings.TrimSuffix(jsonTags[0], "s")
expected[jsonTags[0]] = ResourceDescription{SingularName: singularName}
}
// Please add your resource to the SupportedResources() function in resources.go
// if you are adding a new resource.
assert.Equal(t, expected, SupportedResources())
}

View File

@ -2,9 +2,12 @@ package files
import (
"context"
"errors"
"fmt"
"io/fs"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
@ -35,6 +38,9 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
b.Files, err = sync.RunOnce(ctx)
if err != nil {
if errors.Is(err, fs.ErrPermission) {
return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.FilePath)
}
return diag.FromErr(err)
}

View File

@ -3,8 +3,10 @@ package lock
import (
"context"
"errors"
"io/fs"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/locker"
@ -51,12 +53,17 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
if err != nil {
log.Errorf(ctx, "Failed to acquire deployment lock: %v", err)
if errors.Is(err, fs.ErrPermission) {
return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath)
}
notExistsError := filer.NoSuchDirectoryError{}
if errors.As(err, &notExistsError) {
// If we get a "doesn't exist" error from the API this indicates
// we either don't have permissions or the path is invalid.
return diag.Errorf("cannot write to deployment root (this can indicate a previous deploy was done with a different identity): %s", b.Config.Workspace.RootPath)
return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath)
}
return diag.FromErr(err)
}

View File

@ -4,6 +4,7 @@ import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
"github.com/hashicorp/terraform-exec/tfexec"
@ -34,6 +35,10 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// Apply terraform according to the computed plan
err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path))
if err != nil {
diags := permissions.TryExtendTerraformPermissionError(ctx, b, err)
if diags != nil {
return diags
}
return diag.Errorf("terraform apply: %v", err)
}

View File

@ -40,7 +40,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value,
// Configure permissions for this resource.
if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key)
permissions.ClusterId = fmt.Sprintf("${databricks_cluster.%s.id}", key)
out.Permissions["cluster_"+key] = permissions
}

View File

@ -81,7 +81,7 @@ func TestConvertCluster(t *testing.T) {
// Assert equality on the permissions
assert.Equal(t, &schema.ResourcePermissions{
JobId: "${databricks_cluster.my_cluster.id}",
ClusterId: "${databricks_cluster.my_cluster.id}",
AccessControl: []schema.ResourcePermissionsAccessControl{
{
PermissionLevel: "CAN_RUN",

View File

@ -8,8 +8,10 @@ import (
"reflect"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/config/variable"
"github.com/databricks/cli/libs/jsonschema"
"github.com/databricks/databricks-sdk-go/service/jobs"
)
func interpolationPattern(s string) string {
@ -66,6 +68,31 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.
}
}
func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
switch typ {
case reflect.TypeOf(resources.Job{}):
// This field has been deprecated in jobs API v2.1 and is always set to
// "MULTI_TASK" in the backend. We should not expose it to the user.
delete(s.Properties, "format")
// These fields are only meant to be set by the DABs client (ie the CLI)
// and thus should not be exposed to the user. These are used to annotate
// jobs that were created by DABs.
delete(s.Properties, "deployment")
delete(s.Properties, "edit_mode")
case reflect.TypeOf(jobs.GitSource{}):
// These fields are readonly and are not meant to be set by the user.
delete(s.Properties, "job_source")
delete(s.Properties, "git_snapshot")
default:
// Do nothing
}
return s
}
func main() {
if len(os.Args) != 2 {
fmt.Println("Usage: go run main.go <output-file>")
@ -90,6 +117,7 @@ func main() {
s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
p.addDescriptions,
p.addEnums,
removeJobsFields,
addInterpolationPatterns,
})
if err != nil {

View File

@ -0,0 +1,4 @@
resources:
jobs:
foo:
format: SINGLE_TASK

View File

@ -0,0 +1,6 @@
resources:
jobs:
foo:
deployment:
kind: BUNDLE
metadata_file_path: /a/b/c

View File

@ -0,0 +1,6 @@
targets:
foo:
resources:
jobs:
bar:
edit_mode: whatever

View File

@ -0,0 +1,8 @@
resources:
jobs:
foo:
git_source:
git_provider: GITHUB
git_url: www.whatever.com
git_snapshot:
used_commit: abcdef

View File

@ -0,0 +1,9 @@
resources:
jobs:
foo:
git_source:
git_provider: GITHUB
git_url: www.whatever.com
job_source:
import_from_git_branch: master
job_config_path: def

View File

@ -32,7 +32,6 @@ resources:
name: myjob
continuous:
pause_status: PAUSED
edit_mode: EDITABLE
max_concurrent_runs: 10
description: "my job description"
email_notifications:
@ -43,10 +42,12 @@ resources:
dependencies:
- python=3.7
client: "myclient"
format: MULTI_TASK
tags:
foo: bar
bar: baz
git_source:
git_provider: gitHub
git_url: www.github.com/a/b
tasks:
- task_key: mytask
notebook_task:

View File

@ -1,3 +1,3 @@
package schema
const ProviderVersion = "1.52.0"
const ProviderVersion = "1.53.0"

View File

@ -10,6 +10,7 @@ type DataSourceCurrentMetastoreMetastoreInfo struct {
DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"`
DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"`
DeltaSharingScope string `json:"delta_sharing_scope,omitempty"`
ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"`
GlobalMetastoreId string `json:"global_metastore_id,omitempty"`
MetastoreId string `json:"metastore_id,omitempty"`
Name string `json:"name,omitempty"`

View File

@ -10,6 +10,7 @@ type DataSourceMetastoreMetastoreInfo struct {
DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"`
DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"`
DeltaSharingScope string `json:"delta_sharing_scope,omitempty"`
ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"`
GlobalMetastoreId string `json:"global_metastore_id,omitempty"`
MetastoreId string `json:"metastore_id,omitempty"`
Name string `json:"name,omitempty"`

View File

@ -0,0 +1,8 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type DataSourceMlflowModels struct {
Id string `json:"id,omitempty"`
Names []string `json:"names,omitempty"`
}

View File

@ -30,6 +30,7 @@ type DataSources struct {
Metastores map[string]any `json:"databricks_metastores,omitempty"`
MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"`
MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"`
MlflowModels map[string]any `json:"databricks_mlflow_models,omitempty"`
MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"`
MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"`
NodeType map[string]any `json:"databricks_node_type,omitempty"`
@ -85,6 +86,7 @@ func NewDataSources() *DataSources {
Metastores: make(map[string]any),
MlflowExperiment: make(map[string]any),
MlflowModel: make(map[string]any),
MlflowModels: make(map[string]any),
MwsCredentials: make(map[string]any),
MwsWorkspaces: make(map[string]any),
NodeType: make(map[string]any),

View File

@ -0,0 +1,49 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceBudgetAlertConfigurationsActionConfigurations struct {
ActionConfigurationId string `json:"action_configuration_id,omitempty"`
ActionType string `json:"action_type,omitempty"`
Target string `json:"target,omitempty"`
}
type ResourceBudgetAlertConfigurations struct {
AlertConfigurationId string `json:"alert_configuration_id,omitempty"`
QuantityThreshold string `json:"quantity_threshold,omitempty"`
QuantityType string `json:"quantity_type,omitempty"`
TimePeriod string `json:"time_period,omitempty"`
TriggerType string `json:"trigger_type,omitempty"`
ActionConfigurations []ResourceBudgetAlertConfigurationsActionConfigurations `json:"action_configurations,omitempty"`
}
type ResourceBudgetFilterTagsValue struct {
Operator string `json:"operator,omitempty"`
Values []string `json:"values,omitempty"`
}
type ResourceBudgetFilterTags struct {
Key string `json:"key,omitempty"`
Value *ResourceBudgetFilterTagsValue `json:"value,omitempty"`
}
type ResourceBudgetFilterWorkspaceId struct {
Operator string `json:"operator,omitempty"`
Values []int `json:"values,omitempty"`
}
type ResourceBudgetFilter struct {
Tags []ResourceBudgetFilterTags `json:"tags,omitempty"`
WorkspaceId *ResourceBudgetFilterWorkspaceId `json:"workspace_id,omitempty"`
}
type ResourceBudget struct {
AccountId string `json:"account_id,omitempty"`
BudgetConfigurationId string `json:"budget_configuration_id,omitempty"`
CreateTime int `json:"create_time,omitempty"`
DisplayName string `json:"display_name,omitempty"`
Id string `json:"id,omitempty"`
UpdateTime int `json:"update_time,omitempty"`
AlertConfigurations []ResourceBudgetAlertConfigurations `json:"alert_configurations,omitempty"`
Filter *ResourceBudgetFilter `json:"filter,omitempty"`
}

View File

@ -2,6 +2,57 @@
package schema
type ResourceModelServingAiGatewayGuardrailsInputPii struct {
Behavior string `json:"behavior"`
}
type ResourceModelServingAiGatewayGuardrailsInput struct {
InvalidKeywords []string `json:"invalid_keywords,omitempty"`
Safety bool `json:"safety,omitempty"`
ValidTopics []string `json:"valid_topics,omitempty"`
Pii *ResourceModelServingAiGatewayGuardrailsInputPii `json:"pii,omitempty"`
}
type ResourceModelServingAiGatewayGuardrailsOutputPii struct {
Behavior string `json:"behavior"`
}
type ResourceModelServingAiGatewayGuardrailsOutput struct {
InvalidKeywords []string `json:"invalid_keywords,omitempty"`
Safety bool `json:"safety,omitempty"`
ValidTopics []string `json:"valid_topics,omitempty"`
Pii *ResourceModelServingAiGatewayGuardrailsOutputPii `json:"pii,omitempty"`
}
type ResourceModelServingAiGatewayGuardrails struct {
Input *ResourceModelServingAiGatewayGuardrailsInput `json:"input,omitempty"`
Output *ResourceModelServingAiGatewayGuardrailsOutput `json:"output,omitempty"`
}
type ResourceModelServingAiGatewayInferenceTableConfig struct {
CatalogName string `json:"catalog_name,omitempty"`
Enabled bool `json:"enabled,omitempty"`
SchemaName string `json:"schema_name,omitempty"`
TableNamePrefix string `json:"table_name_prefix,omitempty"`
}
type ResourceModelServingAiGatewayRateLimits struct {
Calls int `json:"calls"`
Key string `json:"key,omitempty"`
RenewalPeriod string `json:"renewal_period"`
}
type ResourceModelServingAiGatewayUsageTrackingConfig struct {
Enabled bool `json:"enabled,omitempty"`
}
type ResourceModelServingAiGateway struct {
Guardrails *ResourceModelServingAiGatewayGuardrails `json:"guardrails,omitempty"`
InferenceTableConfig *ResourceModelServingAiGatewayInferenceTableConfig `json:"inference_table_config,omitempty"`
RateLimits []ResourceModelServingAiGatewayRateLimits `json:"rate_limits,omitempty"`
UsageTrackingConfig *ResourceModelServingAiGatewayUsageTrackingConfig `json:"usage_tracking_config,omitempty"`
}
type ResourceModelServingConfigAutoCaptureConfig struct {
CatalogName string `json:"catalog_name,omitempty"`
Enabled bool `json:"enabled,omitempty"`
@ -139,6 +190,7 @@ type ResourceModelServing struct {
Name string `json:"name"`
RouteOptimized bool `json:"route_optimized,omitempty"`
ServingEndpointId string `json:"serving_endpoint_id,omitempty"`
AiGateway *ResourceModelServingAiGateway `json:"ai_gateway,omitempty"`
Config *ResourceModelServingConfig `json:"config,omitempty"`
RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"`
Tags []ResourceModelServingTags `json:"tags,omitempty"`

View File

@ -4,7 +4,7 @@ package schema
type ResourcePermissionsAccessControl struct {
GroupName string `json:"group_name,omitempty"`
PermissionLevel string `json:"permission_level"`
PermissionLevel string `json:"permission_level,omitempty"`
ServicePrincipalName string `json:"service_principal_name,omitempty"`
UserName string `json:"user_name,omitempty"`
}

View File

@ -238,6 +238,7 @@ type ResourcePipelineTrigger struct {
type ResourcePipeline struct {
AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"`
BudgetPolicyId string `json:"budget_policy_id,omitempty"`
Catalog string `json:"catalog,omitempty"`
Cause string `json:"cause,omitempty"`
Channel string `json:"channel,omitempty"`
@ -254,6 +255,7 @@ type ResourcePipeline struct {
Name string `json:"name,omitempty"`
Photon bool `json:"photon,omitempty"`
RunAsUserName string `json:"run_as_user_name,omitempty"`
Schema string `json:"schema,omitempty"`
Serverless bool `json:"serverless,omitempty"`
State string `json:"state,omitempty"`
Storage string `json:"storage,omitempty"`

View File

@ -4,9 +4,11 @@ package schema
type ResourceSqlTableColumn struct {
Comment string `json:"comment,omitempty"`
Identity string `json:"identity,omitempty"`
Name string `json:"name"`
Nullable bool `json:"nullable,omitempty"`
Type string `json:"type,omitempty"`
TypeJson string `json:"type_json,omitempty"`
}
type ResourceSqlTable struct {

View File

@ -10,6 +10,7 @@ type Resources struct {
AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"`
Budget map[string]any `json:"databricks_budget,omitempty"`
Catalog map[string]any `json:"databricks_catalog,omitempty"`
CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
Cluster map[string]any `json:"databricks_cluster,omitempty"`
@ -112,6 +113,7 @@ func NewResources() *Resources {
AzureAdlsGen1Mount: make(map[string]any),
AzureAdlsGen2Mount: make(map[string]any),
AzureBlobMount: make(map[string]any),
Budget: make(map[string]any),
Catalog: make(map[string]any),
CatalogWorkspaceBinding: make(map[string]any),
Cluster: make(map[string]any),

View File

@ -21,7 +21,7 @@ type Root struct {
const ProviderHost = "registry.terraform.io"
const ProviderSource = "databricks/databricks"
const ProviderVersion = "1.52.0"
const ProviderVersion = "1.53.0"
func NewRoot() *Root {
return &Root{

View File

@ -0,0 +1,110 @@
package permissions
import (
"context"
"fmt"
"sort"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/set"
)
type permissionDiagnostics struct{}
func PermissionDiagnostics() bundle.Mutator {
return &permissionDiagnostics{}
}
func (m *permissionDiagnostics) Name() string {
return "CheckPermissions"
}
func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
if len(b.Config.Permissions) == 0 {
// Only warn if there is an explicit top-level permissions section
return nil
}
canManageBundle, _ := analyzeBundlePermissions(b)
if canManageBundle {
return nil
}
return diag.Diagnostics{{
Severity: diag.Warning,
Summary: fmt.Sprintf("permissions section should include %s or one of their groups with CAN_MANAGE permissions", b.Config.Workspace.CurrentUser.UserName),
Locations: []dyn.Location{b.Config.GetLocation("permissions")},
ID: diag.PermissionNotIncluded,
}}
}
// analyzeBundlePermissions analyzes the top-level permissions of the bundle.
// This permission set is important since it determines the permissions of the
// target workspace folder.
//
// Returns:
// - isManager: true if the current user is can manage the bundle resources.
// - assistance: advice on who to contact as to manage this project
func analyzeBundlePermissions(b *bundle.Bundle) (bool, string) {
canManageBundle := false
otherManagers := set.NewSet[string]()
if b.Config.RunAs != nil && b.Config.RunAs.UserName != "" && b.Config.RunAs.UserName != b.Config.Workspace.CurrentUser.UserName {
// The run_as user is another human that could be contacted
// about this bundle.
otherManagers.Add(b.Config.RunAs.UserName)
}
currentUser := b.Config.Workspace.CurrentUser.UserName
targetPermissions := b.Config.Permissions
for _, p := range targetPermissions {
if p.Level != CAN_MANAGE {
continue
}
if p.UserName == currentUser || p.ServicePrincipalName == currentUser {
canManageBundle = true
continue
}
if isGroupOfCurrentUser(b, p.GroupName) {
canManageBundle = true
continue
}
// Permission doesn't apply to current user; add to otherManagers
otherManager := p.UserName
if otherManager == "" {
otherManager = p.GroupName
}
if otherManager == "" {
// Skip service principals
continue
}
otherManagers.Add(otherManager)
}
assistance := "For assistance, contact the owners of this project."
if otherManagers.Size() > 0 {
list := otherManagers.Values()
sort.Strings(list)
assistance = fmt.Sprintf(
"For assistance, users or groups with appropriate permissions may include: %s.",
strings.Join(list, ", "),
)
}
return canManageBundle, assistance
}
func isGroupOfCurrentUser(b *bundle.Bundle, groupName string) bool {
currentUserGroups := b.Config.Workspace.CurrentUser.User.Groups
for _, g := range currentUserGroups {
if g.Display == groupName {
return true
}
}
return false
}

View File

@ -0,0 +1,52 @@
package permissions_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/require"
)
func TestPermissionDiagnosticsApplySuccess(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", UserName: "testuser@databricks.com"},
})
diags := permissions.PermissionDiagnostics().Apply(context.Background(), b)
require.NoError(t, diags.Error())
}
func TestPermissionDiagnosticsApplyFail(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_VIEW", UserName: "testuser@databricks.com"},
})
diags := permissions.PermissionDiagnostics().Apply(context.Background(), b)
require.Equal(t, diags[0].Severity, diag.Warning)
require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions")
}
func mockBundle(permissions []resources.Permission) *bundle.Bundle {
return &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
CurrentUser: &config.User{
User: &iam.User{
UserName: "testuser@databricks.com",
DisplayName: "Test User",
Groups: []iam.ComplexValue{
{Display: "testgroup"},
},
},
},
},
Permissions: permissions,
},
}
}

View File

@ -0,0 +1,52 @@
package permissions
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/cli/libs/log"
)
// ReportPossiblePermissionDenied generates a diagnostic message when a permission denied error is encountered.
//
// Note that since the workspace API doesn't always distinguish between permission denied and path errors,
// we must treat this as a "possible permission error". See acquire.go for more about this.
func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) diag.Diagnostics {
log.Errorf(ctx, "Failed to update, encountered possible permission error: %v", path)
me := b.Config.Workspace.CurrentUser.User
userName := me.UserName
if iamutil.IsServicePrincipal(me) {
userName = me.DisplayName
}
canManageBundle, assistance := analyzeBundlePermissions(b)
if !canManageBundle {
return diag.Diagnostics{{
Summary: fmt.Sprintf("unable to deploy to %s as %s.\n"+
"Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n"+
"%s\n"+
"They may need to redeploy the bundle to apply the new permissions.\n"+
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.",
path, userName, assistance),
Severity: diag.Error,
ID: diag.PathPermissionDenied,
}}
}
// According databricks.yml, the current user has the right permissions.
// But we're still seeing permission errors. So someone else will need
// to redeploy the bundle with the right set of permissions.
return diag.Diagnostics{{
Summary: fmt.Sprintf("unable to deploy to %s as %s. Cannot apply local deployment permissions.\n"+
"%s\n"+
"They can redeploy the project to apply the latest set of permissions.\n"+
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.",
path, userName, assistance),
Severity: diag.Error,
ID: diag.CannotChangePathPermissions,
}}
}

View File

@ -0,0 +1,76 @@
package permissions_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/permissions"
"github.com/stretchr/testify/require"
)
func TestPermissionsReportPermissionDeniedWithGroup(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", GroupName: "testgroup"},
})
diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath")
expected := "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n" +
"For assistance, contact the owners of this project.\n" +
"They can redeploy the project to apply the latest set of permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, diags.Error(), expected)
}
func TestPermissionsReportPermissionDeniedWithOtherGroup(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", GroupName: "othergroup"},
})
diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath")
expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" +
"Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" +
"For assistance, users or groups with appropriate permissions may include: othergroup.\n" +
"They may need to redeploy the bundle to apply the new permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, diags.Error(), expected)
}
func TestPermissionsReportPermissionDeniedWithoutPermission(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_VIEW", UserName: "testuser@databricks.com"},
})
diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath")
expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" +
"Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" +
"For assistance, contact the owners of this project.\n" +
"They may need to redeploy the bundle to apply the new permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, diags.Error(), expected)
}
func TestPermissionsReportPermissionDeniedNilPermission(t *testing.T) {
b := mockBundle(nil)
diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath")
expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" +
"Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" +
"For assistance, contact the owners of this project.\n" +
"They may need to redeploy the bundle to apply the new permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions"
require.ErrorContains(t, diags.Error(), expected)
}
func TestPermissionsReportFindOtherOwners(t *testing.T) {
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", GroupName: "testgroup"},
{Level: "CAN_MANAGE", UserName: "alice@databricks.com"},
})
diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath")
require.ErrorContains(t, diags.Error(), "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n"+
"For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n"+
"They can redeploy the project to apply the latest set of permissions.\n"+
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.")
}

View File

@ -0,0 +1,47 @@
package permissions
import (
"context"
"fmt"
"regexp"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/log"
)
func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) diag.Diagnostics {
_, assistance := analyzeBundlePermissions(b)
// In a best-effort attempt to provide actionable error messages, we match
// against a few specific error messages that come from the Jobs and Pipelines API.
// For matching errors we provide a more specific error message that includes
// details on how to resolve the issue.
if !strings.Contains(err.Error(), "cannot update permissions") &&
!strings.Contains(err.Error(), "permissions on pipeline") &&
!strings.Contains(err.Error(), "cannot read permissions") &&
!strings.Contains(err.Error(), "cannot set run_as to user") {
return nil
}
log.Errorf(ctx, "Terraform error during deployment: %v", err.Error())
// Best-effort attempt to extract the resource name from the error message.
re := regexp.MustCompile(`databricks_(\w*)\.(\w*)`)
match := re.FindStringSubmatch(err.Error())
resource := "resource"
if len(match) > 1 {
resource = match[2]
}
return diag.Diagnostics{{
Summary: fmt.Sprintf("permission denied creating or updating %s.\n"+
"%s\n"+
"They can redeploy the project to apply the latest set of permissions.\n"+
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.",
resource, assistance),
Severity: diag.Error,
ID: diag.ResourcePermissionDenied,
}}
}

View File

@ -0,0 +1,97 @@
package permissions_test
import (
"context"
"errors"
"testing"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/require"
)
func TestTryExtendTerraformPermissionError1(t *testing.T) {
ctx := context.Background()
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", UserName: "alice@databricks.com"},
})
err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+
"\n"+
"Error: cannot update permissions: ...\n"+
"\n"+
" with databricks_pipeline.my_project_pipeline,\n"+
" on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+
" 39: }")).Error()
expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" +
"For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n" +
"They can redeploy the project to apply the latest set of permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions"
require.ErrorContains(t, err, expected)
}
func TestTryExtendTerraformPermissionError2(t *testing.T) {
ctx := context.Background()
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", UserName: "alice@databricks.com"},
{Level: "CAN_MANAGE", UserName: "bob@databricks.com"},
})
err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+
"\n"+
"Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+
"\n"+
" with databricks_pipeline.my_project_pipeline,\n"+
" on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+
" 39: }")).Error()
expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" +
"For assistance, users or groups with appropriate permissions may include: alice@databricks.com, bob@databricks.com.\n" +
"They can redeploy the project to apply the latest set of permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, err, expected)
}
func TestTryExtendTerraformPermissionError3(t *testing.T) {
ctx := context.Background()
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", UserName: "testuser@databricks.com"},
})
err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+
"\n"+
"Error: cannot read permissions: 1706906c-c0a2-4c25-9f57-3a7aa3cb8b90 does not have Owner permissions on Job with ID: ElasticJobId(28263044278868). Please contact the owner or an administrator for access.\n"+
"\n"+
" with databricks_pipeline.my_project_pipeline,\n"+
" on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+
" 39: }")).Error()
expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" +
"For assistance, contact the owners of this project.\n" +
"They can redeploy the project to apply the latest set of permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, err, expected)
}
func TestTryExtendTerraformPermissionErrorNotOwner(t *testing.T) {
ctx := context.Background()
b := mockBundle([]resources.Permission{
{Level: "CAN_MANAGE", GroupName: "data_team@databricks.com"},
})
b.Config.RunAs = &jobs.JobRunAs{
UserName: "testuser@databricks.com",
}
err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+
"\n"+
"Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+
"\n"+
" with databricks_pipeline.my_project_pipeline,\n"+
" on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+
" 39: }")).Error()
expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" +
"For assistance, users or groups with appropriate permissions may include: data_team@databricks.com.\n" +
"They can redeploy the project to apply the latest set of permissions.\n" +
"Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions."
require.ErrorContains(t, err, expected)
}

View File

@ -62,6 +62,8 @@ func Initialize() bundle.Mutator {
"workspace",
"variables",
),
// Provide permission config errors & warnings after initializing all variables
permissions.PermissionDiagnostics(),
mutator.SetRunAs(),
mutator.OverrideCompute(),
mutator.ProcessTargetMode(),

View File

@ -58,6 +58,20 @@ const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }}
`
const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }}
{{- range $index, $element := .Paths }}
{{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }}
{{- end }}
{{- range $index, $element := .Locations }}
{{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }}
{{- end }}
{{- if .Detail }}
{{ .Detail }}
{{- end }}
`
const summaryHeaderTemplate = `{{- if .Name -}}
Name: {{ .Name | bold }}
{{- if .Target }}
@ -114,9 +128,20 @@ func buildTrailer(diags diag.Diagnostics) string {
if warnings := len(diags.Filter(diag.Warning)); warnings > 0 {
parts = append(parts, color.YellowString(pluralize(warnings, "warning", "warnings")))
}
if len(parts) > 0 {
return fmt.Sprintf("Found %s\n", strings.Join(parts, " and "))
} else {
if recommendations := len(diags.Filter(diag.Recommendation)); recommendations > 0 {
parts = append(parts, color.BlueString(pluralize(recommendations, "recommendation", "recommendations")))
}
switch {
case len(parts) >= 3:
first := strings.Join(parts[:len(parts)-1], ", ")
last := parts[len(parts)-1]
return fmt.Sprintf("Found %s, and %s\n", first, last)
case len(parts) == 2:
return fmt.Sprintf("Found %s and %s\n", parts[0], parts[1])
case len(parts) == 1:
return fmt.Sprintf("Found %s\n", parts[0])
default:
// No diagnostics to print.
return color.GreenString("Validation OK!\n")
}
}
@ -149,6 +174,7 @@ func renderSummaryHeaderTemplate(out io.Writer, b *bundle.Bundle) error {
func renderDiagnosticsOnly(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error {
errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate))
warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate))
recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate))
// Print errors and warnings.
for _, d := range diags {
@ -158,6 +184,8 @@ func renderDiagnosticsOnly(out io.Writer, b *bundle.Bundle, diags diag.Diagnosti
t = errorT
case diag.Warning:
t = warningT
case diag.Recommendation:
t = recommendationT
}
for i := range d.Locations {

View File

@ -50,6 +50,19 @@ func TestRenderTextOutput(t *testing.T) {
"\n" +
"Found 1 error\n",
},
{
name: "nil bundle and 1 recommendation",
diags: diag.Diagnostics{
{
Severity: diag.Recommendation,
Summary: "recommendation",
},
},
opts: RenderOptions{RenderSummaryTable: true},
expected: "Recommendation: recommendation\n" +
"\n" +
"Found 1 recommendation\n",
},
{
name: "bundle during 'load' and 1 error",
bundle: loadingBundle,
@ -89,7 +102,7 @@ func TestRenderTextOutput(t *testing.T) {
"Found 2 warnings\n",
},
{
name: "bundle during 'load' and 2 errors, 1 warning with details",
name: "bundle during 'load' and 2 errors, 1 warning and 1 recommendation with details",
bundle: loadingBundle,
diags: diag.Diagnostics{
diag.Diagnostic{
@ -110,6 +123,12 @@ func TestRenderTextOutput(t *testing.T) {
Detail: "detail (3)",
Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Recommendation,
Summary: "recommendation (4)",
Detail: "detail (4)",
Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}},
},
},
opts: RenderOptions{RenderSummaryTable: true},
expected: "Error: error (1)\n" +
@ -127,10 +146,114 @@ func TestRenderTextOutput(t *testing.T) {
"\n" +
"detail (3)\n" +
"\n" +
"Recommendation: recommendation (4)\n" +
" in foo.py:4:1\n" +
"\n" +
"detail (4)\n" +
"\n" +
"Name: test-bundle\n" +
"Target: test-target\n" +
"\n" +
"Found 2 errors and 1 warning\n",
"Found 2 errors, 1 warning, and 1 recommendation\n",
},
{
name: "bundle during 'load' and 1 error and 1 warning",
bundle: loadingBundle,
diags: diag.Diagnostics{
diag.Diagnostic{
Severity: diag.Error,
Summary: "error (1)",
Detail: "detail (1)",
Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Warning,
Summary: "warning (2)",
Detail: "detail (2)",
Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}},
},
},
opts: RenderOptions{RenderSummaryTable: true},
expected: "Error: error (1)\n" +
" in foo.py:1:1\n" +
"\n" +
"detail (1)\n" +
"\n" +
"Warning: warning (2)\n" +
" in foo.py:2:1\n" +
"\n" +
"detail (2)\n" +
"\n" +
"Name: test-bundle\n" +
"Target: test-target\n" +
"\n" +
"Found 1 error and 1 warning\n",
},
{
name: "bundle during 'load' and 1 errors, 2 warning and 2 recommendations with details",
bundle: loadingBundle,
diags: diag.Diagnostics{
diag.Diagnostic{
Severity: diag.Error,
Summary: "error (1)",
Detail: "detail (1)",
Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Warning,
Summary: "warning (2)",
Detail: "detail (2)",
Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Warning,
Summary: "warning (3)",
Detail: "detail (3)",
Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Recommendation,
Summary: "recommendation (4)",
Detail: "detail (4)",
Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Recommendation,
Summary: "recommendation (5)",
Detail: "detail (5)",
Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}},
},
},
opts: RenderOptions{RenderSummaryTable: true},
expected: "Error: error (1)\n" +
" in foo.py:1:1\n" +
"\n" +
"detail (1)\n" +
"\n" +
"Warning: warning (2)\n" +
" in foo.py:2:1\n" +
"\n" +
"detail (2)\n" +
"\n" +
"Warning: warning (3)\n" +
" in foo.py:3:1\n" +
"\n" +
"detail (3)\n" +
"\n" +
"Recommendation: recommendation (4)\n" +
" in foo.py:4:1\n" +
"\n" +
"detail (4)\n" +
"\n" +
"Recommendation: recommendation (5)\n" +
" in foo.py:5:1\n" +
"\n" +
"detail (5)\n" +
"\n" +
"Name: test-bundle\n" +
"Target: test-target\n" +
"\n" +
"Found 1 error, 2 warnings, and 2 recommendations\n",
},
{
name: "bundle during 'init'",
@ -163,7 +286,7 @@ func TestRenderTextOutput(t *testing.T) {
"Validation OK!\n",
},
{
name: "nil bundle without summary with 1 error and 1 warning",
name: "nil bundle without summary with 1 error, 1 warning and 1 recommendation",
bundle: nil,
diags: diag.Diagnostics{
diag.Diagnostic{
@ -178,6 +301,12 @@ func TestRenderTextOutput(t *testing.T) {
Detail: "detail (2)",
Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}},
},
diag.Diagnostic{
Severity: diag.Recommendation,
Summary: "recommendation (3)",
Detail: "detail (3)",
Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}},
},
},
opts: RenderOptions{RenderSummaryTable: false},
expected: "Error: error (1)\n" +
@ -189,6 +318,11 @@ func TestRenderTextOutput(t *testing.T) {
" in foo.py:3:1\n" +
"\n" +
"detail (2)\n" +
"\n" +
"Recommendation: recommendation (3)\n" +
" in foo.py:5:1\n" +
"\n" +
"detail (3)\n" +
"\n",
},
}
@ -309,6 +443,30 @@ func TestRenderDiagnostics(t *testing.T) {
"\n" +
"'name' is required\n\n",
},
{
name: "recommendation with multiple paths and locations",
diags: diag.Diagnostics{
{
Severity: diag.Recommendation,
Summary: "summary",
Detail: "detail",
Paths: []dyn.Path{
dyn.MustPathFromString("resources.jobs.xxx"),
dyn.MustPathFromString("resources.jobs.yyy"),
},
Locations: []dyn.Location{
{File: "foo.yaml", Line: 1, Column: 2},
{File: "bar.yaml", Line: 3, Column: 4},
},
},
},
expected: "Recommendation: summary\n" +
" at resources.jobs.xxx\n" +
" resources.jobs.yyy\n" +
" in foo.yaml:1:2\n" +
" bar.yaml:3:4\n\n" +
"detail\n\n",
},
}
for _, tc := range testCases {

View File

@ -39,7 +39,7 @@ func TestJsonSchema(t *testing.T) {
// Assert job fields have their descriptions loaded.
resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job")
fields := []string{"name", "continuous", "deployment", "tasks", "trigger"}
fields := []string{"name", "continuous", "tasks", "trigger"}
for _, field := range fields {
assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description)
}
@ -53,7 +53,7 @@ func TestJsonSchema(t *testing.T) {
// Assert descriptions are loaded for pipelines
pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline")
fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"}
fields = []string{"name", "catalog", "clusters", "channel", "continuous", "development"}
for _, field := range fields {
assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description)
}

View File

@ -213,18 +213,10 @@
"description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous"
},
"deployment": {
"description": "Deployment information for jobs managed by external sources.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment"
},
"description": {
"description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.",
"$ref": "#/$defs/string"
},
"edit_mode": {
"description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode"
},
"email_notifications": {
"description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications"
@ -233,10 +225,6 @@
"description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.",
"$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment"
},
"format": {
"description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format"
},
"git_source": {
"description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource"
@ -402,6 +390,10 @@
{
"type": "object",
"properties": {
"ai_gateway": {
"description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig"
},
"config": {
"description": "The core config of the serving endpoint.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput"
@ -472,6 +464,10 @@
{
"type": "object",
"properties": {
"budget_policy_id": {
"description": "Budget policy of this pipeline.",
"$ref": "#/$defs/string"
},
"catalog": {
"description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.",
"$ref": "#/$defs/string"
@ -539,6 +535,10 @@
"description": "Whether Photon is enabled for this pipeline.",
"$ref": "#/$defs/bool"
},
"schema": {
"description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.",
"$ref": "#/$defs/string"
},
"serverless": {
"description": "Whether serverless compute is enabled for this pipeline.",
"$ref": "#/$defs/bool"
@ -1206,6 +1206,9 @@
"profile": {
"$ref": "#/$defs/string"
},
"resource_path": {
"$ref": "#/$defs/string"
},
"root_path": {
"$ref": "#/$defs/string"
},
@ -2532,9 +2535,6 @@
"description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider"
},
"git_snapshot": {
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot"
},
"git_tag": {
"description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.",
"$ref": "#/$defs/string"
@ -2542,10 +2542,6 @@
"git_url": {
"description": "URL of the repository to be cloned by this job.",
"$ref": "#/$defs/string"
},
"job_source": {
"description": "The source of the job specification in the remote repository when the job is source controlled.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource"
}
},
"additionalProperties": false,
@ -2632,7 +2628,7 @@
"type": "object",
"properties": {
"no_alert_for_skipped_runs": {
"description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.",
"description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.",
"$ref": "#/$defs/bool"
},
"on_duration_warning_threshold_exceeded": {
@ -3073,6 +3069,7 @@
"$ref": "#/$defs/map/string"
},
"pipeline_params": {
"description": "Controls whether the pipeline should perform a full refresh",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams"
},
"python_named_params": {
@ -3547,7 +3544,7 @@
"type": "object",
"properties": {
"no_alert_for_skipped_runs": {
"description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.",
"description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.",
"$ref": "#/$defs/bool"
},
"on_duration_warning_threshold_exceeded": {
@ -4365,6 +4362,207 @@
}
]
},
"serving.AiGatewayConfig": {
"anyOf": [
{
"type": "object",
"properties": {
"guardrails": {
"description": "Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails"
},
"inference_table_config": {
"description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig"
},
"rate_limits": {
"description": "Configuration for rate limits which can be set to limit endpoint traffic.",
"$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit"
},
"usage_tracking_config": {
"description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig"
}
},
"additionalProperties": false
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayGuardrailParameters": {
"anyOf": [
{
"type": "object",
"properties": {
"invalid_keywords": {
"description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.",
"$ref": "#/$defs/slice/string"
},
"pii": {
"description": "Configuration for guardrail PII filter.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior"
},
"safety": {
"description": "Indicates whether the safety filter is enabled.",
"$ref": "#/$defs/bool"
},
"valid_topics": {
"description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.",
"$ref": "#/$defs/slice/string"
}
},
"additionalProperties": false
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayGuardrailPiiBehavior": {
"anyOf": [
{
"type": "object",
"properties": {
"behavior": {
"description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior",
"enum": [
"NONE",
"BLOCK"
]
}
},
"additionalProperties": false,
"required": [
"behavior"
]
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayGuardrailPiiBehaviorBehavior": {
"type": "string"
},
"serving.AiGatewayGuardrails": {
"anyOf": [
{
"type": "object",
"properties": {
"input": {
"description": "Configuration for input guardrail filters.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters"
},
"output": {
"description": "Configuration for output guardrail filters.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters"
}
},
"additionalProperties": false
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayInferenceTableConfig": {
"anyOf": [
{
"type": "object",
"properties": {
"catalog_name": {
"description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.",
"$ref": "#/$defs/string"
},
"enabled": {
"description": "Indicates whether the inference table is enabled.",
"$ref": "#/$defs/bool"
},
"schema_name": {
"description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.",
"$ref": "#/$defs/string"
},
"table_name_prefix": {
"description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.",
"$ref": "#/$defs/string"
}
},
"additionalProperties": false
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayRateLimit": {
"anyOf": [
{
"type": "object",
"properties": {
"calls": {
"description": "Used to specify how many calls are allowed for a key within the renewal_period.",
"$ref": "#/$defs/int"
},
"key": {
"description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey",
"enum": [
"user",
"endpoint"
]
},
"renewal_period": {
"description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod",
"enum": [
"minute"
]
}
},
"additionalProperties": false,
"required": [
"calls",
"renewal_period"
]
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AiGatewayRateLimitKey": {
"type": "string"
},
"serving.AiGatewayRateLimitRenewalPeriod": {
"type": "string"
},
"serving.AiGatewayUsageTrackingConfig": {
"anyOf": [
{
"type": "object",
"properties": {
"enabled": {
"description": "Whether to enable usage tracking.",
"$ref": "#/$defs/bool"
}
},
"additionalProperties": false
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.AmazonBedrockConfig": {
"anyOf": [
{
@ -5569,6 +5767,20 @@
}
]
},
"serving.AiGatewayRateLimit": {
"anyOf": [
{
"type": "array",
"items": {
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit"
}
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"serving.EndpointTag": {
"anyOf": [
{

View File

@ -3,7 +3,6 @@ package config_tests
import (
"context"
"fmt"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
@ -113,8 +112,9 @@ func TestRunAsErrorForPipelines(t *testing.T) {
diags := bundle.Apply(ctx, b, mutator.SetRunAs())
err := diags.Error()
configPath := filepath.FromSlash("run_as/not_allowed/pipelines/databricks.yml")
assert.EqualError(t, err, fmt.Sprintf("pipelines are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath))
assert.ErrorContains(t, err, "pipelines do not support a setting a run_as user that is different from the owner.\n"+
"Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+
"See https://docs")
}
func TestRunAsNoErrorForPipelines(t *testing.T) {
@ -152,8 +152,9 @@ func TestRunAsErrorForModelServing(t *testing.T) {
diags := bundle.Apply(ctx, b, mutator.SetRunAs())
err := diags.Error()
configPath := filepath.FromSlash("run_as/not_allowed/model_serving/databricks.yml")
assert.EqualError(t, err, fmt.Sprintf("model_serving_endpoints are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath))
assert.ErrorContains(t, err, "model_serving_endpoints do not support a setting a run_as user that is different from the owner.\n"+
"Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+
"See https://docs")
}
func TestRunAsNoErrorForModelServingEndpoints(t *testing.T) {
@ -191,8 +192,7 @@ func TestRunAsErrorWhenBothUserAndSpSpecified(t *testing.T) {
diags := bundle.Apply(ctx, b, mutator.SetRunAs())
err := diags.Error()
configPath := filepath.FromSlash("run_as/not_allowed/both_sp_and_user/databricks.yml")
assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name \"my_service_principal\" is specified at %s:6:27. A user_name \"my_user_name\" is defined at %s:7:14", configPath, configPath))
assert.ErrorContains(t, err, "run_as section cannot specify both user_name and service_principal_name")
}
func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) {
@ -202,19 +202,19 @@ func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) {
}{
{
name: "empty_run_as",
err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:8", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_run_as/databricks.yml")),
err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
},
{
name: "empty_sp",
err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_sp/databricks.yml")),
err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
},
{
name: "empty_user",
err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user/databricks.yml")),
err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
},
{
name: "empty_user_and_sp",
err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user_and_sp/databricks.yml")),
err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified",
},
}
@ -257,8 +257,7 @@ func TestRunAsErrorNeitherUserOrSpSpecifiedAtTargetOverride(t *testing.T) {
diags := bundle.Apply(ctx, b, mutator.SetRunAs())
err := diags.Error()
configPath := filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/override/override.yml")
assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:12", configPath))
assert.EqualError(t, err, "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified")
}
func TestLegacyRunAs(t *testing.T) {

View File

@ -81,6 +81,7 @@ func newCreate() *cobra.Command {
cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`)
// TODO: array: resources
cmd.Use = "create NAME"
cmd.Short = `Create an app.`
@ -910,6 +911,7 @@ func newUpdate() *cobra.Command {
cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`)
// TODO: array: resources
cmd.Use = "update NAME"
cmd.Short = `Update an app.`

View File

@ -53,13 +53,13 @@ func New() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var createOverrides []func(
*cobra.Command,
*workspace.CreateCredentials,
*workspace.CreateCredentialsRequest,
)
func newCreate() *cobra.Command {
cmd := &cobra.Command{}
var createReq workspace.CreateCredentials
var createReq workspace.CreateCredentialsRequest
var createJson flags.JsonFlag
// TODO: short flags
@ -79,8 +79,9 @@ func newCreate() *cobra.Command {
Arguments:
GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers
are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise,
bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.`
are gitHub, bitbucketCloud, gitLab, azureDevOpsServices,
gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and
awsCodeCommit.`
cmd.Annotations = make(map[string]string)
@ -136,13 +137,13 @@ func newCreate() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var deleteOverrides []func(
*cobra.Command,
*workspace.DeleteGitCredentialRequest,
*workspace.DeleteCredentialsRequest,
)
func newDelete() *cobra.Command {
cmd := &cobra.Command{}
var deleteReq workspace.DeleteGitCredentialRequest
var deleteReq workspace.DeleteCredentialsRequest
// TODO: short flags
@ -209,13 +210,13 @@ func newDelete() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var getOverrides []func(
*cobra.Command,
*workspace.GetGitCredentialRequest,
*workspace.GetCredentialsRequest,
)
func newGet() *cobra.Command {
cmd := &cobra.Command{}
var getReq workspace.GetGitCredentialRequest
var getReq workspace.GetCredentialsRequest
// TODO: short flags
@ -322,33 +323,48 @@ func newList() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var updateOverrides []func(
*cobra.Command,
*workspace.UpdateCredentials,
*workspace.UpdateCredentialsRequest,
)
func newUpdate() *cobra.Command {
cmd := &cobra.Command{}
var updateReq workspace.UpdateCredentials
var updateReq workspace.UpdateCredentialsRequest
var updateJson flags.JsonFlag
// TODO: short flags
cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
cmd.Flags().StringVar(&updateReq.GitProvider, "git-provider", updateReq.GitProvider, `Git provider.`)
cmd.Flags().StringVar(&updateReq.GitUsername, "git-username", updateReq.GitUsername, `The username or email provided with your Git provider account, depending on which provider you are using.`)
cmd.Flags().StringVar(&updateReq.PersonalAccessToken, "personal-access-token", updateReq.PersonalAccessToken, `The personal access token used to authenticate to the corresponding Git provider.`)
cmd.Use = "update CREDENTIAL_ID"
cmd.Use = "update CREDENTIAL_ID GIT_PROVIDER"
cmd.Short = `Update a credential.`
cmd.Long = `Update a credential.
Updates the specified Git credential.
Arguments:
CREDENTIAL_ID: The ID for the corresponding credential to access.`
CREDENTIAL_ID: The ID for the corresponding credential to access.
GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers
are gitHub, bitbucketCloud, gitLab, azureDevOpsServices,
gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and
awsCodeCommit.`
cmd.Annotations = make(map[string]string)
cmd.Args = func(cmd *cobra.Command, args []string) error {
if cmd.Flags().Changed("json") {
err := root.ExactArgs(1)(cmd, args)
if err != nil {
return fmt.Errorf("when --json flag is specified, provide only CREDENTIAL_ID as positional arguments. Provide 'git_provider' in your JSON input")
}
return nil
}
check := root.ExactArgs(2)
return check(cmd, args)
}
cmd.PreRunE = root.MustWorkspaceClient
cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
ctx := cmd.Context()
@ -360,27 +376,13 @@ func newUpdate() *cobra.Command {
return err
}
}
if len(args) == 0 {
promptSpinner := cmdio.Spinner(ctx)
promptSpinner <- "No CREDENTIAL_ID argument specified. Loading names for Git Credentials drop-down."
names, err := w.GitCredentials.CredentialInfoGitProviderToCredentialIdMap(ctx)
close(promptSpinner)
if err != nil {
return fmt.Errorf("failed to load names for Git Credentials drop-down. Please manually specify required arguments. Original error: %w", err)
}
id, err := cmdio.Select(ctx, names, "The ID for the corresponding credential to access")
if err != nil {
return err
}
args = append(args, id)
}
if len(args) != 1 {
return fmt.Errorf("expected to have the id for the corresponding credential to access")
}
_, err = fmt.Sscan(args[0], &updateReq.CredentialId)
if err != nil {
return fmt.Errorf("invalid CREDENTIAL_ID: %s", args[0])
}
if !cmd.Flags().Changed("json") {
updateReq.GitProvider = args[1]
}
err = w.GitCredentials.Update(ctx, updateReq)
if err != nil {

View File

@ -954,6 +954,7 @@ func newUpdate() *cobra.Command {
// TODO: array: notifications
cmd.Flags().BoolVar(&updateReq.Photon, "photon", updateReq.Photon, `Whether Photon is enabled for this pipeline.`)
cmd.Flags().StringVar(&updateReq.PipelineId, "pipeline-id", updateReq.PipelineId, `Unique identifier for this pipeline.`)
cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`)
cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`)
cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`)
cmd.Flags().StringVar(&updateReq.Target, "target", updateReq.Target, `Target schema (database) to add tables in this pipeline to.`)

View File

@ -19,7 +19,7 @@ func listOverride(listCmd *cobra.Command, listReq *workspace.ListReposRequest) {
{{end}}`)
}
func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepo) {
func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepoRequest) {
createCmd.Use = "create URL [PROVIDER]"
createCmd.Args = func(cmd *cobra.Command, args []string) error {
// If the provider argument is not specified, we try to detect it from the URL.
@ -95,7 +95,7 @@ func getOverride(getCmd *cobra.Command, getReq *workspace.GetRepoRequest) {
}
}
func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepo) {
func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepoRequest) {
updateCmd.Use = "update REPO_ID_OR_PATH"
updateJson := updateCmd.Flag("json").Value.(*flags.JsonFlag)

View File

@ -61,13 +61,13 @@ func New() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var createOverrides []func(
*cobra.Command,
*workspace.CreateRepo,
*workspace.CreateRepoRequest,
)
func newCreate() *cobra.Command {
cmd := &cobra.Command{}
var createReq workspace.CreateRepo
var createReq workspace.CreateRepoRequest
var createJson flags.JsonFlag
// TODO: short flags
@ -87,8 +87,9 @@ func newCreate() *cobra.Command {
Arguments:
URL: URL of the Git repository to be linked.
PROVIDER: Git provider. This field is case-insensitive. The available Git providers
are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise,
bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.`
are gitHub, bitbucketCloud, gitLab, azureDevOpsServices,
gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and
awsCodeCommit.`
cmd.Annotations = make(map[string]string)
@ -164,7 +165,7 @@ func newDelete() *cobra.Command {
Deletes the specified repo.
Arguments:
REPO_ID: The ID for the corresponding repo to access.`
REPO_ID: ID of the Git folder (repo) object in the workspace.`
cmd.Annotations = make(map[string]string)
@ -181,14 +182,14 @@ func newDelete() *cobra.Command {
if err != nil {
return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err)
}
id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access")
id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace")
if err != nil {
return err
}
args = append(args, id)
}
if len(args) != 1 {
return fmt.Errorf("expected to have the id for the corresponding repo to access")
return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace")
}
_, err = fmt.Sscan(args[0], &deleteReq.RepoId)
if err != nil {
@ -237,7 +238,7 @@ func newGet() *cobra.Command {
Returns the repo with the given repo ID.
Arguments:
REPO_ID: The ID for the corresponding repo to access.`
REPO_ID: ID of the Git folder (repo) object in the workspace.`
cmd.Annotations = make(map[string]string)
@ -254,14 +255,14 @@ func newGet() *cobra.Command {
if err != nil {
return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err)
}
id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access")
id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace")
if err != nil {
return err
}
args = append(args, id)
}
if len(args) != 1 {
return fmt.Errorf("expected to have the id for the corresponding repo to access")
return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace")
}
_, err = fmt.Sscan(args[0], &getReq.RepoId)
if err != nil {
@ -451,8 +452,8 @@ func newList() *cobra.Command {
cmd.Short = `Get repos.`
cmd.Long = `Get repos.
Returns repos that the calling user has Manage permissions on. Results are
paginated with each page containing twenty repos.`
Returns repos that the calling user has Manage permissions on. Use
next_page_token to iterate through additional pages.`
cmd.Annotations = make(map[string]string)
@ -569,13 +570,13 @@ func newSetPermissions() *cobra.Command {
// Functions can be added from the `init()` function in manually curated files in this directory.
var updateOverrides []func(
*cobra.Command,
*workspace.UpdateRepo,
*workspace.UpdateRepoRequest,
)
func newUpdate() *cobra.Command {
cmd := &cobra.Command{}
var updateReq workspace.UpdateRepo
var updateReq workspace.UpdateRepoRequest
var updateJson flags.JsonFlag
// TODO: short flags
@ -593,7 +594,7 @@ func newUpdate() *cobra.Command {
latest commit on the same branch.
Arguments:
REPO_ID: The ID for the corresponding repo to access.`
REPO_ID: ID of the Git folder (repo) object in the workspace.`
cmd.Annotations = make(map[string]string)
@ -616,14 +617,14 @@ func newUpdate() *cobra.Command {
if err != nil {
return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err)
}
id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access")
id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace")
if err != nil {
return err
}
args = append(args, id)
}
if len(args) != 1 {
return fmt.Errorf("expected to have the id for the corresponding repo to access")
return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace")
}
_, err = fmt.Sscan(args[0], &updateReq.RepoId)
if err != nil {

8
go.mod
View File

@ -7,7 +7,7 @@ toolchain go1.22.7
require (
github.com/Masterminds/semver/v3 v3.3.0 // MIT
github.com/briandowns/spinner v1.23.1 // Apache 2.0
github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0
github.com/databricks/databricks-sdk-go v0.48.0 // Apache 2.0
github.com/fatih/color v1.17.0 // MIT
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/google/uuid v1.6.0 // BSD-3-Clause
@ -27,8 +27,8 @@ require (
golang.org/x/mod v0.21.0
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.8.0
golang.org/x/term v0.24.0
golang.org/x/text v0.18.0
golang.org/x/term v0.25.0
golang.org/x/text v0.19.0
gopkg.in/ini.v1 v1.67.0 // Apache 2.0
gopkg.in/yaml.v3 v3.0.1
)
@ -64,7 +64,7 @@ require (
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/api v0.182.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect

16
go.sum generated
View File

@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ=
github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
github.com/databricks/databricks-sdk-go v0.48.0 h1:46KtsnRo+FGhC3izUXbpL0PXBNomvsdignYDhJZlm9s=
github.com/databricks/databricks-sdk-go v0.48.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -212,14 +212,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

View File

@ -519,7 +519,7 @@ func TemporaryRepo(t *testing.T, w *databricks.WorkspaceClient) string {
repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("integration-test-repo-"))
t.Logf("Creating repo:%s", repoPath)
repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{
repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{
Url: "https://github.com/databricks/cli",
Provider: "github",
Path: repoPath,

View File

@ -12,7 +12,7 @@ import (
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/databricks-sdk-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -126,7 +126,7 @@ func TestAccBundleInitHelpers(t *testing.T) {
}{
{
funcName: "{{short_name}}",
expected: auth.GetShortUserName(me),
expected: iamutil.GetShortUserName(me),
},
{
funcName: "{{user_name}}",
@ -138,7 +138,7 @@ func TestAccBundleInitHelpers(t *testing.T) {
},
{
funcName: "{{is_service_principal}}",
expected: strconv.FormatBool(auth.IsServicePrincipal(me.UserName)),
expected: strconv.FormatBool(iamutil.IsServicePrincipal(me)),
},
{
funcName: "{{smallest_node_type}}",

View File

@ -29,7 +29,7 @@ func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databr
assert.NoError(t, err)
remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix))
repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{
repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{
Path: remoteProjectRoot,
Url: EmptyRepoUrl,
Provider: "gitHub",

View File

@ -34,7 +34,7 @@ func synthesizeTemporaryRepoPath(t *testing.T, w *databricks.WorkspaceClient, ct
func createTemporaryRepo(t *testing.T, w *databricks.WorkspaceClient, ctx context.Context) (int64, string) {
repoPath := synthesizeTemporaryRepoPath(t, w, ctx)
repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{
repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{
Path: repoPath,
Url: repoUrl,
Provider: "gitHub",

View File

@ -38,7 +38,7 @@ func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Contex
require.NoError(t, err)
repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-"))
repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{
repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{
Path: repoPath,
Url: repoUrl,
Provider: "gitHub",

View File

@ -1,6 +1,7 @@
package diag
import (
"errors"
"fmt"
"github.com/databricks/cli/libs/dyn"
@ -24,6 +25,9 @@ type Diagnostic struct {
// Paths are paths to the values in the configuration tree that the diagnostic is associated with.
// It may be nil if there are no associated paths.
Paths []dyn.Path
// A diagnostic ID. Only used for select diagnostic messages.
ID ID
}
// Errorf creates a new error diagnostic.
@ -69,7 +73,7 @@ func Infof(format string, args ...any) Diagnostics {
}
}
// Diagsnostics holds zero or more instances of [Diagnostic].
// Diagnostics holds zero or more instances of [Diagnostic].
type Diagnostics []Diagnostic
// Append adds a new diagnostic to the end of the list.
@ -96,7 +100,14 @@ func (ds Diagnostics) HasError() bool {
func (ds Diagnostics) Error() error {
for _, d := range ds {
if d.Severity == Error {
return fmt.Errorf(d.Summary)
message := d.Detail
if message == "" {
message = d.Summary
}
if d.ID != "" {
message = string(d.ID) + ": " + message
}
return errors.New(message)
}
}
return nil

16
libs/diag/id.go Normal file
View File

@ -0,0 +1,16 @@
package diag
type ID string
// For select diagnostic messages we use IDs to identify them
// for support or tooling purposes.
// It is a non-goal to have an exhaustive list of IDs.
const (
// We have many subtly different permission errors.
// These are numbered for easy reference and tooling support.
PathPermissionDenied ID = "EPERM1"
ResourcePermissionDenied ID = "EPERM2"
CannotChangePathPermissions ID = "EPERM3"
RunAsDenied ID = "EPERM4"
PermissionNotIncluded ID = "EPERM5"
)

View File

@ -6,4 +6,5 @@ const (
Error Severity = iota
Warning
Info
Recommendation
)

View File

@ -103,6 +103,18 @@ func (err CannotDeleteRootError) Is(other error) bool {
return other == fs.ErrInvalid
}
type PermissionError struct {
path string
}
func (err PermissionError) Error() string {
return fmt.Sprintf("access denied: %s", err.path)
}
func (err PermissionError) Is(other error) bool {
return other == fs.ErrPermission
}
// Filer is used to access files in a workspace.
// It has implementations for accessing files in WSFS and in DBFS.
type Filer interface {

View File

@ -178,6 +178,9 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io
// Create parent directory.
err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath))
if err != nil {
if errors.As(err, &aerr) && aerr.StatusCode == http.StatusForbidden {
return PermissionError{absPath}
}
return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err)
}
@ -203,6 +206,11 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io
return FileAlreadyExistsError{absPath}
}
// This API returns StatusForbidden when you have read access but don't have write access to a file
if aerr.StatusCode == http.StatusForbidden {
return PermissionError{absPath}
}
return err
}
@ -295,11 +303,11 @@ func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D
return nil, err
}
// This API returns a 404 if the specified path does not exist.
// NOTE: This API returns a 404 if the specified path does not exist,
// but can also do so if we don't have read access.
if aerr.StatusCode == http.StatusNotFound {
return nil, NoSuchDirectoryError{path.Dir(absPath)}
}
return nil, err
}

View File

@ -1,15 +1,16 @@
package auth
package iamutil
import (
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/google/uuid"
)
// Determines whether a given user name is a service principal.
// Determines whether a given user is a service principal.
// This function uses a heuristic: if the user name is a UUID, then we assume
// it's a service principal. Unfortunately, the service principal listing API is too
// slow for our purposes. And the "users" and "service principals get" APIs
// only allow access by workspace admins.
func IsServicePrincipal(userName string) bool {
_, err := uuid.Parse(userName)
func IsServicePrincipal(user *iam.User) bool {
_, err := uuid.Parse(user.UserName)
return err == nil
}

View File

@ -1,19 +1,24 @@
package auth
package iamutil
import (
"testing"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/assert"
)
func TestIsServicePrincipal_ValidUUID(t *testing.T) {
userId := "8b948b2e-d2b5-4b9e-8274-11b596f3b652"
isSP := IsServicePrincipal(userId)
user := &iam.User{
UserName: "8b948b2e-d2b5-4b9e-8274-11b596f3b652",
}
isSP := IsServicePrincipal(user)
assert.True(t, isSP, "Expected user ID to be recognized as a service principal")
}
func TestIsServicePrincipal_InvalidUUID(t *testing.T) {
userId := "invalid"
isSP := IsServicePrincipal(userId)
user := &iam.User{
UserName: "invalid",
}
isSP := IsServicePrincipal(user)
assert.False(t, isSP, "Expected user ID to not be recognized as a service principal")
}

View File

@ -1,4 +1,4 @@
package auth
package iamutil
import (
"strings"
@ -12,7 +12,7 @@ import (
// including dots, which are not supported in e.g. experiment names.
func GetShortUserName(user *iam.User) string {
name := user.UserName
if IsServicePrincipal(user.UserName) && user.DisplayName != "" {
if IsServicePrincipal(user) && user.DisplayName != "" {
name = user.DisplayName
}
local, _, _ := strings.Cut(name, "@")

View File

@ -1,4 +1,4 @@
package auth
package iamutil
import (
"testing"

View File

@ -14,6 +14,11 @@ type Set[T any] struct {
data map[string]T
}
// Values returns a slice of the set's values
func (s *Set[T]) Values() []T {
return maps.Values(s.data)
}
// NewSetFromF initialise a new set with initial values and a hash function
// to define uniqueness of value
func NewSetFromF[T any](values []T, f hashFunc[T]) *Set[T] {
@ -69,6 +74,11 @@ func (s *Set[T]) Has(item T) bool {
return ok
}
// Size returns the number of elements in the set
func (s *Set[T]) Size() int {
return len(s.data)
}
// Returns an iterable slice of values from set
func (s *Set[T]) Iter() []T {
return maps.Values(s.data)

View File

@ -11,7 +11,7 @@ import (
"text/template"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/iamutil"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/iam"
@ -119,7 +119,7 @@ func loadHelpers(ctx context.Context) template.FuncMap {
return "", err
}
}
return auth.GetShortUserName(cachedUser), nil
return iamutil.GetShortUserName(cachedUser), nil
},
// Get the default workspace catalog. If there is no default, or if
// Unity Catalog is not enabled, return an empty string.
@ -128,8 +128,8 @@ func loadHelpers(ctx context.Context) template.FuncMap {
metastore, err := w.Metastores.Current(ctx)
if err != nil {
var aerr *apierr.APIError
if errors.As(err, &aerr) && aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" {
// Workspace doesn't have a metastore assigned, ignore error
if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST") {
// Ignore: access denied or workspace doesn't have a metastore assigned
empty_default := ""
cachedCatalog = &empty_default
return "", nil
@ -151,7 +151,7 @@ func loadHelpers(ctx context.Context) template.FuncMap {
return false, err
}
}
result := auth.IsServicePrincipal(cachedUser.UserName)
result := iamutil.IsServicePrincipal(cachedUser)
cachedIsServicePrincipal = &result
return result, nil
},