Compare commits

...

11 Commits

Author SHA1 Message Date
Shreyas Goenka d460bd6859
fix 2024-12-02 16:29:50 +01:00
Shreyas Goenka 8d790efcf9
address comments 2024-12-02 16:27:23 +01:00
Shreyas Goenka 8e25bb4a47
Merge remote-tracking branch 'origin' into feature/uc-volumes 2024-12-02 14:13:08 +01:00
Shreyas Goenka 3461018e8a
better error message 2024-12-02 14:01:10 +01:00
Shreyas Goenka 406c0736ff
use apierr.ErrNotFound 2024-12-02 11:48:04 +01:00
Shreyas Goenka 7d544f4af4
lowercase volumes 2024-12-02 11:46:55 +01:00
Shreyas Goenka 07f888c436
remove UC 2024-12-02 11:45:32 +01:00
Shreyas Goenka 5f2db1e2bd
move .internal var 2024-12-02 11:44:00 +01:00
Shreyas Goenka 42bf6aeecf
revert bundletest move 2024-12-02 11:42:00 +01:00
shreyas-goenka e86a949d99
Add the `bundle_uuid` helper function for templates (#1947)
## Changes
This PR adds the `bundle_uuid` helper function that'll return a stable
identifier for the bundle for the duration of the `bundle init` command.

This is also the UUID that'll be set in the telemetry event sent during
`databricks bundle init` and would be used to correlate revenue from
bundle init with resource deployments.

Template authors should add the uuid field to their `databricks.yml`
file they generate:
```
bundle:
  # A stable identified for your DAB project. We use this UUID in the Databricks backend 
  # to correlate and identify multiple deployments of the same DAB project. 
  uuid: {{ bundle_uuid }}
```

## Tests
Unit test
2024-12-02 10:29:29 +00:00
Denis Bilenko 00bd98f898
Move loadGitDetails mutator to Initialize phase (#1944)
This will require API call when run inside a workspace, which will
require workspace client (we don't have one at the current point). We
want to keep Load phase quick, since it's common across all commands.
2024-12-02 09:49:32 +00:00
30 changed files with 109 additions and 40 deletions

View File

@ -7,8 +7,8 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"

View File

@ -49,4 +49,8 @@ type Bundle struct {
// Databricks CLI version constraints required to run the bundle.
DatabricksCliVersion string `json:"databricks_cli_version,omitempty"`
// A stable generated UUID for the bundle. This is normally serialized by
// Databricks first party template when a user runs bundle init.
Uuid string `json:"uuid,omitempty"`
}

View File

@ -6,10 +6,10 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/catalog"

View File

@ -5,10 +5,10 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/stretchr/testify/assert"

View File

@ -7,9 +7,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/pipelines"

View File

@ -26,7 +26,6 @@ func DefaultMutators() []bundle.Mutator {
ComputeIdToClusterId(),
InitializeVariables(),
DefineDefaultTarget(),
LoadGitDetails(),
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
// Note: This mutator must run before the target overrides are merged.

View File

@ -6,9 +6,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
)

View File

@ -6,9 +6,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

View File

@ -6,10 +6,10 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/dashboards"

View File

@ -10,11 +10,11 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/config/variable"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/vfs"

View File

@ -5,9 +5,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/compute"

View File

@ -5,9 +5,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/bundle/metadata"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/jobs"

View File

@ -6,9 +6,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/compute"

View File

@ -8,6 +8,10 @@ import (
"github.com/databricks/cli/libs/filer"
)
// We upload artifacts to the workspace in a directory named ".internal" to have
// a well defined location for artifacts that have been uploaded by the DABs.
const InternalDirName = ".internal"
// This function returns a filer for uploading artifacts to the configured location.
// Supported locations:
// 1. WSFS

View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"net/http"
"path"
"strings"
@ -82,20 +81,25 @@ func filerForVolume(ctx context.Context, b *bundle.Bundle) (filer.Filer, string,
baseErr := diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("failed to fetch metadata for %s: %s", volumePath, err),
Summary: fmt.Sprintf("unable to determine if volume at %s exists: %s", volumePath, err),
Locations: b.Config.GetLocations("workspace.artifact_path"),
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
}
var aerr *apierr.APIError
if errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound {
if errors.Is(err, apierr.ErrNotFound) {
// Since the API returned a 404, the volume does not exist.
// Modify the error message to provide more context.
baseErr.Summary = fmt.Sprintf("volume %s does not exist: %s", volumePath, err)
// If the volume is defined in the bundle, provide a more helpful error diagnostic,
// with more details and location information.
path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName)
if !ok {
return nil, "", diag.Diagnostics{baseErr}
}
baseErr.Detail = `You are using a UC volume in your artifact_path that is managed by
baseErr.Detail = `You are using a volume in your artifact_path that is managed by
this bundle but which has not been deployed yet. Please first deploy
the UC volume using 'bundle deploy' and then switch over to using it in
the volume using 'bundle deploy' and then switch over to using it in
the artifact_path.`
baseErr.Paths = append(baseErr.Paths, path)
baseErr.Locations = append(baseErr.Locations, locations...)

View File

@ -7,9 +7,9 @@ import (
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/bundletest"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/filer"
@ -87,7 +87,33 @@ func TestFindVolumeInBundle(t *testing.T) {
assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
}
func TestFilerForVolumeNotInBundle(t *testing.T) {
func TestFilerForVolumeForErrorFromAPI(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
ArtifactPath: "/Volumes/main/my_schema/my_volume",
},
},
}
bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &sdkconfig.Config{}
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(fmt.Errorf("error from API"))
b.SetWorkpaceClient(m.WorkspaceClient)
_, _, diags := filerForVolume(context.Background(), b)
assert.Equal(t, diag.Diagnostics{
{
Severity: diag.Error,
Summary: "unable to determine if volume at /Volumes/main/my_schema/my_volume exists: error from API",
Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
}}, diags)
}
func TestFilerForVolumeWithVolumeNotFound(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
@ -100,20 +126,20 @@ func TestFilerForVolumeNotInBundle(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &sdkconfig.Config{}
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(fmt.Errorf("error from API"))
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(apierr.NotFound("some error message"))
b.SetWorkpaceClient(m.WorkspaceClient)
_, _, diags := filerForVolume(context.Background(), b)
assert.Equal(t, diag.Diagnostics{
{
Severity: diag.Error,
Summary: "failed to fetch metadata for /Volumes/main/my_schema/doesnotexist: error from API",
Summary: "volume /Volumes/main/my_schema/doesnotexist does not exist: some error message",
Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
}}, diags)
}
func TestFilerForVolumeInBundle(t *testing.T) {
func TestFilerForVolumeNotFoundAndInBundle(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
@ -139,22 +165,19 @@ func TestFilerForVolumeInBundle(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
m.WorkspaceClient.Config = &sdkconfig.Config{}
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(&apierr.APIError{
StatusCode: 404,
Message: "error from API",
})
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(apierr.NotFound("error from API"))
b.SetWorkpaceClient(m.WorkspaceClient)
_, _, diags := GetFilerForLibraries(context.Background(), b)
assert.Equal(t, diag.Diagnostics{
{
Severity: diag.Error,
Summary: "failed to fetch metadata for /Volumes/main/my_schema/my_volume: error from API",
Summary: "volume /Volumes/main/my_schema/my_volume does not exist: error from API",
Locations: []dyn.Location{{"config.yml", 1, 2}, {"volume.yml", 1, 2}},
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")},
Detail: `You are using a UC volume in your artifact_path that is managed by
Detail: `You are using a volume in your artifact_path that is managed by
this bundle but which has not been deployed yet. Please first deploy
the UC volume using 'bundle deploy' and then switch over to using it in
the volume using 'bundle deploy' and then switch over to using it in
the artifact_path.`,
},
}, diags)

View File

@ -8,10 +8,6 @@ import (
"github.com/databricks/cli/libs/filer"
)
// We upload artifacts to the workspace in a directory named ".internal" to have
// a well defined location for artifacts that have been uploaded by the DABs.
const InternalDirName = ".internal"
func filerForWorkspace(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) {
uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName)
f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath)

View File

@ -96,7 +96,7 @@ properties such as the 'catalog' or 'storage' are changed:`
// One or more volumes is being recreated.
if len(volumeActions) != 0 {
msg := `
This action will result in the deletion or recreation of the following Volumes.
This action will result in the deletion or recreation of the following volumes.
For managed volumes, the files stored in the volume are also deleted from your
cloud tenant within 30 days. For external volumes, the metadata about the volume
is removed from the catalog, but the underlying files are not deleted:`

View File

@ -39,6 +39,7 @@ func Initialize() bundle.Mutator {
mutator.MergePipelineClusters(),
mutator.InitializeWorkspaceClient(),
mutator.PopulateCurrentUser(),
mutator.LoadGitDetails(),
mutator.DefineDefaultWorkspaceRoot(),
mutator.ExpandWorkspaceRoot(),

View File

@ -1,15 +1,19 @@
package config_tests
import (
"context"
"fmt"
"strings"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/stretchr/testify/assert"
)
func TestGitAutoLoadWithEnvironment(t *testing.T) {
b := load(t, "./environments_autoload_git")
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
assert.True(t, b.Config.Bundle.Git.Inferred)
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
@ -17,6 +21,7 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) {
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_autoload_git", "production")
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")

View File

@ -14,6 +14,7 @@ import (
func TestGitAutoLoad(t *testing.T) {
b := load(t, "./autoload_git")
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
assert.True(t, b.Config.Bundle.Git.Inferred)
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
@ -21,6 +22,7 @@ func TestGitAutoLoad(t *testing.T) {
func TestGitManuallySetBranch(t *testing.T) {
b := loadTarget(t, "./autoload_git", "production")
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
@ -34,6 +36,7 @@ func TestGitBundleBranchValidation(t *testing.T) {
})
b := load(t, "./git_branch_validation")
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "feature-a", b.Config.Bundle.Git.Branch)
assert.Equal(t, "feature-b", b.Config.Bundle.Git.ActualBranch)

View File

@ -259,7 +259,7 @@ func TestAccUploadArtifactFileToVolumeThatDoesNotExist(t *testing.T) {
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
assert.Error(t, err)
assert.Equal(t, fmt.Sprintf(`Error: failed to fetch metadata for /Volumes/main/%s/doesnotexist: Not Found
assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/doesnotexist does not exist: Not Found
at workspace.artifact_path
in databricks.yml:6:18
@ -296,15 +296,15 @@ func TestAccUploadArtifactToVolumeNotYetDeployed(t *testing.T) {
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
assert.Error(t, err)
assert.Equal(t, fmt.Sprintf(`Error: failed to fetch metadata for /Volumes/main/%s/my_volume: Not Found
assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/my_volume does not exist: Not Found
at workspace.artifact_path
resources.volumes.foo
in databricks.yml:6:18
databricks.yml:11:7
You are using a UC volume in your artifact_path that is managed by
You are using a volume in your artifact_path that is managed by
this bundle but which has not been deployed yet. Please first deploy
the UC volume using 'bundle deploy' and then switch over to using it in
the volume using 'bundle deploy' and then switch over to using it in
the artifact_path.
`, schemaName), stdout.String())

View File

@ -284,7 +284,7 @@ func TestAccDeployUcVolume(t *testing.T) {
t.Setenv("BUNDLE_ROOT", bundleRoot)
stdout, stderr, err := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--var=schema_name=${resources.schemas.schema2.name}").Run()
assert.Error(t, err)
assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following Volumes.
assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following volumes.
For managed volumes, the files stored in the volume are also deleted from your
cloud tenant within 30 days. For external volumes, the metadata about the volume
is removed from the catalog, but the underlying files are not deleted:

View File

@ -35,6 +35,13 @@ var cachedUser *iam.User
var cachedIsServicePrincipal *bool
var cachedCatalog *string
// UUID that is stable for the duration of the template execution. This can be used
// to populate the `bundle.uuid` field in databricks.yml by template authors.
//
// It's automatically logged in our telemetry logs when `databricks bundle init`
// is run and can be used to attribute DBU revenue to bundle templates.
var bundleUuid = uuid.New().String()
func loadHelpers(ctx context.Context) template.FuncMap {
w := root.WorkspaceClient(ctx)
return template.FuncMap{
@ -57,6 +64,9 @@ func loadHelpers(ctx context.Context) template.FuncMap {
"uuid": func() string {
return uuid.New().String()
},
"bundle_uuid": func() string {
return bundleUuid
},
// A key value pair. This is used with the map function to generate maps
// to use inside a template
"pair": func(k string, v any) pair {

View File

@ -32,6 +32,24 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
assert.Equal(t, `{{ fail "abc" }}`, cleanContent)
}
func TestTemplateBundleUuidFunction(t *testing.T) {
ctx := context.Background()
ctx = root.SetWorkspaceClient(ctx, nil)
helpers := loadHelpers(ctx)
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/bundle-uuid/template", "./testdata/bundle-uuid/library")
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
assert.Len(t, r.files, 2)
c1 := strings.Trim(string(r.files[0].(*inMemoryFile).content), "\n\r")
assert.Equal(t, strings.Repeat(bundleUuid, 3), c1)
c2 := strings.Trim(string(r.files[1].(*inMemoryFile).content), "\n\r")
assert.Equal(t, strings.Repeat(bundleUuid, 5), c2)
}
func TestTemplateRegexpCompileFunction(t *testing.T) {
ctx := context.Background()

View File

View File

@ -0,0 +1 @@
{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}

View File

@ -0,0 +1 @@
{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}