mirror of https://github.com/databricks/cli.git
Compare commits
11 Commits
d1ec088d70
...
d460bd6859
Author | SHA1 | Date |
---|---|---|
|
d460bd6859 | |
|
8d790efcf9 | |
|
8e25bb4a47 | |
|
3461018e8a | |
|
406c0736ff | |
|
7d544f4af4 | |
|
07f888c436 | |
|
5f2db1e2bd | |
|
42bf6aeecf | |
|
e86a949d99 | |
|
00bd98f898 |
|
@ -7,8 +7,8 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/internal/testutil"
|
"github.com/databricks/cli/internal/testutil"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
|
@ -49,4 +49,8 @@ type Bundle struct {
|
||||||
|
|
||||||
// Databricks CLI version constraints required to run the bundle.
|
// Databricks CLI version constraints required to run the bundle.
|
||||||
DatabricksCliVersion string `json:"databricks_cli_version,omitempty"`
|
DatabricksCliVersion string `json:"databricks_cli_version,omitempty"`
|
||||||
|
|
||||||
|
// A stable generated UUID for the bundle. This is normally serialized by
|
||||||
|
// Databricks first party template when a user runs bundle init.
|
||||||
|
Uuid string `json:"uuid,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,10 +6,10 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
"github.com/databricks/cli/libs/dbr"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/catalog"
|
"github.com/databricks/databricks-sdk-go/service/catalog"
|
||||||
|
|
|
@ -5,10 +5,10 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
|
@ -7,9 +7,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||||
|
|
|
@ -26,7 +26,6 @@ func DefaultMutators() []bundle.Mutator {
|
||||||
ComputeIdToClusterId(),
|
ComputeIdToClusterId(),
|
||||||
InitializeVariables(),
|
InitializeVariables(),
|
||||||
DefineDefaultTarget(),
|
DefineDefaultTarget(),
|
||||||
LoadGitDetails(),
|
|
||||||
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
|
pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
|
||||||
|
|
||||||
// Note: This mutator must run before the target overrides are merged.
|
// Note: This mutator must run before the target overrides are merged.
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
|
@ -6,10 +6,10 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
"github.com/databricks/databricks-sdk-go/service/dashboards"
|
||||||
|
|
|
@ -10,11 +10,11 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
"github.com/databricks/cli/bundle/config/variable"
|
"github.com/databricks/cli/bundle/config/variable"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
|
|
|
@ -5,9 +5,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
|
|
|
@ -5,9 +5,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/bundle/metadata"
|
"github.com/databricks/cli/bundle/metadata"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/internal/testutil"
|
"github.com/databricks/cli/internal/testutil"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/databricks-sdk-go/service/compute"
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
|
|
|
@ -8,6 +8,10 @@ import (
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// We upload artifacts to the workspace in a directory named ".internal" to have
|
||||||
|
// a well defined location for artifacts that have been uploaded by the DABs.
|
||||||
|
const InternalDirName = ".internal"
|
||||||
|
|
||||||
// This function returns a filer for uploading artifacts to the configured location.
|
// This function returns a filer for uploading artifacts to the configured location.
|
||||||
// Supported locations:
|
// Supported locations:
|
||||||
// 1. WSFS
|
// 1. WSFS
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
@ -82,20 +81,25 @@ func filerForVolume(ctx context.Context, b *bundle.Bundle) (filer.Filer, string,
|
||||||
|
|
||||||
baseErr := diag.Diagnostic{
|
baseErr := diag.Diagnostic{
|
||||||
Severity: diag.Error,
|
Severity: diag.Error,
|
||||||
Summary: fmt.Sprintf("failed to fetch metadata for %s: %s", volumePath, err),
|
Summary: fmt.Sprintf("unable to determine if volume at %s exists: %s", volumePath, err),
|
||||||
Locations: b.Config.GetLocations("workspace.artifact_path"),
|
Locations: b.Config.GetLocations("workspace.artifact_path"),
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
||||||
}
|
}
|
||||||
|
|
||||||
var aerr *apierr.APIError
|
if errors.Is(err, apierr.ErrNotFound) {
|
||||||
if errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound {
|
// Since the API returned a 404, the volume does not exist.
|
||||||
|
// Modify the error message to provide more context.
|
||||||
|
baseErr.Summary = fmt.Sprintf("volume %s does not exist: %s", volumePath, err)
|
||||||
|
|
||||||
|
// If the volume is defined in the bundle, provide a more helpful error diagnostic,
|
||||||
|
// with more details and location information.
|
||||||
path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName)
|
path, locations, ok := findVolumeInBundle(b, catalogName, schemaName, volumeName)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, "", diag.Diagnostics{baseErr}
|
return nil, "", diag.Diagnostics{baseErr}
|
||||||
}
|
}
|
||||||
baseErr.Detail = `You are using a UC volume in your artifact_path that is managed by
|
baseErr.Detail = `You are using a volume in your artifact_path that is managed by
|
||||||
this bundle but which has not been deployed yet. Please first deploy
|
this bundle but which has not been deployed yet. Please first deploy
|
||||||
the UC volume using 'bundle deploy' and then switch over to using it in
|
the volume using 'bundle deploy' and then switch over to using it in
|
||||||
the artifact_path.`
|
the artifact_path.`
|
||||||
baseErr.Paths = append(baseErr.Paths, path)
|
baseErr.Paths = append(baseErr.Paths, path)
|
||||||
baseErr.Locations = append(baseErr.Locations, locations...)
|
baseErr.Locations = append(baseErr.Locations, locations...)
|
||||||
|
|
|
@ -7,9 +7,9 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/bundle/bundletest"
|
|
||||||
"github.com/databricks/cli/bundle/config"
|
"github.com/databricks/cli/bundle/config"
|
||||||
"github.com/databricks/cli/bundle/config/resources"
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
|
@ -87,7 +87,33 @@ func TestFindVolumeInBundle(t *testing.T) {
|
||||||
assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
|
assert.Equal(t, dyn.MustPathFromString("resources.volumes.foo"), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFilerForVolumeNotInBundle(t *testing.T) {
|
func TestFilerForVolumeForErrorFromAPI(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Workspace: config.Workspace{
|
||||||
|
ArtifactPath: "/Volumes/main/my_schema/my_volume",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}})
|
||||||
|
|
||||||
|
m := mocks.NewMockWorkspaceClient(t)
|
||||||
|
m.WorkspaceClient.Config = &sdkconfig.Config{}
|
||||||
|
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(fmt.Errorf("error from API"))
|
||||||
|
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||||
|
|
||||||
|
_, _, diags := filerForVolume(context.Background(), b)
|
||||||
|
assert.Equal(t, diag.Diagnostics{
|
||||||
|
{
|
||||||
|
Severity: diag.Error,
|
||||||
|
Summary: "unable to determine if volume at /Volumes/main/my_schema/my_volume exists: error from API",
|
||||||
|
Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
|
||||||
|
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
||||||
|
}}, diags)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilerForVolumeWithVolumeNotFound(t *testing.T) {
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Workspace: config.Workspace{
|
Workspace: config.Workspace{
|
||||||
|
@ -100,20 +126,20 @@ func TestFilerForVolumeNotInBundle(t *testing.T) {
|
||||||
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
m := mocks.NewMockWorkspaceClient(t)
|
||||||
m.WorkspaceClient.Config = &sdkconfig.Config{}
|
m.WorkspaceClient.Config = &sdkconfig.Config{}
|
||||||
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(fmt.Errorf("error from API"))
|
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/doesnotexist").Return(apierr.NotFound("some error message"))
|
||||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||||
|
|
||||||
_, _, diags := filerForVolume(context.Background(), b)
|
_, _, diags := filerForVolume(context.Background(), b)
|
||||||
assert.Equal(t, diag.Diagnostics{
|
assert.Equal(t, diag.Diagnostics{
|
||||||
{
|
{
|
||||||
Severity: diag.Error,
|
Severity: diag.Error,
|
||||||
Summary: "failed to fetch metadata for /Volumes/main/my_schema/doesnotexist: error from API",
|
Summary: "volume /Volumes/main/my_schema/doesnotexist does not exist: some error message",
|
||||||
Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
|
Locations: []dyn.Location{{File: "config.yml", Line: 1, Column: 2}},
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")},
|
||||||
}}, diags)
|
}}, diags)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFilerForVolumeInBundle(t *testing.T) {
|
func TestFilerForVolumeNotFoundAndInBundle(t *testing.T) {
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
Config: config.Root{
|
Config: config.Root{
|
||||||
Workspace: config.Workspace{
|
Workspace: config.Workspace{
|
||||||
|
@ -139,22 +165,19 @@ func TestFilerForVolumeInBundle(t *testing.T) {
|
||||||
|
|
||||||
m := mocks.NewMockWorkspaceClient(t)
|
m := mocks.NewMockWorkspaceClient(t)
|
||||||
m.WorkspaceClient.Config = &sdkconfig.Config{}
|
m.WorkspaceClient.Config = &sdkconfig.Config{}
|
||||||
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(&apierr.APIError{
|
m.GetMockFilesAPI().EXPECT().GetDirectoryMetadataByDirectoryPath(mock.Anything, "/Volumes/main/my_schema/my_volume").Return(apierr.NotFound("error from API"))
|
||||||
StatusCode: 404,
|
|
||||||
Message: "error from API",
|
|
||||||
})
|
|
||||||
b.SetWorkpaceClient(m.WorkspaceClient)
|
b.SetWorkpaceClient(m.WorkspaceClient)
|
||||||
|
|
||||||
_, _, diags := GetFilerForLibraries(context.Background(), b)
|
_, _, diags := GetFilerForLibraries(context.Background(), b)
|
||||||
assert.Equal(t, diag.Diagnostics{
|
assert.Equal(t, diag.Diagnostics{
|
||||||
{
|
{
|
||||||
Severity: diag.Error,
|
Severity: diag.Error,
|
||||||
Summary: "failed to fetch metadata for /Volumes/main/my_schema/my_volume: error from API",
|
Summary: "volume /Volumes/main/my_schema/my_volume does not exist: error from API",
|
||||||
Locations: []dyn.Location{{"config.yml", 1, 2}, {"volume.yml", 1, 2}},
|
Locations: []dyn.Location{{"config.yml", 1, 2}, {"volume.yml", 1, 2}},
|
||||||
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")},
|
Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path"), dyn.MustPathFromString("resources.volumes.foo")},
|
||||||
Detail: `You are using a UC volume in your artifact_path that is managed by
|
Detail: `You are using a volume in your artifact_path that is managed by
|
||||||
this bundle but which has not been deployed yet. Please first deploy
|
this bundle but which has not been deployed yet. Please first deploy
|
||||||
the UC volume using 'bundle deploy' and then switch over to using it in
|
the volume using 'bundle deploy' and then switch over to using it in
|
||||||
the artifact_path.`,
|
the artifact_path.`,
|
||||||
},
|
},
|
||||||
}, diags)
|
}, diags)
|
||||||
|
|
|
@ -8,10 +8,6 @@ import (
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
)
|
)
|
||||||
|
|
||||||
// We upload artifacts to the workspace in a directory named ".internal" to have
|
|
||||||
// a well defined location for artifacts that have been uploaded by the DABs.
|
|
||||||
const InternalDirName = ".internal"
|
|
||||||
|
|
||||||
func filerForWorkspace(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) {
|
func filerForWorkspace(b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) {
|
||||||
uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName)
|
uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName)
|
||||||
f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath)
|
f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath)
|
||||||
|
|
|
@ -96,7 +96,7 @@ properties such as the 'catalog' or 'storage' are changed:`
|
||||||
// One or more volumes is being recreated.
|
// One or more volumes is being recreated.
|
||||||
if len(volumeActions) != 0 {
|
if len(volumeActions) != 0 {
|
||||||
msg := `
|
msg := `
|
||||||
This action will result in the deletion or recreation of the following Volumes.
|
This action will result in the deletion or recreation of the following volumes.
|
||||||
For managed volumes, the files stored in the volume are also deleted from your
|
For managed volumes, the files stored in the volume are also deleted from your
|
||||||
cloud tenant within 30 days. For external volumes, the metadata about the volume
|
cloud tenant within 30 days. For external volumes, the metadata about the volume
|
||||||
is removed from the catalog, but the underlying files are not deleted:`
|
is removed from the catalog, but the underlying files are not deleted:`
|
||||||
|
|
|
@ -39,6 +39,7 @@ func Initialize() bundle.Mutator {
|
||||||
mutator.MergePipelineClusters(),
|
mutator.MergePipelineClusters(),
|
||||||
mutator.InitializeWorkspaceClient(),
|
mutator.InitializeWorkspaceClient(),
|
||||||
mutator.PopulateCurrentUser(),
|
mutator.PopulateCurrentUser(),
|
||||||
|
mutator.LoadGitDetails(),
|
||||||
|
|
||||||
mutator.DefineDefaultWorkspaceRoot(),
|
mutator.DefineDefaultWorkspaceRoot(),
|
||||||
mutator.ExpandWorkspaceRoot(),
|
mutator.ExpandWorkspaceRoot(),
|
||||||
|
|
|
@ -1,15 +1,19 @@
|
||||||
package config_tests
|
package config_tests
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config/mutator"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGitAutoLoadWithEnvironment(t *testing.T) {
|
func TestGitAutoLoadWithEnvironment(t *testing.T) {
|
||||||
b := load(t, "./environments_autoload_git")
|
b := load(t, "./environments_autoload_git")
|
||||||
|
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
|
||||||
assert.True(t, b.Config.Bundle.Git.Inferred)
|
assert.True(t, b.Config.Bundle.Git.Inferred)
|
||||||
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
||||||
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
|
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
|
||||||
|
@ -17,6 +21,7 @@ func TestGitAutoLoadWithEnvironment(t *testing.T) {
|
||||||
|
|
||||||
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
|
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
|
||||||
b := loadTarget(t, "./environments_autoload_git", "production")
|
b := loadTarget(t, "./environments_autoload_git", "production")
|
||||||
|
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
|
||||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||||
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||||
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
|
|
||||||
func TestGitAutoLoad(t *testing.T) {
|
func TestGitAutoLoad(t *testing.T) {
|
||||||
b := load(t, "./autoload_git")
|
b := load(t, "./autoload_git")
|
||||||
|
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
|
||||||
assert.True(t, b.Config.Bundle.Git.Inferred)
|
assert.True(t, b.Config.Bundle.Git.Inferred)
|
||||||
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
||||||
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
|
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
|
||||||
|
@ -21,6 +22,7 @@ func TestGitAutoLoad(t *testing.T) {
|
||||||
|
|
||||||
func TestGitManuallySetBranch(t *testing.T) {
|
func TestGitManuallySetBranch(t *testing.T) {
|
||||||
b := loadTarget(t, "./autoload_git", "production")
|
b := loadTarget(t, "./autoload_git", "production")
|
||||||
|
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
|
||||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||||
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
|
||||||
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
|
||||||
|
@ -34,6 +36,7 @@ func TestGitBundleBranchValidation(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
b := load(t, "./git_branch_validation")
|
b := load(t, "./git_branch_validation")
|
||||||
|
bundle.Apply(context.Background(), b, mutator.LoadGitDetails())
|
||||||
assert.False(t, b.Config.Bundle.Git.Inferred)
|
assert.False(t, b.Config.Bundle.Git.Inferred)
|
||||||
assert.Equal(t, "feature-a", b.Config.Bundle.Git.Branch)
|
assert.Equal(t, "feature-a", b.Config.Bundle.Git.Branch)
|
||||||
assert.Equal(t, "feature-b", b.Config.Bundle.Git.ActualBranch)
|
assert.Equal(t, "feature-b", b.Config.Bundle.Git.ActualBranch)
|
||||||
|
|
|
@ -259,7 +259,7 @@ func TestAccUploadArtifactFileToVolumeThatDoesNotExist(t *testing.T) {
|
||||||
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
|
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
|
||||||
|
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.Equal(t, fmt.Sprintf(`Error: failed to fetch metadata for /Volumes/main/%s/doesnotexist: Not Found
|
assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/doesnotexist does not exist: Not Found
|
||||||
at workspace.artifact_path
|
at workspace.artifact_path
|
||||||
in databricks.yml:6:18
|
in databricks.yml:6:18
|
||||||
|
|
||||||
|
@ -296,15 +296,15 @@ func TestAccUploadArtifactToVolumeNotYetDeployed(t *testing.T) {
|
||||||
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
|
stdout, stderr, err := internal.RequireErrorRun(t, "bundle", "deploy")
|
||||||
|
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.Equal(t, fmt.Sprintf(`Error: failed to fetch metadata for /Volumes/main/%s/my_volume: Not Found
|
assert.Equal(t, fmt.Sprintf(`Error: volume /Volumes/main/%s/my_volume does not exist: Not Found
|
||||||
at workspace.artifact_path
|
at workspace.artifact_path
|
||||||
resources.volumes.foo
|
resources.volumes.foo
|
||||||
in databricks.yml:6:18
|
in databricks.yml:6:18
|
||||||
databricks.yml:11:7
|
databricks.yml:11:7
|
||||||
|
|
||||||
You are using a UC volume in your artifact_path that is managed by
|
You are using a volume in your artifact_path that is managed by
|
||||||
this bundle but which has not been deployed yet. Please first deploy
|
this bundle but which has not been deployed yet. Please first deploy
|
||||||
the UC volume using 'bundle deploy' and then switch over to using it in
|
the volume using 'bundle deploy' and then switch over to using it in
|
||||||
the artifact_path.
|
the artifact_path.
|
||||||
|
|
||||||
`, schemaName), stdout.String())
|
`, schemaName), stdout.String())
|
||||||
|
|
|
@ -284,7 +284,7 @@ func TestAccDeployUcVolume(t *testing.T) {
|
||||||
t.Setenv("BUNDLE_ROOT", bundleRoot)
|
t.Setenv("BUNDLE_ROOT", bundleRoot)
|
||||||
stdout, stderr, err := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--var=schema_name=${resources.schemas.schema2.name}").Run()
|
stdout, stderr, err := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--var=schema_name=${resources.schemas.schema2.name}").Run()
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following Volumes.
|
assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following volumes.
|
||||||
For managed volumes, the files stored in the volume are also deleted from your
|
For managed volumes, the files stored in the volume are also deleted from your
|
||||||
cloud tenant within 30 days. For external volumes, the metadata about the volume
|
cloud tenant within 30 days. For external volumes, the metadata about the volume
|
||||||
is removed from the catalog, but the underlying files are not deleted:
|
is removed from the catalog, but the underlying files are not deleted:
|
||||||
|
|
|
@ -35,6 +35,13 @@ var cachedUser *iam.User
|
||||||
var cachedIsServicePrincipal *bool
|
var cachedIsServicePrincipal *bool
|
||||||
var cachedCatalog *string
|
var cachedCatalog *string
|
||||||
|
|
||||||
|
// UUID that is stable for the duration of the template execution. This can be used
|
||||||
|
// to populate the `bundle.uuid` field in databricks.yml by template authors.
|
||||||
|
//
|
||||||
|
// It's automatically logged in our telemetry logs when `databricks bundle init`
|
||||||
|
// is run and can be used to attribute DBU revenue to bundle templates.
|
||||||
|
var bundleUuid = uuid.New().String()
|
||||||
|
|
||||||
func loadHelpers(ctx context.Context) template.FuncMap {
|
func loadHelpers(ctx context.Context) template.FuncMap {
|
||||||
w := root.WorkspaceClient(ctx)
|
w := root.WorkspaceClient(ctx)
|
||||||
return template.FuncMap{
|
return template.FuncMap{
|
||||||
|
@ -57,6 +64,9 @@ func loadHelpers(ctx context.Context) template.FuncMap {
|
||||||
"uuid": func() string {
|
"uuid": func() string {
|
||||||
return uuid.New().String()
|
return uuid.New().String()
|
||||||
},
|
},
|
||||||
|
"bundle_uuid": func() string {
|
||||||
|
return bundleUuid
|
||||||
|
},
|
||||||
// A key value pair. This is used with the map function to generate maps
|
// A key value pair. This is used with the map function to generate maps
|
||||||
// to use inside a template
|
// to use inside a template
|
||||||
"pair": func(k string, v any) pair {
|
"pair": func(k string, v any) pair {
|
||||||
|
|
|
@ -32,6 +32,24 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
|
||||||
assert.Equal(t, `{{ fail "abc" }}`, cleanContent)
|
assert.Equal(t, `{{ fail "abc" }}`, cleanContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTemplateBundleUuidFunction(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
|
helpers := loadHelpers(ctx)
|
||||||
|
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/bundle-uuid/template", "./testdata/bundle-uuid/library")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = r.walk()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Len(t, r.files, 2)
|
||||||
|
c1 := strings.Trim(string(r.files[0].(*inMemoryFile).content), "\n\r")
|
||||||
|
assert.Equal(t, strings.Repeat(bundleUuid, 3), c1)
|
||||||
|
c2 := strings.Trim(string(r.files[1].(*inMemoryFile).content), "\n\r")
|
||||||
|
assert.Equal(t, strings.Repeat(bundleUuid, 5), c2)
|
||||||
|
}
|
||||||
|
|
||||||
func TestTemplateRegexpCompileFunction(t *testing.T) {
|
func TestTemplateRegexpCompileFunction(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}
|
|
@ -0,0 +1 @@
|
||||||
|
{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}{{bundle_uuid}}
|
Loading…
Reference in New Issue