Mutators to define current user and default artifact path (#112)

This commit is contained in:
Pieter Noordhuis 2022-12-01 11:17:29 +01:00 committed by GitHub
parent 230811031f
commit 34af98a8c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 136 additions and 4 deletions

View File

@ -65,13 +65,13 @@ func (m *build) Apply(_ context.Context, b *bundle.Bundle) ([]bundle.Mutator, er
// Check that an artifact path is defined.
remotePath := b.Config.Workspace.ArtifactPath.Workspace
if remotePath == nil {
if remotePath == "" {
return nil, fmt.Errorf("remote artifact path not configured")
}
// Store absolute paths.
artifact.LocalPath = filepath.Join(b.Config.Path, artifact.Path)
artifact.RemotePath = path.Join(*remotePath, stripExtension(artifact.Path))
artifact.RemotePath = path.Join(remotePath, stripExtension(artifact.Path))
return nil, nil
}

View File

@ -0,0 +1,40 @@
package mutator
import (
"context"
"fmt"
"path"
"github.com/databricks/bricks/bundle"
)
type defaultArtifactPath struct{}
// DefaultArtifactPath configures the artifact path if it isn't already set.
func DefaultArtifactPath() bundle.Mutator {
return &defaultArtifactPath{}
}
func (m *defaultArtifactPath) Name() string {
return "DefaultArtifactPath"
}
func (m *defaultArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
if b.Config.Workspace.ArtifactPath.IsSet() {
return nil, nil
}
me := b.Config.Workspace.CurrentUser
if me == nil {
return nil, fmt.Errorf("cannot configured default artifact path if current user isn't set")
}
// We assume we deal with notebooks only for the time being.
// When we need to upload wheel files or other non-notebook files,
// the workspace must support "Files in Workspace".
// If it doesn't, we need to resort to storing artifacts on DBFS.
home := fmt.Sprintf("/Users/%s", me.UserName)
root := path.Join(home, ".bundle", b.Config.Bundle.Name, b.Config.Bundle.Environment, "artifacts")
b.Config.Workspace.ArtifactPath.Workspace = root
return nil, nil
}

View File

@ -0,0 +1,50 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/bricks/bundle"
"github.com/databricks/bricks/bundle/config"
"github.com/databricks/bricks/bundle/config/mutator"
"github.com/databricks/databricks-sdk-go/service/scim"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDefaultArtifactPath(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Bundle: config.Bundle{
Name: "name",
Environment: "environment",
},
Workspace: config.Workspace{
CurrentUser: &scim.User{
UserName: "foo@bar.com",
},
},
},
}
_, err := mutator.DefaultArtifactPath().Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, "/Users/foo@bar.com/.bundle/name/environment/artifacts", bundle.Config.Workspace.ArtifactPath.Workspace)
}
func TestDefaultArtifactPathAlreadySet(t *testing.T) {
bundle := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
ArtifactPath: config.PathLike{
Workspace: "/foo/bar",
},
CurrentUser: &scim.User{
UserName: "foo@bar.com",
},
},
},
}
_, err := mutator.DefaultArtifactPath().Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, "/foo/bar", bundle.Config.Workspace.ArtifactPath.Workspace)
}

View File

@ -0,0 +1,29 @@
package mutator
import (
"context"
"github.com/databricks/bricks/bundle"
)
type populateCurrentUser struct{}
// PopulateCurrentUser sets the `current_user` property on the workspace.
func PopulateCurrentUser() bundle.Mutator {
return &populateCurrentUser{}
}
func (m *populateCurrentUser) Name() string {
return "PopulateCurrentUser"
}
func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) ([]bundle.Mutator, error) {
w := b.WorkspaceClient()
me, err := w.CurrentUser.Me(ctx)
if err != nil {
return nil, err
}
b.Config.Workspace.CurrentUser = me
return nil, nil
}

View File

@ -0,0 +1,3 @@
package mutator
// We need to implement workspace client mocking to implement this test.

View File

@ -2,14 +2,20 @@ package config
import (
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/scim"
)
type PathLike struct {
// Workspace contains a WSFS path.
Workspace *string `json:"workspace,omitempty"`
Workspace string `json:"workspace,omitempty"`
// DBFS contains a DBFS path.
DBFS *string `json:"dbfs,omitempty"`
DBFS string `json:"dbfs,omitempty"`
}
// IsSet returns whether either path is non-nil.
func (p PathLike) IsSet() bool {
return p.Workspace != "" || p.DBFS != ""
}
// Workspace defines configurables at the workspace level.
@ -37,6 +43,10 @@ type Workspace struct {
AzureEnvironment string `json:"azure_environment,omitempty"`
AzureLoginAppID string `json:"azure_login_app_id,omitempty"`
// CurrentUser holds the current user.
// This is set after configuration initialization.
CurrentUser *scim.User `json:"current_user,omitempty"`
// Remote path for artifacts.
// This can specify a workspace path, a DBFS path, or both.
// Some artifacts must be stored in the workspace (e.g. notebooks).