mirror of https://github.com/databricks/cli.git
Compare commits
1 Commits
69dd9bdd74
...
02486e46b8
Author | SHA1 | Date |
---|---|---|
Lennart Kats (databricks) | 02486e46b8 |
|
@ -5,12 +5,14 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
"github.com/databricks/cli/bundle"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/diag"
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/env"
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/databricks/cli/libs/vfs"
|
"github.com/databricks/cli/libs/vfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const envDatabricksRuntimeVersion = "DATABRICKS_RUNTIME_VERSION"
|
||||||
|
|
||||||
type configureWSFS struct{}
|
type configureWSFS struct{}
|
||||||
|
|
||||||
func ConfigureWSFS() bundle.Mutator {
|
func ConfigureWSFS() bundle.Mutator {
|
||||||
|
@ -30,7 +32,7 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
|
||||||
}
|
}
|
||||||
|
|
||||||
// The executable must be running on DBR.
|
// The executable must be running on DBR.
|
||||||
if !dbr.RunsOnRuntime(ctx) {
|
if _, ok := env.Lookup(ctx, envDatabricksRuntimeVersion); !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,65 +0,0 @@
|
||||||
package mutator_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle"
|
|
||||||
"github.com/databricks/cli/bundle/config/mutator"
|
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/vfs"
|
|
||||||
"github.com/databricks/databricks-sdk-go/config"
|
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func mockBundleForConfigureWSFS(t *testing.T, syncRootPath string) *bundle.Bundle {
|
|
||||||
// The native path of the sync root on Windows will never match the /Workspace prefix,
|
|
||||||
// so the test case for nominal behavior will always fail.
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("this test is not applicable on Windows")
|
|
||||||
}
|
|
||||||
|
|
||||||
b := &bundle.Bundle{
|
|
||||||
SyncRoot: vfs.MustNew(syncRootPath),
|
|
||||||
}
|
|
||||||
|
|
||||||
w := mocks.NewMockWorkspaceClient(t)
|
|
||||||
w.WorkspaceClient.Config = &config.Config{}
|
|
||||||
b.SetWorkpaceClient(w.WorkspaceClient)
|
|
||||||
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigureWSFS_SkipsIfNotWorkspacePrefix(t *testing.T) {
|
|
||||||
b := mockBundleForConfigureWSFS(t, "/foo")
|
|
||||||
originalSyncRoot := b.SyncRoot
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
assert.Equal(t, originalSyncRoot, b.SyncRoot)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigureWSFS_SkipsIfNotRunningOnRuntime(t *testing.T) {
|
|
||||||
b := mockBundleForConfigureWSFS(t, "/Workspace/foo")
|
|
||||||
originalSyncRoot := b.SyncRoot
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
ctx = dbr.MockRuntime(ctx, false)
|
|
||||||
diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
assert.Equal(t, originalSyncRoot, b.SyncRoot)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigureWSFS_SwapSyncRoot(t *testing.T) {
|
|
||||||
b := mockBundleForConfigureWSFS(t, "/Workspace/foo")
|
|
||||||
originalSyncRoot := b.SyncRoot
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
ctx = dbr.MockRuntime(ctx, true)
|
|
||||||
diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
|
|
||||||
assert.Empty(t, diags)
|
|
||||||
assert.NotEqual(t, originalSyncRoot, b.SyncRoot)
|
|
||||||
}
|
|
|
@ -65,8 +65,9 @@ func TestInitializeURLs(t *testing.T) {
|
||||||
},
|
},
|
||||||
QualityMonitors: map[string]*resources.QualityMonitor{
|
QualityMonitors: map[string]*resources.QualityMonitor{
|
||||||
"qualityMonitor1": {
|
"qualityMonitor1": {
|
||||||
|
CreateMonitor: &catalog.CreateMonitor{
|
||||||
TableName: "catalog.schema.qualityMonitor1",
|
TableName: "catalog.schema.qualityMonitor1",
|
||||||
CreateMonitor: &catalog.CreateMonitor{},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Schemas: map[string]*resources.Schema{
|
Schemas: map[string]*resources.Schema{
|
||||||
|
|
|
@ -44,11 +44,6 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
|
||||||
return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind())
|
return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip prefixing if the path does not start with /, it might be variable reference or smth else.
|
|
||||||
if !strings.HasPrefix(path, "/") {
|
|
||||||
return pv, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range skipPrefixes {
|
for _, prefix := range skipPrefixes {
|
||||||
if strings.HasPrefix(path, prefix) {
|
if strings.HasPrefix(path, prefix) {
|
||||||
return pv, nil
|
return pv, nil
|
||||||
|
|
|
@ -31,14 +31,6 @@ func TestPrependWorkspacePrefix(t *testing.T) {
|
||||||
path: "/Volumes/Users/test",
|
path: "/Volumes/Users/test",
|
||||||
expected: "/Volumes/Users/test",
|
expected: "/Volumes/Users/test",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
path: "~/test",
|
|
||||||
expected: "~/test",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
path: "${workspace.file_path}/test",
|
|
||||||
expected: "${workspace.file_path}/test",
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
|
|
|
@ -102,23 +102,16 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
|
||||||
"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
|
"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
|
||||||
},
|
},
|
||||||
QualityMonitors: map[string]*resources.QualityMonitor{
|
QualityMonitors: map[string]*resources.QualityMonitor{
|
||||||
"qualityMonitor1": {
|
"qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}},
|
||||||
TableName: "qualityMonitor1",
|
|
||||||
CreateMonitor: &catalog.CreateMonitor{
|
|
||||||
OutputSchemaName: "catalog.schema",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"qualityMonitor2": {
|
"qualityMonitor2": {
|
||||||
TableName: "qualityMonitor2",
|
|
||||||
CreateMonitor: &catalog.CreateMonitor{
|
CreateMonitor: &catalog.CreateMonitor{
|
||||||
OutputSchemaName: "catalog.schema",
|
TableName: "qualityMonitor2",
|
||||||
Schedule: &catalog.MonitorCronSchedule{},
|
Schedule: &catalog.MonitorCronSchedule{},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"qualityMonitor3": {
|
"qualityMonitor3": {
|
||||||
TableName: "qualityMonitor3",
|
|
||||||
CreateMonitor: &catalog.CreateMonitor{
|
CreateMonitor: &catalog.CreateMonitor{
|
||||||
OutputSchemaName: "catalog.schema",
|
TableName: "qualityMonitor3",
|
||||||
Schedule: &catalog.MonitorCronSchedule{
|
Schedule: &catalog.MonitorCronSchedule{
|
||||||
PauseStatus: catalog.MonitorCronSchedulePauseStatusUnpaused,
|
PauseStatus: catalog.MonitorCronSchedulePauseStatusUnpaused,
|
||||||
},
|
},
|
||||||
|
|
|
@ -13,15 +13,17 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type QualityMonitor struct {
|
type QualityMonitor struct {
|
||||||
|
// Represents the Input Arguments for Terraform and will get
|
||||||
|
// converted to a HCL representation for CRUD
|
||||||
|
*catalog.CreateMonitor
|
||||||
|
|
||||||
|
// This represents the id which is the full name of the monitor
|
||||||
|
// (catalog_name.schema_name.table_name) that can be used
|
||||||
|
// as a reference in other resources. This value is returned by terraform.
|
||||||
ID string `json:"id,omitempty" bundle:"readonly"`
|
ID string `json:"id,omitempty" bundle:"readonly"`
|
||||||
|
|
||||||
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
|
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
|
||||||
URL string `json:"url,omitempty" bundle:"internal"`
|
URL string `json:"url,omitempty" bundle:"internal"`
|
||||||
|
|
||||||
// The table name is a required field but not included as a JSON field in [catalog.CreateMonitor].
|
|
||||||
TableName string `json:"table_name"`
|
|
||||||
|
|
||||||
// This struct defines the creation payload for a monitor.
|
|
||||||
*catalog.CreateMonitor
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *QualityMonitor) UnmarshalJSON(b []byte) error {
|
func (s *QualityMonitor) UnmarshalJSON(b []byte) error {
|
||||||
|
|
|
@ -15,8 +15,8 @@ import (
|
||||||
|
|
||||||
func TestConvertQualityMonitor(t *testing.T) {
|
func TestConvertQualityMonitor(t *testing.T) {
|
||||||
var src = resources.QualityMonitor{
|
var src = resources.QualityMonitor{
|
||||||
TableName: "test_table_name",
|
|
||||||
CreateMonitor: &catalog.CreateMonitor{
|
CreateMonitor: &catalog.CreateMonitor{
|
||||||
|
TableName: "test_table_name",
|
||||||
AssetsDir: "assets_dir",
|
AssetsDir: "assets_dir",
|
||||||
OutputSchemaName: "output_schema_name",
|
OutputSchemaName: "output_schema_name",
|
||||||
InferenceLog: &catalog.MonitorInferenceLog{
|
InferenceLog: &catalog.MonitorInferenceLog{
|
||||||
|
|
|
@ -4,7 +4,6 @@ bundle:
|
||||||
resources:
|
resources:
|
||||||
quality_monitors:
|
quality_monitors:
|
||||||
myqualitymonitor:
|
myqualitymonitor:
|
||||||
table_name: catalog.schema.quality_monitor
|
|
||||||
inference_log:
|
inference_log:
|
||||||
granularities:
|
granularities:
|
||||||
- a
|
- a
|
||||||
|
|
|
@ -684,9 +684,6 @@
|
||||||
"description": "Configuration for monitoring snapshot tables.",
|
"description": "Configuration for monitoring snapshot tables.",
|
||||||
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot"
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot"
|
||||||
},
|
},
|
||||||
"table_name": {
|
|
||||||
"$ref": "#/$defs/string"
|
|
||||||
},
|
|
||||||
"time_series": {
|
"time_series": {
|
||||||
"description": "Configuration for monitoring time series tables.",
|
"description": "Configuration for monitoring time series tables.",
|
||||||
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries"
|
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries"
|
||||||
|
@ -698,7 +695,6 @@
|
||||||
},
|
},
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"required": [
|
"required": [
|
||||||
"table_name",
|
|
||||||
"assets_dir",
|
"assets_dir",
|
||||||
"output_schema_name"
|
"output_schema_name"
|
||||||
]
|
]
|
||||||
|
|
|
@ -3,7 +3,6 @@ package bundle
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
|
@ -110,24 +109,6 @@ func getUrlForNativeTemplate(name string) string {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func getFsForNativeTemplate(name string) (fs.FS, error) {
|
|
||||||
builtin, err := template.Builtin()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is a built-in template, the return value will be non-nil.
|
|
||||||
var templateFS fs.FS
|
|
||||||
for _, entry := range builtin {
|
|
||||||
if entry.Name == name {
|
|
||||||
templateFS = entry.FS
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return templateFS, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isRepoUrl(url string) bool {
|
func isRepoUrl(url string) bool {
|
||||||
result := false
|
result := false
|
||||||
for _, prefix := range gitUrlPrefixes {
|
for _, prefix := range gitUrlPrefixes {
|
||||||
|
@ -217,20 +198,9 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
|
||||||
if templateDir != "" {
|
if templateDir != "" {
|
||||||
return errors.New("--template-dir can only be used with a Git repository URL")
|
return errors.New("--template-dir can only be used with a Git repository URL")
|
||||||
}
|
}
|
||||||
|
|
||||||
templateFS, err := getFsForNativeTemplate(templatePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// If this is not a built-in template, then it must be a local file system path.
|
|
||||||
if templateFS == nil {
|
|
||||||
templateFS = os.DirFS(templatePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// skip downloading the repo because input arg is not a URL. We assume
|
// skip downloading the repo because input arg is not a URL. We assume
|
||||||
// it's a path on the local file system in that case
|
// it's a path on the local file system in that case
|
||||||
return template.Materialize(ctx, configFile, templateFS, outputDir)
|
return template.Materialize(ctx, configFile, templatePath, outputDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create a temporary directory with the name of the repository. The '*'
|
// Create a temporary directory with the name of the repository. The '*'
|
||||||
|
@ -254,8 +224,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
|
||||||
|
|
||||||
// Clean up downloaded repository once the template is materialized.
|
// Clean up downloaded repository once the template is materialized.
|
||||||
defer os.RemoveAll(repoDir)
|
defer os.RemoveAll(repoDir)
|
||||||
templateFS := os.DirFS(filepath.Join(repoDir, templateDir))
|
return template.Materialize(ctx, configFile, filepath.Join(repoDir, templateDir), outputDir)
|
||||||
return template.Materialize(ctx, configFile, templateFS, outputDir)
|
|
||||||
}
|
}
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/cmd/root"
|
"github.com/databricks/cli/cmd/root"
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -85,7 +84,7 @@ func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceCl
|
||||||
cmd, m := setupCommand(t)
|
cmd, m := setupCommand(t)
|
||||||
|
|
||||||
fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) {
|
fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) {
|
||||||
fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
|
fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
|
||||||
"dir": {FakeName: "root", FakeDir: true},
|
"dir": {FakeName: "root", FakeDir: true},
|
||||||
"dir/dirA": {FakeDir: true},
|
"dir/dirA": {FakeDir: true},
|
||||||
"dir/dirB": {FakeDir: true},
|
"dir/dirB": {FakeDir: true},
|
||||||
|
|
|
@ -11,7 +11,6 @@ import (
|
||||||
|
|
||||||
"github.com/databricks/cli/internal/build"
|
"github.com/databricks/cli/internal/build"
|
||||||
"github.com/databricks/cli/libs/cmdio"
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
"github.com/databricks/cli/libs/dbr"
|
|
||||||
"github.com/databricks/cli/libs/log"
|
"github.com/databricks/cli/libs/log"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
@ -74,9 +73,6 @@ func New(ctx context.Context) *cobra.Command {
|
||||||
// get the context back
|
// get the context back
|
||||||
ctx = cmd.Context()
|
ctx = cmd.Context()
|
||||||
|
|
||||||
// Detect if the CLI is running on DBR and store this on the context.
|
|
||||||
ctx = dbr.DetectRuntime(ctx)
|
|
||||||
|
|
||||||
// Configure our user agent with the command that's about to be executed.
|
// Configure our user agent with the command that's about to be executed.
|
||||||
ctx = withCommandInUserAgent(ctx, cmd)
|
ctx = withCommandInUserAgent(ctx, cmd)
|
||||||
ctx = withCommandExecIdInUserAgent(ctx)
|
ctx = withCommandExecIdInUserAgent(ctx)
|
||||||
|
|
|
@ -42,7 +42,7 @@ func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateN
|
||||||
cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
|
cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
|
||||||
ctx = cmdio.InContext(ctx, cmd)
|
ctx = cmdio.InContext(ctx, cmd)
|
||||||
|
|
||||||
err = template.Materialize(ctx, configFilePath, os.DirFS(templateRoot), bundleRoot)
|
err = template.Materialize(ctx, configFilePath, templateRoot, bundleRoot)
|
||||||
return bundleRoot, err
|
return bundleRoot, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -723,63 +723,6 @@ func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) {
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
assert.ErrorIs(t, err, fs.ErrNotExist)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAccWorkspaceFilesExtensionsNotebooksAreNotReadAsFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
wf, _ := setupWsfsExtensionsFiler(t)
|
|
||||||
|
|
||||||
// Create a notebook
|
|
||||||
err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Reading foo should fail. Even though the WSFS name for the notebook is foo
|
|
||||||
// reading the notebook should only work with the .ipynb extension.
|
|
||||||
_, err = wf.Read(ctx, "foo")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
_, err = wf.Read(ctx, "foo.ipynb")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAccWorkspaceFilesExtensionsNotebooksAreNotStatAsFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
wf, _ := setupWsfsExtensionsFiler(t)
|
|
||||||
|
|
||||||
// Create a notebook
|
|
||||||
err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Stating foo should fail. Even though the WSFS name for the notebook is foo
|
|
||||||
// stating the notebook should only work with the .ipynb extension.
|
|
||||||
_, err = wf.Stat(ctx, "foo")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
_, err = wf.Stat(ctx, "foo.ipynb")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAccWorkspaceFilesExtensionsNotebooksAreNotDeletedAsFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
wf, _ := setupWsfsExtensionsFiler(t)
|
|
||||||
|
|
||||||
// Create a notebook
|
|
||||||
err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Deleting foo should fail. Even though the WSFS name for the notebook is foo
|
|
||||||
// deleting the notebook should only work with the .ipynb extension.
|
|
||||||
err = wf.Delete(ctx, "foo")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
err = wf.Delete(ctx, "foo.ipynb")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) {
|
func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|
|
@ -1,49 +0,0 @@
|
||||||
package dbr
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
// key is a package-local type to use for context keys.
|
|
||||||
//
|
|
||||||
// Using an unexported type for context keys prevents key collisions across
|
|
||||||
// packages since external packages cannot create values of this type.
|
|
||||||
type key int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// dbrKey is the context key for the detection result.
|
|
||||||
// The value of 1 is arbitrary and can be any number.
|
|
||||||
// Other keys in the same package must have different values.
|
|
||||||
dbrKey = key(1)
|
|
||||||
)
|
|
||||||
|
|
||||||
// DetectRuntime detects whether or not the current
|
|
||||||
// process is running inside a Databricks Runtime environment.
|
|
||||||
// It return a new context with the detection result set.
|
|
||||||
func DetectRuntime(ctx context.Context) context.Context {
|
|
||||||
if v := ctx.Value(dbrKey); v != nil {
|
|
||||||
panic("dbr.DetectRuntime called twice on the same context")
|
|
||||||
}
|
|
||||||
return context.WithValue(ctx, dbrKey, detect(ctx))
|
|
||||||
}
|
|
||||||
|
|
||||||
// MockRuntime is a helper function to mock the detection result.
|
|
||||||
// It returns a new context with the detection result set.
|
|
||||||
func MockRuntime(ctx context.Context, b bool) context.Context {
|
|
||||||
if v := ctx.Value(dbrKey); v != nil {
|
|
||||||
panic("dbr.MockRuntime called twice on the same context")
|
|
||||||
}
|
|
||||||
return context.WithValue(ctx, dbrKey, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunsOnRuntime returns the detection result from the context.
|
|
||||||
// It expects a context returned by [DetectRuntime] or [MockRuntime].
|
|
||||||
//
|
|
||||||
// We store this value in a context to avoid having to use either
|
|
||||||
// a global variable, passing a boolean around everywhere, or
|
|
||||||
// performing the same detection multiple times.
|
|
||||||
func RunsOnRuntime(ctx context.Context) bool {
|
|
||||||
v := ctx.Value(dbrKey)
|
|
||||||
if v == nil {
|
|
||||||
panic("dbr.RunsOnRuntime called without calling dbr.DetectRuntime first")
|
|
||||||
}
|
|
||||||
return v.(bool)
|
|
||||||
}
|
|
|
@ -1,59 +0,0 @@
|
||||||
package dbr
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestContext_DetectRuntimePanics(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// Run detection.
|
|
||||||
ctx = DetectRuntime(ctx)
|
|
||||||
|
|
||||||
// Expect a panic if the detection is run twice.
|
|
||||||
assert.Panics(t, func() {
|
|
||||||
ctx = DetectRuntime(ctx)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContext_MockRuntimePanics(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// Run detection.
|
|
||||||
ctx = MockRuntime(ctx, true)
|
|
||||||
|
|
||||||
// Expect a panic if the mock function is run twice.
|
|
||||||
assert.Panics(t, func() {
|
|
||||||
MockRuntime(ctx, true)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContext_RunsOnRuntimePanics(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// Expect a panic if the detection is not run.
|
|
||||||
assert.Panics(t, func() {
|
|
||||||
RunsOnRuntime(ctx)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContext_RunsOnRuntime(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// Run detection.
|
|
||||||
ctx = DetectRuntime(ctx)
|
|
||||||
|
|
||||||
// Expect no panic because detection has run.
|
|
||||||
assert.NotPanics(t, func() {
|
|
||||||
RunsOnRuntime(ctx)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestContext_RunsOnRuntimeWithMock(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
assert.True(t, RunsOnRuntime(MockRuntime(ctx, true)))
|
|
||||||
assert.False(t, RunsOnRuntime(MockRuntime(ctx, false)))
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
package dbr
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/env"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Dereference [os.Stat] to allow mocking in tests.
|
|
||||||
var statFunc = os.Stat
|
|
||||||
|
|
||||||
// detect returns true if the current process is running on a Databricks Runtime.
|
|
||||||
// Its return value is meant to be cached in the context.
|
|
||||||
func detect(ctx context.Context) bool {
|
|
||||||
// Databricks Runtime implies Linux.
|
|
||||||
// Return early on other operating systems.
|
|
||||||
if runtime.GOOS != "linux" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Databricks Runtime always has the DATABRICKS_RUNTIME_VERSION environment variable set.
|
|
||||||
if value, ok := env.Lookup(ctx, "DATABRICKS_RUNTIME_VERSION"); !ok || value == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expect to see a "/databricks" directory.
|
|
||||||
if fi, err := statFunc("/databricks"); err != nil || !fi.IsDir() {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// All checks passed.
|
|
||||||
return true
|
|
||||||
}
|
|
|
@ -1,83 +0,0 @@
|
||||||
package dbr
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io/fs"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/env"
|
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func requireLinux(t *testing.T) {
|
|
||||||
if runtime.GOOS != "linux" {
|
|
||||||
t.Skipf("skipping test on %s", runtime.GOOS)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func configureStatFunc(t *testing.T, fi fs.FileInfo, err error) {
|
|
||||||
originalFunc := statFunc
|
|
||||||
statFunc = func(name string) (fs.FileInfo, error) {
|
|
||||||
assert.Equal(t, "/databricks", name)
|
|
||||||
return fi, err
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
statFunc = originalFunc
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetect_NotLinux(t *testing.T) {
|
|
||||||
if runtime.GOOS == "linux" {
|
|
||||||
t.Skip("skipping test on Linux OS")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
assert.False(t, detect(ctx))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetect_Env(t *testing.T) {
|
|
||||||
requireLinux(t)
|
|
||||||
|
|
||||||
// Configure other checks to pass.
|
|
||||||
configureStatFunc(t, fakefs.FileInfo{FakeDir: true}, nil)
|
|
||||||
|
|
||||||
t.Run("empty", func(t *testing.T) {
|
|
||||||
ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "")
|
|
||||||
assert.False(t, detect(ctx))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("non-empty cluster", func(t *testing.T) {
|
|
||||||
ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "15.4")
|
|
||||||
assert.True(t, detect(ctx))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("non-empty serverless", func(t *testing.T) {
|
|
||||||
ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "client.1.13")
|
|
||||||
assert.True(t, detect(ctx))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetect_Stat(t *testing.T) {
|
|
||||||
requireLinux(t)
|
|
||||||
|
|
||||||
// Configure other checks to pass.
|
|
||||||
ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "non-empty")
|
|
||||||
|
|
||||||
t.Run("error", func(t *testing.T) {
|
|
||||||
configureStatFunc(t, nil, fs.ErrNotExist)
|
|
||||||
assert.False(t, detect(ctx))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("not a directory", func(t *testing.T) {
|
|
||||||
configureStatFunc(t, fakefs.FileInfo{}, nil)
|
|
||||||
assert.False(t, detect(ctx))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("directory", func(t *testing.T) {
|
|
||||||
configureStatFunc(t, fakefs.FileInfo{FakeDir: true}, nil)
|
|
||||||
assert.True(t, detect(ctx))
|
|
||||||
})
|
|
||||||
}
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/textutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
// structInfo holds the type information we need to efficiently
|
// structInfo holds the type information we need to efficiently
|
||||||
|
@ -84,6 +85,14 @@ func buildStructInfo(typ reflect.Type) structInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
name, _, _ := strings.Cut(sf.Tag.Get("json"), ",")
|
name, _, _ := strings.Cut(sf.Tag.Get("json"), ",")
|
||||||
|
if typ.Name() == "QualityMonitor" && name == "-" {
|
||||||
|
urlName, _, _ := strings.Cut(sf.Tag.Get("url"), ",")
|
||||||
|
if urlName == "" || urlName == "-" {
|
||||||
|
name = textutil.CamelToSnakeCase(sf.Name)
|
||||||
|
} else {
|
||||||
|
name = urlName
|
||||||
|
}
|
||||||
|
}
|
||||||
if name == "" || name == "-" {
|
if name == "" || name == "-" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,87 +0,0 @@
|
||||||
package fakefs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var ErrNotImplemented = fmt.Errorf("not implemented")
|
|
||||||
|
|
||||||
// DirEntry is a fake implementation of [fs.DirEntry].
|
|
||||||
type DirEntry struct {
|
|
||||||
fs.FileInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (entry DirEntry) Type() fs.FileMode {
|
|
||||||
typ := fs.ModePerm
|
|
||||||
if entry.IsDir() {
|
|
||||||
typ |= fs.ModeDir
|
|
||||||
}
|
|
||||||
return typ
|
|
||||||
}
|
|
||||||
|
|
||||||
func (entry DirEntry) Info() (fs.FileInfo, error) {
|
|
||||||
return entry.FileInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileInfo is a fake implementation of [fs.FileInfo].
|
|
||||||
type FileInfo struct {
|
|
||||||
FakeName string
|
|
||||||
FakeSize int64
|
|
||||||
FakeDir bool
|
|
||||||
FakeMode fs.FileMode
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) Name() string {
|
|
||||||
return info.FakeName
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) Size() int64 {
|
|
||||||
return info.FakeSize
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) Mode() fs.FileMode {
|
|
||||||
return info.FakeMode
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) ModTime() time.Time {
|
|
||||||
return time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) IsDir() bool {
|
|
||||||
return info.FakeDir
|
|
||||||
}
|
|
||||||
|
|
||||||
func (info FileInfo) Sys() any {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// File is a fake implementation of [fs.File].
|
|
||||||
type File struct {
|
|
||||||
FileInfo fs.FileInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f File) Close() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f File) Read(p []byte) (n int, err error) {
|
|
||||||
return 0, ErrNotImplemented
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f File) Stat() (fs.FileInfo, error) {
|
|
||||||
return f.FileInfo, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FS is a fake implementation of [fs.FS].
|
|
||||||
type FS map[string]fs.File
|
|
||||||
|
|
||||||
func (f FS) Open(name string) (fs.File, error) {
|
|
||||||
e, ok := f[name]
|
|
||||||
if !ok {
|
|
||||||
return nil, fs.ErrNotExist
|
|
||||||
}
|
|
||||||
|
|
||||||
return e, nil
|
|
||||||
}
|
|
|
@ -1,38 +0,0 @@
|
||||||
package fakefs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io/fs"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFile(t *testing.T) {
|
|
||||||
var fakefile fs.File = File{
|
|
||||||
FileInfo: FileInfo{
|
|
||||||
FakeName: "file",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := fakefile.Read([]byte{})
|
|
||||||
assert.ErrorIs(t, err, ErrNotImplemented)
|
|
||||||
|
|
||||||
fi, err := fakefile.Stat()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, "file", fi.Name())
|
|
||||||
|
|
||||||
err = fakefile.Close()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFS(t *testing.T) {
|
|
||||||
var fakefs fs.FS = FS{
|
|
||||||
"file": File{},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := fakefs.Open("doesntexist")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
_, err = fakefs.Open("file")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/cmd/root"
|
"github.com/databricks/cli/cmd/root"
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/databricks/cli/libs/filer"
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
"github.com/databricks/databricks-sdk-go/experimental/mocks"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -18,7 +17,7 @@ func setupCompleter(t *testing.T, onlyDirs bool) *completer {
|
||||||
// Needed to make type context.valueCtx for mockFilerForPath
|
// Needed to make type context.valueCtx for mockFilerForPath
|
||||||
ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient)
|
ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient)
|
||||||
|
|
||||||
fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
|
fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
|
||||||
"dir": {FakeName: "root", FakeDir: true},
|
"dir": {FakeName: "root", FakeDir: true},
|
||||||
"dir/dirA": {FakeDir: true},
|
"dir/dirA": {FakeDir: true},
|
||||||
"dir/dirB": {FakeDir: true},
|
"dir/dirB": {FakeDir: true},
|
||||||
|
|
|
@ -8,12 +8,58 @@ import (
|
||||||
"path"
|
"path"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type FakeDirEntry struct {
|
||||||
|
FakeFileInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (entry FakeDirEntry) Type() fs.FileMode {
|
||||||
|
typ := fs.ModePerm
|
||||||
|
if entry.FakeDir {
|
||||||
|
typ |= fs.ModeDir
|
||||||
|
}
|
||||||
|
return typ
|
||||||
|
}
|
||||||
|
|
||||||
|
func (entry FakeDirEntry) Info() (fs.FileInfo, error) {
|
||||||
|
return entry.FakeFileInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type FakeFileInfo struct {
|
||||||
|
FakeName string
|
||||||
|
FakeSize int64
|
||||||
|
FakeDir bool
|
||||||
|
FakeMode fs.FileMode
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) Name() string {
|
||||||
|
return info.FakeName
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) Size() int64 {
|
||||||
|
return info.FakeSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) Mode() fs.FileMode {
|
||||||
|
return info.FakeMode
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) ModTime() time.Time {
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) IsDir() bool {
|
||||||
|
return info.FakeDir
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info FakeFileInfo) Sys() any {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type FakeFiler struct {
|
type FakeFiler struct {
|
||||||
entries map[string]fakefs.FileInfo
|
entries map[string]FakeFileInfo
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error {
|
func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error {
|
||||||
|
@ -51,7 +97,7 @@ func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
out = append(out, fakefs.DirEntry{FileInfo: v})
|
out = append(out, FakeDirEntry{v})
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
|
sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
|
||||||
|
@ -71,11 +117,7 @@ func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error)
|
||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFakeFiler creates a new fake [Filer] instance with the given entries.
|
func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler {
|
||||||
// It sets the [Name] field of each entry to the base name of the path.
|
|
||||||
//
|
|
||||||
// This is meant to be used in tests.
|
|
||||||
func NewFakeFiler(entries map[string]fakefs.FileInfo) *FakeFiler {
|
|
||||||
fakeFiler := &FakeFiler{
|
fakeFiler := &FakeFiler{
|
||||||
entries: entries,
|
entries: entries,
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,98 +0,0 @@
|
||||||
package filer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFakeFiler_Read(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"file": {},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
r, err := f.Read(ctx, "file")
|
|
||||||
require.NoError(t, err)
|
|
||||||
contents, err := io.ReadAll(r)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Contents of every file is "foo".
|
|
||||||
assert.Equal(t, "foo", string(contents))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_Read_NotFound(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"foo": {},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
_, err := f.Read(ctx, "bar")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_ReadDir_NotFound(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"dir1": {FakeDir: true},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
_, err := f.ReadDir(ctx, "dir2")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_ReadDir_NotADirectory(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"file": {},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
_, err := f.ReadDir(ctx, "file")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrInvalid)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_ReadDir(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"dir1": {FakeDir: true},
|
|
||||||
"dir1/file2": {},
|
|
||||||
"dir1/dir2": {FakeDir: true},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
entries, err := f.ReadDir(ctx, "dir1/")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, entries, 2)
|
|
||||||
|
|
||||||
// The entries are sorted by name.
|
|
||||||
assert.Equal(t, "dir2", entries[0].Name())
|
|
||||||
assert.True(t, entries[0].IsDir())
|
|
||||||
assert.Equal(t, "file2", entries[1].Name())
|
|
||||||
assert.False(t, entries[1].IsDir())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_Stat(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"file": {},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
info, err := f.Stat(ctx, "file")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, "file", info.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFakeFiler_Stat_NotFound(t *testing.T) {
|
|
||||||
f := NewFakeFiler(map[string]fakefs.FileInfo{
|
|
||||||
"foo": {},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
_, err := f.Stat(ctx, "bar")
|
|
||||||
assert.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
}
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -36,7 +35,7 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeFS() fs.FS {
|
func fakeFS() fs.FS {
|
||||||
fakeFiler := NewFakeFiler(map[string]fakefs.FileInfo{
|
fakeFiler := NewFakeFiler(map[string]FakeFileInfo{
|
||||||
".": {FakeName: "root", FakeDir: true},
|
".": {FakeName: "root", FakeDir: true},
|
||||||
"dirA": {FakeDir: true},
|
"dirA": {FakeDir: true},
|
||||||
"dirB": {FakeDir: true},
|
"dirB": {FakeDir: true},
|
||||||
|
|
|
@ -244,17 +244,6 @@ func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string,
|
||||||
|
|
||||||
// Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
|
// Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
|
||||||
func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
|
func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
|
||||||
// Ensure that the file / notebook exists. We do this check here to avoid reading
|
|
||||||
// the content of a notebook called `foo` when the user actually wanted
|
|
||||||
// to read the content of a file called `foo`.
|
|
||||||
//
|
|
||||||
// To read the content of a notebook called `foo` in the workspace the user
|
|
||||||
// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
|
|
||||||
_, err := w.Stat(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
r, err := w.wsfs.Read(ctx, name)
|
r, err := w.wsfs.Read(ctx, name)
|
||||||
|
|
||||||
// If the file is not found, it might be a notebook.
|
// If the file is not found, it might be a notebook.
|
||||||
|
@ -287,18 +276,7 @@ func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string
|
||||||
return ReadOnlyError{"delete"}
|
return ReadOnlyError{"delete"}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure that the file / notebook exists. We do this check here to avoid
|
err := w.wsfs.Delete(ctx, name, mode...)
|
||||||
// deleting the a notebook called `foo` when the user actually wanted to
|
|
||||||
// delete a file called `foo`.
|
|
||||||
//
|
|
||||||
// To delete a notebook called `foo` in the workspace the user should use the
|
|
||||||
// name with the extension included like `foo.ipynb` or `foo.sql`.
|
|
||||||
_, err := w.Stat(ctx, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = w.wsfs.Delete(ctx, name, mode...)
|
|
||||||
|
|
||||||
// If the file is not found, it might be a notebook.
|
// If the file is not found, it might be a notebook.
|
||||||
if errors.As(err, &FileDoesNotExistError{}) {
|
if errors.As(err, &FileDoesNotExistError{}) {
|
||||||
|
@ -337,24 +315,7 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string)
|
||||||
return wsfsFileInfo{ObjectInfo: stat.ObjectInfo}, nil
|
return wsfsFileInfo{ObjectInfo: stat.ObjectInfo}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
return info, err
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// If an object is found and it is a notebook, return a FileDoesNotExistError.
|
|
||||||
// If a notebook is found by the workspace files client, without having stripped
|
|
||||||
// the extension, this implies that no file with the same name exists.
|
|
||||||
//
|
|
||||||
// This check is done to avoid returning the stat for a notebook called `foo`
|
|
||||||
// when the user actually wanted to stat a file called `foo`.
|
|
||||||
//
|
|
||||||
// To stat the metadata of a notebook called `foo` in the workspace the user
|
|
||||||
// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
|
|
||||||
if info.Sys().(workspace.ObjectInfo).ObjectType == workspace.ObjectTypeNotebook {
|
|
||||||
return nil, FileDoesNotExistError{name}
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Note: The import API returns opaque internal errors for namespace clashes
|
// Note: The import API returns opaque internal errors for namespace clashes
|
||||||
|
|
|
@ -3,9 +3,7 @@ package jsonschema
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
|
@ -257,12 +255,7 @@ func (schema *Schema) validate() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load(path string) (*Schema, error) {
|
func Load(path string) (*Schema, error) {
|
||||||
dir, file := filepath.Split(path)
|
b, err := os.ReadFile(path)
|
||||||
return LoadFS(os.DirFS(dir), file)
|
|
||||||
}
|
|
||||||
|
|
||||||
func LoadFS(fsys fs.FS, path string) (*Schema, error) {
|
|
||||||
b, err := fs.ReadFile(fsys, path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package jsonschema
|
package jsonschema
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
@ -306,9 +305,3 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) {
|
||||||
err = s.validate()
|
err = s.validate()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSchema_LoadFS(t *testing.T) {
|
|
||||||
fsys := os.DirFS("./testdata/schema-load-int")
|
|
||||||
_, err := LoadFS(fsys, "schema-valid.json")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/fakefs"
|
|
||||||
"github.com/databricks/databricks-sdk-go/service/workspace"
|
"github.com/databricks/databricks-sdk-go/service/workspace"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
@ -101,21 +100,11 @@ func TestDetectFileWithLongHeader(t *testing.T) {
|
||||||
assert.False(t, nb)
|
assert.False(t, nb)
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileInfoWithWorkspaceInfo struct {
|
|
||||||
fakefs.FileInfo
|
|
||||||
|
|
||||||
oi workspace.ObjectInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f fileInfoWithWorkspaceInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
|
|
||||||
return f.oi
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetectWithObjectInfo(t *testing.T) {
|
func TestDetectWithObjectInfo(t *testing.T) {
|
||||||
fakefs := fakefs.FS{
|
fakeFS := &fakeFS{
|
||||||
"file.py": fakefs.File{
|
fakeFile{
|
||||||
FileInfo: fileInfoWithWorkspaceInfo{
|
fakeFileInfo{
|
||||||
oi: workspace.ObjectInfo{
|
workspace.ObjectInfo{
|
||||||
ObjectType: workspace.ObjectTypeNotebook,
|
ObjectType: workspace.ObjectTypeNotebook,
|
||||||
Language: workspace.LanguagePython,
|
Language: workspace.LanguagePython,
|
||||||
},
|
},
|
||||||
|
@ -123,7 +112,7 @@ func TestDetectWithObjectInfo(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
nb, lang, err := DetectWithFS(fakefs, "file.py")
|
nb, lang, err := DetectWithFS(fakeFS, "doesntmatter")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.True(t, nb)
|
assert.True(t, nb)
|
||||||
assert.Equal(t, workspace.LanguagePython, lang)
|
assert.Equal(t, workspace.LanguagePython, lang)
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
package notebook
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/workspace"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeFS struct {
|
||||||
|
fakeFile
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeFile struct {
|
||||||
|
fakeFileInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFile) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFile) Read(p []byte) (n int, err error) {
|
||||||
|
return 0, fmt.Errorf("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFile) Stat() (fs.FileInfo, error) {
|
||||||
|
return f.fakeFileInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeFileInfo struct {
|
||||||
|
oi workspace.ObjectInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
|
||||||
|
return f.oi
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) Name() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) Size() int64 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) Mode() fs.FileMode {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) ModTime() time.Time {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) IsDir() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFileInfo) Sys() any {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFS) Open(name string) (fs.File, error) {
|
||||||
|
return f.fakeFile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFS) Stat(name string) (fs.FileInfo, error) {
|
||||||
|
panic("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||||
|
panic("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeFS) ReadFile(name string) ([]byte, error) {
|
||||||
|
panic("not implemented")
|
||||||
|
}
|
|
@ -1,47 +0,0 @@
|
||||||
package template
|
|
||||||
|
|
||||||
import (
|
|
||||||
"embed"
|
|
||||||
"io/fs"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:embed all:templates
|
|
||||||
var builtinTemplates embed.FS
|
|
||||||
|
|
||||||
// BuiltinTemplate represents a template that is built into the CLI.
|
|
||||||
type BuiltinTemplate struct {
|
|
||||||
Name string
|
|
||||||
FS fs.FS
|
|
||||||
}
|
|
||||||
|
|
||||||
// Builtin returns the list of all built-in templates.
|
|
||||||
func Builtin() ([]BuiltinTemplate, error) {
|
|
||||||
templates, err := fs.Sub(builtinTemplates, "templates")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
entries, err := fs.ReadDir(templates, ".")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var out []BuiltinTemplate
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !entry.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
templateFS, err := fs.Sub(templates, entry.Name())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
out = append(out, BuiltinTemplate{
|
|
||||||
Name: entry.Name(),
|
|
||||||
FS: templateFS,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return out, nil
|
|
||||||
}
|
|
|
@ -1,28 +0,0 @@
|
||||||
package template
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io/fs"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuiltin(t *testing.T) {
|
|
||||||
out, err := Builtin()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, out, 3)
|
|
||||||
|
|
||||||
// Confirm names.
|
|
||||||
assert.Equal(t, "dbt-sql", out[0].Name)
|
|
||||||
assert.Equal(t, "default-python", out[1].Name)
|
|
||||||
assert.Equal(t, "default-sql", out[2].Name)
|
|
||||||
|
|
||||||
// Confirm that the filesystems work.
|
|
||||||
_, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
_, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
_, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/cmdio"
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
"github.com/databricks/cli/libs/jsonschema"
|
"github.com/databricks/cli/libs/jsonschema"
|
||||||
|
@ -29,8 +28,9 @@ type config struct {
|
||||||
schema *jsonschema.Schema
|
schema *jsonschema.Schema
|
||||||
}
|
}
|
||||||
|
|
||||||
func newConfig(ctx context.Context, templateFS fs.FS, schemaPath string) (*config, error) {
|
func newConfig(ctx context.Context, schemaPath string) (*config, error) {
|
||||||
schema, err := jsonschema.LoadFS(templateFS, schemaPath)
|
// Read config schema
|
||||||
|
schema, err := jsonschema.Load(schemaPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,8 +3,6 @@ package template
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
@ -18,7 +16,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) {
|
||||||
testDir := "./testdata/config-assign-from-file"
|
testDir := "./testdata/config-assign-from-file"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||||
|
@ -34,7 +32,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
|
||||||
testDir := "./testdata/config-assign-from-file"
|
testDir := "./testdata/config-assign-from-file"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
c.values = map[string]any{
|
c.values = map[string]any{
|
||||||
|
@ -54,7 +52,7 @@ func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T)
|
||||||
testDir := "./testdata/config-assign-from-file-invalid-int"
|
testDir := "./testdata/config-assign-from-file-invalid-int"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||||
|
@ -65,7 +63,7 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te
|
||||||
testDir := "./testdata/config-assign-from-file-unknown-property"
|
testDir := "./testdata/config-assign-from-file-unknown-property"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||||
|
@ -80,10 +78,10 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) {
|
||||||
testDir := "./testdata/config-assign-from-default-value"
|
testDir := "./testdata/config-assign-from-default-value"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
|
r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = c.assignDefaultValues(r)
|
err = c.assignDefaultValues(r)
|
||||||
|
@ -99,10 +97,10 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
|
||||||
testDir := "./testdata/config-assign-from-templated-default-value"
|
testDir := "./testdata/config-assign-from-templated-default-value"
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library"), t.TempDir())
|
r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir())
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Note: only the string value is templated.
|
// Note: only the string value is templated.
|
||||||
|
@ -118,7 +116,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
|
||||||
|
|
||||||
func TestTemplateConfigValidateValuesDefined(t *testing.T) {
|
func TestTemplateConfigValidateValuesDefined(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
c.values = map[string]any{
|
c.values = map[string]any{
|
||||||
|
@ -133,7 +131,7 @@ func TestTemplateConfigValidateValuesDefined(t *testing.T) {
|
||||||
|
|
||||||
func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
|
func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
c.values = map[string]any{
|
c.values = map[string]any{
|
||||||
|
@ -149,7 +147,7 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
|
||||||
|
|
||||||
func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
|
func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
c.values = map[string]any{
|
c.values = map[string]any{
|
||||||
|
@ -166,7 +164,7 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
|
||||||
|
|
||||||
func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) {
|
func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
c.values = map[string]any{
|
c.values = map[string]any{
|
||||||
|
@ -273,8 +271,7 @@ func TestTemplateEnumValidation(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) {
|
func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) {
|
||||||
ctx := context.Background()
|
_, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json")
|
||||||
_, err := newConfig(ctx, os.DirFS("./testdata/config-test-schema"), "invalid-test-schema.json")
|
|
||||||
assert.EqualError(t, err, "template property property-without-description is missing a description")
|
assert.EqualError(t, err, "template property property-without-description is missing a description")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,8 @@ import (
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Interface representing a file to be materialized from a template into a project
|
// Interface representing a file to be materialized from a template into a project
|
||||||
|
@ -18,10 +19,6 @@ type file interface {
|
||||||
|
|
||||||
// Write file to disk at the destination path.
|
// Write file to disk at the destination path.
|
||||||
PersistToDisk() error
|
PersistToDisk() error
|
||||||
|
|
||||||
// contents returns the file contents as a byte slice.
|
|
||||||
// This is used for testing purposes.
|
|
||||||
contents() ([]byte, error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type destinationPath struct {
|
type destinationPath struct {
|
||||||
|
@ -49,8 +46,8 @@ type copyFile struct {
|
||||||
|
|
||||||
dstPath *destinationPath
|
dstPath *destinationPath
|
||||||
|
|
||||||
// [fs.FS] rooted at template root. Used to read srcPath.
|
// Filer rooted at template root. Used to read srcPath.
|
||||||
srcFS fs.FS
|
srcFiler filer.Filer
|
||||||
|
|
||||||
// Relative path from template root for file to be copied.
|
// Relative path from template root for file to be copied.
|
||||||
srcPath string
|
srcPath string
|
||||||
|
@ -66,7 +63,7 @@ func (f *copyFile) PersistToDisk() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
srcFile, err := f.srcFS.Open(f.srcPath)
|
srcFile, err := f.srcFiler.Read(f.ctx, f.srcPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -80,10 +77,6 @@ func (f *copyFile) PersistToDisk() error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *copyFile) contents() ([]byte, error) {
|
|
||||||
return fs.ReadFile(f.srcFS, f.srcPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
type inMemoryFile struct {
|
type inMemoryFile struct {
|
||||||
dstPath *destinationPath
|
dstPath *destinationPath
|
||||||
|
|
||||||
|
@ -106,7 +99,3 @@ func (f *inMemoryFile) PersistToDisk() error {
|
||||||
}
|
}
|
||||||
return os.WriteFile(path, f.content, f.perm)
|
return os.WriteFile(path, f.content, f.perm)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *inMemoryFile) contents() ([]byte, error) {
|
|
||||||
return slices.Clone(f.content), nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"runtime"
|
"runtime"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -32,7 +33,10 @@ func testInMemoryFile(t *testing.T, perm fs.FileMode) {
|
||||||
|
|
||||||
func testCopyFile(t *testing.T, perm fs.FileMode) {
|
func testCopyFile(t *testing.T, perm fs.FileMode) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
err := os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
|
|
||||||
|
templateFiler, err := filer.NewLocalClient(tmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
err = os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
f := ©File{
|
f := ©File{
|
||||||
|
@ -43,7 +47,7 @@ func testCopyFile(t *testing.T, perm fs.FileMode) {
|
||||||
},
|
},
|
||||||
perm: perm,
|
perm: perm,
|
||||||
srcPath: "source",
|
srcPath: "source",
|
||||||
srcFS: os.DirFS(tmpDir),
|
srcFiler: templateFiler,
|
||||||
}
|
}
|
||||||
err = f.PersistToDisk()
|
err = f.PersistToDisk()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
|
@ -22,7 +22,7 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -39,7 +39,7 @@ func TestTemplateRegexpCompileFunction(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -57,7 +57,7 @@ func TestTemplateRandIntFunction(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -75,7 +75,7 @@ func TestTemplateUuidFunction(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -92,7 +92,7 @@ func TestTemplateUrlFunction(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -109,7 +109,7 @@ func TestTemplateMapPairFunction(t *testing.T) {
|
||||||
|
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -132,7 +132,7 @@ func TestWorkspaceHost(t *testing.T) {
|
||||||
ctx = root.SetWorkspaceClient(ctx, w)
|
ctx = root.SetWorkspaceClient(ctx, w)
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
@ -157,7 +157,7 @@ func TestWorkspaceHostNotConfigured(t *testing.T) {
|
||||||
ctx = root.SetWorkspaceClient(ctx, w)
|
ctx = root.SetWorkspaceClient(ctx, w)
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||||
|
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
|
|
@ -2,9 +2,13 @@ package template
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"embed"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/cmdio"
|
"github.com/databricks/cli/libs/cmdio"
|
||||||
)
|
)
|
||||||
|
@ -13,20 +17,39 @@ const libraryDirName = "library"
|
||||||
const templateDirName = "template"
|
const templateDirName = "template"
|
||||||
const schemaFileName = "databricks_template_schema.json"
|
const schemaFileName = "databricks_template_schema.json"
|
||||||
|
|
||||||
|
//go:embed all:templates
|
||||||
|
var builtinTemplates embed.FS
|
||||||
|
|
||||||
// This function materializes the input templates as a project, using user defined
|
// This function materializes the input templates as a project, using user defined
|
||||||
// configurations.
|
// configurations.
|
||||||
// Parameters:
|
// Parameters:
|
||||||
//
|
//
|
||||||
// ctx: context containing a cmdio object. This is used to prompt the user
|
// ctx: context containing a cmdio object. This is used to prompt the user
|
||||||
// configFilePath: file path containing user defined config values
|
// configFilePath: file path containing user defined config values
|
||||||
// templateFS: root of the template definition
|
// templateRoot: root of the template definition
|
||||||
// outputDir: root of directory where to initialize the template
|
// outputDir: root of directory where to initialize the template
|
||||||
func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, outputDir string) error {
|
func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir string) error {
|
||||||
if _, err := fs.Stat(templateFS, schemaFileName); errors.Is(err, fs.ErrNotExist) {
|
// Use a temporary directory in case any builtin templates like default-python are used
|
||||||
return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
|
tempDir, err := os.MkdirTemp("", "templates")
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
templateRoot, err = prepareBuiltinTemplates(templateRoot, tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
config, err := newConfig(ctx, templateFS, schemaFileName)
|
templatePath := filepath.Join(templateRoot, templateDirName)
|
||||||
|
libraryPath := filepath.Join(templateRoot, libraryDirName)
|
||||||
|
schemaPath := filepath.Join(templateRoot, schemaFileName)
|
||||||
|
helpers := loadHelpers(ctx)
|
||||||
|
|
||||||
|
if _, err := os.Stat(schemaPath); errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
config, err := newConfig(ctx, schemaPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -39,8 +62,7 @@ func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, o
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
|
||||||
r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName, outputDir)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -89,3 +111,44 @@ func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, o
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the given templateRoot matches
|
||||||
|
func prepareBuiltinTemplates(templateRoot string, tempDir string) (string, error) {
|
||||||
|
// Check that `templateRoot` is a clean basename, i.e. `some_path` and not `./some_path` or "."
|
||||||
|
// Return early if that's not the case.
|
||||||
|
if templateRoot == "." || path.Base(templateRoot) != templateRoot {
|
||||||
|
return templateRoot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := fs.Stat(builtinTemplates, path.Join("templates", templateRoot))
|
||||||
|
if err != nil {
|
||||||
|
// The given path doesn't appear to be using out built-in templates
|
||||||
|
return templateRoot, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// We have a built-in template with the same name as templateRoot!
|
||||||
|
// Now we need to make a fully copy of the builtin templates to a real file system
|
||||||
|
// since template.Parse() doesn't support embed.FS.
|
||||||
|
err = fs.WalkDir(builtinTemplates, "templates", func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
targetPath := filepath.Join(tempDir, path)
|
||||||
|
if entry.IsDir() {
|
||||||
|
return os.Mkdir(targetPath, 0755)
|
||||||
|
} else {
|
||||||
|
content, err := fs.ReadFile(builtinTemplates, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(targetPath, content, 0644)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(tempDir, "templates", templateRoot), nil
|
||||||
|
}
|
||||||
|
|
|
@ -3,7 +3,7 @@ package template
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/databricks/cli/cmd/root"
|
"github.com/databricks/cli/cmd/root"
|
||||||
|
@ -19,6 +19,6 @@ func TestMaterializeForNonTemplateDirectory(t *testing.T) {
|
||||||
ctx := root.SetWorkspaceClient(context.Background(), w)
|
ctx := root.SetWorkspaceClient(context.Background(), w)
|
||||||
|
|
||||||
// Try to materialize a non-template directory.
|
// Try to materialize a non-template directory.
|
||||||
err = Materialize(ctx, "", os.DirFS(tmpDir), "")
|
err = Materialize(ctx, "", tmpDir, "")
|
||||||
assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", schemaFileName))
|
assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", filepath.Join(tmpDir, schemaFileName)))
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,12 +8,14 @@ import (
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/libs/filer"
|
||||||
"github.com/databricks/cli/libs/log"
|
"github.com/databricks/cli/libs/log"
|
||||||
"github.com/databricks/databricks-sdk-go/logger"
|
"github.com/databricks/databricks-sdk-go/logger"
|
||||||
)
|
)
|
||||||
|
@ -50,42 +52,32 @@ type renderer struct {
|
||||||
// do not match any glob patterns from this list
|
// do not match any glob patterns from this list
|
||||||
skipPatterns []string
|
skipPatterns []string
|
||||||
|
|
||||||
// [fs.FS] that holds the template's file tree.
|
// Filer rooted at template root. The file tree from this root is walked to
|
||||||
srcFS fs.FS
|
// generate the project
|
||||||
|
templateFiler filer.Filer
|
||||||
|
|
||||||
// Root directory for the project instantiated from the template
|
// Root directory for the project instantiated from the template
|
||||||
instanceRoot string
|
instanceRoot string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRenderer(
|
func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
|
||||||
ctx context.Context,
|
|
||||||
config map[string]any,
|
|
||||||
helpers template.FuncMap,
|
|
||||||
templateFS fs.FS,
|
|
||||||
templateDir string,
|
|
||||||
libraryDir string,
|
|
||||||
instanceRoot string,
|
|
||||||
) (*renderer, error) {
|
|
||||||
// Initialize new template, with helper functions loaded
|
// Initialize new template, with helper functions loaded
|
||||||
tmpl := template.New("").Funcs(helpers)
|
tmpl := template.New("").Funcs(helpers)
|
||||||
|
|
||||||
// Find user-defined templates in the library directory
|
// Load user defined associated templates from the library root
|
||||||
matches, err := fs.Glob(templateFS, path.Join(libraryDir, "*"))
|
libraryGlob := filepath.Join(libraryRoot, "*")
|
||||||
|
matches, err := filepath.Glob(libraryGlob)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse user-defined templates.
|
|
||||||
// Note: we do not call [ParseFS] with the glob directly because
|
|
||||||
// it returns an error if no files match the pattern.
|
|
||||||
if len(matches) != 0 {
|
if len(matches) != 0 {
|
||||||
tmpl, err = tmpl.ParseFS(templateFS, matches...)
|
tmpl, err = tmpl.ParseFiles(matches...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
srcFS, err := fs.Sub(templateFS, path.Clean(templateDir))
|
templateFiler, err := filer.NewLocalClient(templateRoot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -98,7 +90,7 @@ func newRenderer(
|
||||||
baseTemplate: tmpl,
|
baseTemplate: tmpl,
|
||||||
files: make([]file, 0),
|
files: make([]file, 0),
|
||||||
skipPatterns: make([]string, 0),
|
skipPatterns: make([]string, 0),
|
||||||
srcFS: srcFS,
|
templateFiler: templateFiler,
|
||||||
instanceRoot: instanceRoot,
|
instanceRoot: instanceRoot,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
@ -149,7 +141,7 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
|
||||||
|
|
||||||
func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
||||||
// read file permissions
|
// read file permissions
|
||||||
info, err := fs.Stat(r.srcFS, relPathTemplate)
|
info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -171,8 +163,8 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
||||||
},
|
},
|
||||||
perm: perm,
|
perm: perm,
|
||||||
ctx: r.ctx,
|
ctx: r.ctx,
|
||||||
srcFS: r.srcFS,
|
|
||||||
srcPath: relPathTemplate,
|
srcPath: relPathTemplate,
|
||||||
|
srcFiler: r.templateFiler,
|
||||||
}, nil
|
}, nil
|
||||||
} else {
|
} else {
|
||||||
// Trim the .tmpl suffix from file name, if specified in the template
|
// Trim the .tmpl suffix from file name, if specified in the template
|
||||||
|
@ -181,7 +173,7 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// read template file's content
|
// read template file's content
|
||||||
templateReader, err := r.srcFS.Open(relPathTemplate)
|
templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -271,7 +263,7 @@ func (r *renderer) walk() error {
|
||||||
//
|
//
|
||||||
// 2. For directories: They are appended to a slice, which acts as a queue
|
// 2. For directories: They are appended to a slice, which acts as a queue
|
||||||
// allowing BFS traversal of the template file tree
|
// allowing BFS traversal of the template file tree
|
||||||
entries, err := fs.ReadDir(r.srcFS, currentDirectory)
|
entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,9 +3,9 @@ package template
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -41,8 +41,9 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
|
||||||
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
|
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
|
templatePath, err := prepareBuiltinTemplates(template, tempDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
libraryPath := filepath.Join(templatePath, "library")
|
||||||
|
|
||||||
w := &databricks.WorkspaceClient{
|
w := &databricks.WorkspaceClient{
|
||||||
Config: &workspaceConfig.Config{Host: "https://myhost.com"},
|
Config: &workspaceConfig.Config{Host: "https://myhost.com"},
|
||||||
|
@ -57,7 +58,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
||||||
ctx = root.SetWorkspaceClient(ctx, w)
|
ctx = root.SetWorkspaceClient(ctx, w)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
|
|
||||||
renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName, tempDir)
|
renderer, err := newRenderer(ctx, settings, helpers, templatePath, libraryPath, tempDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// Evaluate template
|
// Evaluate template
|
||||||
|
@ -66,7 +67,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
||||||
err = renderer.persistToDisk()
|
err = renderer.persistToDisk()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project"))
|
b, err := bundle.Load(ctx, filepath.Join(tempDir, "template", "my_project"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
|
diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
|
||||||
require.NoError(t, diags.Error())
|
require.NoError(t, diags.Error())
|
||||||
|
@ -95,6 +96,18 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrepareBuiltInTemplatesWithRelativePaths(t *testing.T) {
|
||||||
|
// CWD should not be resolved as a built in template
|
||||||
|
dir, err := prepareBuiltinTemplates(".", t.TempDir())
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, ".", dir)
|
||||||
|
|
||||||
|
// relative path should not be resolved as a built in template
|
||||||
|
dir, err = prepareBuiltinTemplates("./default-python", t.TempDir())
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "./default-python", dir)
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuiltinPythonTemplateValid(t *testing.T) {
|
func TestBuiltinPythonTemplateValid(t *testing.T) {
|
||||||
// Test option combinations
|
// Test option combinations
|
||||||
options := []string{"yes", "no"}
|
options := []string{"yes", "no"}
|
||||||
|
@ -181,7 +194,7 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/email/template", "./testdata/email/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -368,7 +381,7 @@ func TestRendererWalk(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/walk/template", "./testdata/walk/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -379,9 +392,18 @@ func TestRendererWalk(t *testing.T) {
|
||||||
if f.DstPath().relPath != path {
|
if f.DstPath().relPath != path {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
b, err := f.contents()
|
switch v := f.(type) {
|
||||||
|
case *inMemoryFile:
|
||||||
|
return strings.Trim(string(v.content), "\r\n")
|
||||||
|
case *copyFile:
|
||||||
|
r, err := r.templateFiler.Read(context.Background(), v.srcPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
b, err := io.ReadAll(r)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
return strings.Trim(string(b), "\r\n")
|
return strings.Trim(string(b), "\r\n")
|
||||||
|
default:
|
||||||
|
require.FailNow(t, "execution should not reach here")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
require.FailNow(t, "file is absent: "+path)
|
require.FailNow(t, "file is absent: "+path)
|
||||||
return ""
|
return ""
|
||||||
|
@ -400,7 +422,7 @@ func TestRendererFailFunction(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/fail/template", "./testdata/fail/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -413,7 +435,7 @@ func TestRendererSkipsDirsEagerly(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -430,7 +452,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -453,7 +475,7 @@ func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -471,7 +493,7 @@ func TestRendererSkip(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip/template", "./testdata/skip/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -503,7 +525,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
|
||||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -593,7 +615,7 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
|
|
||||||
helpers := loadHelpers(ctx)
|
helpers := loadHelpers(ctx)
|
||||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
|
r, err := newRenderer(ctx, nil, helpers, "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -613,7 +635,7 @@ func TestRendererFileTreeRendering(t *testing.T) {
|
||||||
r, err := newRenderer(ctx, map[string]any{
|
r, err := newRenderer(ctx, map[string]any{
|
||||||
"dir_name": "my_directory",
|
"dir_name": "my_directory",
|
||||||
"file_name": "my_file",
|
"file_name": "my_file",
|
||||||
}, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
|
}, helpers, "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
@ -646,7 +668,7 @@ func TestRendererSubTemplateInPath(t *testing.T) {
|
||||||
testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}"))
|
testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}"))
|
||||||
|
|
||||||
tmpDir := t.TempDir()
|
tmpDir := t.TempDir()
|
||||||
r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library", tmpDir)
|
r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
err = r.walk()
|
err = r.walk()
|
||||||
|
|
Loading…
Reference in New Issue