mirror of https://github.com/databricks/cli.git
Use `fs.FS` interface to read template (#1910)
## Changes While working on the v2 of #1744, I found that: * Template initialization first copies built-in templates to a temporary directory before initializing them * Reading a template's contents goes through a `filer.Filer` but is hardcoded to a local one This change updates the interface for reading templates to be `fs.FS`. This is compatible with the `embed.FS` type for the built-in templates, so they no longer have to be copied to a temporary directory before being used. The alternative is to use a `filer.Filer` throughout, but this would have required even more plumbing, and we don't need to _read_ templates, including notebooks, from the workspace filesystem (yet?). As part of making `template.Materialize` take an `fs.FS` argument, the logic to match a given argument to a particular built-in template in the `init` command has moved to sit next to its implementation. ## Tests Existing tests pass.
This commit is contained in:
Pieter Noordhuis
GitHub
parent
72dde793d8
commit
4fea0219fd
|
|
@ -3,6 +3,7 @@ package bundle
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
|
@ -109,6 +110,24 @@ func getUrlForNativeTemplate(name string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
func getFsForNativeTemplate(name string) (fs.FS, error) {
|
||||
builtin, err := template.Builtin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If this is a built-in template, the return value will be non-nil.
|
||||
var templateFS fs.FS
|
||||
for _, entry := range builtin {
|
||||
if entry.Name == name {
|
||||
templateFS = entry.FS
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return templateFS, nil
|
||||
}
|
||||
|
||||
func isRepoUrl(url string) bool {
|
||||
result := false
|
||||
for _, prefix := range gitUrlPrefixes {
|
||||
|
|
@ -198,9 +217,20 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
|
|||
if templateDir != "" {
|
||||
return errors.New("--template-dir can only be used with a Git repository URL")
|
||||
}
|
||||
|
||||
templateFS, err := getFsForNativeTemplate(templatePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If this is not a built-in template, then it must be a local file system path.
|
||||
if templateFS == nil {
|
||||
templateFS = os.DirFS(templatePath)
|
||||
}
|
||||
|
||||
// skip downloading the repo because input arg is not a URL. We assume
|
||||
// it's a path on the local file system in that case
|
||||
return template.Materialize(ctx, configFile, templatePath, outputDir)
|
||||
return template.Materialize(ctx, configFile, templateFS, outputDir)
|
||||
}
|
||||
|
||||
// Create a temporary directory with the name of the repository. The '*'
|
||||
|
|
@ -224,7 +254,8 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
|
|||
|
||||
// Clean up downloaded repository once the template is materialized.
|
||||
defer os.RemoveAll(repoDir)
|
||||
return template.Materialize(ctx, configFile, filepath.Join(repoDir, templateDir), outputDir)
|
||||
templateFS := os.DirFS(filepath.Join(repoDir, templateDir))
|
||||
return template.Materialize(ctx, configFile, templateFS, outputDir)
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateN
|
|||
cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
|
||||
ctx = cmdio.InContext(ctx, cmd)
|
||||
|
||||
err = template.Materialize(ctx, configFilePath, templateRoot, bundleRoot)
|
||||
err = template.Materialize(ctx, configFilePath, os.DirFS(templateRoot), bundleRoot)
|
||||
return bundleRoot, err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ package jsonschema
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
|
||||
|
|
@ -255,7 +257,12 @@ func (schema *Schema) validate() error {
|
|||
}
|
||||
|
||||
func Load(path string) (*Schema, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
dir, file := filepath.Split(path)
|
||||
return LoadFS(os.DirFS(dir), file)
|
||||
}
|
||||
|
||||
func LoadFS(fsys fs.FS, path string) (*Schema, error) {
|
||||
b, err := fs.ReadFile(fsys, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package jsonschema
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
|
@ -305,3 +306,9 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) {
|
|||
err = s.validate()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSchema_LoadFS(t *testing.T) {
|
||||
fsys := os.DirFS("./testdata/schema-load-int")
|
||||
_, err := LoadFS(fsys, "schema-valid.json")
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
package template
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
//go:embed all:templates
|
||||
var builtinTemplates embed.FS
|
||||
|
||||
// BuiltinTemplate represents a template that is built into the CLI.
|
||||
type BuiltinTemplate struct {
|
||||
Name string
|
||||
FS fs.FS
|
||||
}
|
||||
|
||||
// Builtin returns the list of all built-in templates.
|
||||
func Builtin() ([]BuiltinTemplate, error) {
|
||||
templates, err := fs.Sub(builtinTemplates, "templates")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries, err := fs.ReadDir(templates, ".")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []BuiltinTemplate
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
templateFS, err := fs.Sub(templates, entry.Name())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = append(out, BuiltinTemplate{
|
||||
Name: entry.Name(),
|
||||
FS: templateFS,
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package template
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuiltin(t *testing.T) {
|
||||
out, err := Builtin()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, out, 3)
|
||||
|
||||
// Confirm names.
|
||||
assert.Equal(t, "dbt-sql", out[0].Name)
|
||||
assert.Equal(t, "default-python", out[1].Name)
|
||||
assert.Equal(t, "default-sql", out[2].Name)
|
||||
|
||||
// Confirm that the filesystems work.
|
||||
_, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
|
||||
assert.NoError(t, err)
|
||||
_, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
|
||||
assert.NoError(t, err)
|
||||
_, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
|
||||
"github.com/databricks/cli/libs/cmdio"
|
||||
"github.com/databricks/cli/libs/jsonschema"
|
||||
|
|
@ -28,9 +29,8 @@ type config struct {
|
|||
schema *jsonschema.Schema
|
||||
}
|
||||
|
||||
func newConfig(ctx context.Context, schemaPath string) (*config, error) {
|
||||
// Read config schema
|
||||
schema, err := jsonschema.Load(schemaPath)
|
||||
func newConfig(ctx context.Context, templateFS fs.FS, schemaPath string) (*config, error) {
|
||||
schema, err := jsonschema.LoadFS(templateFS, schemaPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ package template
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
|
@ -16,7 +18,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) {
|
|||
testDir := "./testdata/config-assign-from-file"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||
|
|
@ -32,7 +34,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
|
|||
testDir := "./testdata/config-assign-from-file"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
c.values = map[string]any{
|
||||
|
|
@ -52,7 +54,7 @@ func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T)
|
|||
testDir := "./testdata/config-assign-from-file-invalid-int"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||
|
|
@ -63,7 +65,7 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te
|
|||
testDir := "./testdata/config-assign-from-file-unknown-property"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
|
||||
|
|
@ -78,10 +80,10 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) {
|
|||
testDir := "./testdata/config-assign-from-default-value"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
|
||||
r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
err = c.assignDefaultValues(r)
|
||||
|
|
@ -97,10 +99,10 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
|
|||
testDir := "./testdata/config-assign-from-templated-default-value"
|
||||
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
|
||||
c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir())
|
||||
r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library"), t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Note: only the string value is templated.
|
||||
|
|
@ -116,7 +118,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
|
|||
|
||||
func TestTemplateConfigValidateValuesDefined(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
c.values = map[string]any{
|
||||
|
|
@ -131,7 +133,7 @@ func TestTemplateConfigValidateValuesDefined(t *testing.T) {
|
|||
|
||||
func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
c.values = map[string]any{
|
||||
|
|
@ -147,7 +149,7 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
|
|||
|
||||
func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
c.values = map[string]any{
|
||||
|
|
@ -164,7 +166,7 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
|
|||
|
||||
func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
|
||||
c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
c.values = map[string]any{
|
||||
|
|
@ -271,7 +273,8 @@ func TestTemplateEnumValidation(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) {
|
||||
_, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json")
|
||||
ctx := context.Background()
|
||||
_, err := newConfig(ctx, os.DirFS("./testdata/config-test-schema"), "invalid-test-schema.json")
|
||||
assert.EqualError(t, err, "template property property-without-description is missing a description")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ import (
|
|||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/databricks/cli/libs/filer"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// Interface representing a file to be materialized from a template into a project
|
||||
|
|
@ -19,6 +18,10 @@ type file interface {
|
|||
|
||||
// Write file to disk at the destination path.
|
||||
PersistToDisk() error
|
||||
|
||||
// contents returns the file contents as a byte slice.
|
||||
// This is used for testing purposes.
|
||||
contents() ([]byte, error)
|
||||
}
|
||||
|
||||
type destinationPath struct {
|
||||
|
|
@ -46,8 +49,8 @@ type copyFile struct {
|
|||
|
||||
dstPath *destinationPath
|
||||
|
||||
// Filer rooted at template root. Used to read srcPath.
|
||||
srcFiler filer.Filer
|
||||
// [fs.FS] rooted at template root. Used to read srcPath.
|
||||
srcFS fs.FS
|
||||
|
||||
// Relative path from template root for file to be copied.
|
||||
srcPath string
|
||||
|
|
@ -63,7 +66,7 @@ func (f *copyFile) PersistToDisk() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcFile, err := f.srcFiler.Read(f.ctx, f.srcPath)
|
||||
srcFile, err := f.srcFS.Open(f.srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -77,6 +80,10 @@ func (f *copyFile) PersistToDisk() error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (f *copyFile) contents() ([]byte, error) {
|
||||
return fs.ReadFile(f.srcFS, f.srcPath)
|
||||
}
|
||||
|
||||
type inMemoryFile struct {
|
||||
dstPath *destinationPath
|
||||
|
||||
|
|
@ -99,3 +106,7 @@ func (f *inMemoryFile) PersistToDisk() error {
|
|||
}
|
||||
return os.WriteFile(path, f.content, f.perm)
|
||||
}
|
||||
|
||||
func (f *inMemoryFile) contents() ([]byte, error) {
|
||||
return slices.Clone(f.content), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import (
|
|||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/libs/filer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -33,10 +32,7 @@ func testInMemoryFile(t *testing.T, perm fs.FileMode) {
|
|||
|
||||
func testCopyFile(t *testing.T, perm fs.FileMode) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
templateFiler, err := filer.NewLocalClient(tmpDir)
|
||||
require.NoError(t, err)
|
||||
err = os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
|
||||
err := os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
|
||||
require.NoError(t, err)
|
||||
|
||||
f := ©File{
|
||||
|
|
@ -47,7 +43,7 @@ func testCopyFile(t *testing.T, perm fs.FileMode) {
|
|||
},
|
||||
perm: perm,
|
||||
srcPath: "source",
|
||||
srcFiler: templateFiler,
|
||||
srcFS: os.DirFS(tmpDir),
|
||||
}
|
||||
err = f.PersistToDisk()
|
||||
assert.NoError(t, err)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -39,7 +39,7 @@ func TestTemplateRegexpCompileFunction(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -57,7 +57,7 @@ func TestTemplateRandIntFunction(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -75,7 +75,7 @@ func TestTemplateUuidFunction(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -92,7 +92,7 @@ func TestTemplateUrlFunction(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -109,7 +109,7 @@ func TestTemplateMapPairFunction(t *testing.T) {
|
|||
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -132,7 +132,7 @@ func TestWorkspaceHost(t *testing.T) {
|
|||
ctx = root.SetWorkspaceClient(ctx, w)
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -157,7 +157,7 @@ func TestWorkspaceHostNotConfigured(t *testing.T) {
|
|||
ctx = root.SetWorkspaceClient(ctx, w)
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
|
||||
|
||||
assert.NoError(t, err)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,13 +2,9 @@ package template
|
|||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/databricks/cli/libs/cmdio"
|
||||
)
|
||||
|
|
@ -17,39 +13,20 @@ const libraryDirName = "library"
|
|||
const templateDirName = "template"
|
||||
const schemaFileName = "databricks_template_schema.json"
|
||||
|
||||
//go:embed all:templates
|
||||
var builtinTemplates embed.FS
|
||||
|
||||
// This function materializes the input templates as a project, using user defined
|
||||
// configurations.
|
||||
// Parameters:
|
||||
//
|
||||
// ctx: context containing a cmdio object. This is used to prompt the user
|
||||
// configFilePath: file path containing user defined config values
|
||||
// templateRoot: root of the template definition
|
||||
// templateFS: root of the template definition
|
||||
// outputDir: root of directory where to initialize the template
|
||||
func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir string) error {
|
||||
// Use a temporary directory in case any builtin templates like default-python are used
|
||||
tempDir, err := os.MkdirTemp("", "templates")
|
||||
defer os.RemoveAll(tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
templateRoot, err = prepareBuiltinTemplates(templateRoot, tempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, outputDir string) error {
|
||||
if _, err := fs.Stat(templateFS, schemaFileName); errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
|
||||
}
|
||||
|
||||
templatePath := filepath.Join(templateRoot, templateDirName)
|
||||
libraryPath := filepath.Join(templateRoot, libraryDirName)
|
||||
schemaPath := filepath.Join(templateRoot, schemaFileName)
|
||||
helpers := loadHelpers(ctx)
|
||||
|
||||
if _, err := os.Stat(schemaPath); errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaPath)
|
||||
}
|
||||
|
||||
config, err := newConfig(ctx, schemaPath)
|
||||
config, err := newConfig(ctx, templateFS, schemaFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -62,7 +39,8 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
|
|||
}
|
||||
}
|
||||
|
||||
r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName, outputDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -111,44 +89,3 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// If the given templateRoot matches
|
||||
func prepareBuiltinTemplates(templateRoot string, tempDir string) (string, error) {
|
||||
// Check that `templateRoot` is a clean basename, i.e. `some_path` and not `./some_path` or "."
|
||||
// Return early if that's not the case.
|
||||
if templateRoot == "." || path.Base(templateRoot) != templateRoot {
|
||||
return templateRoot, nil
|
||||
}
|
||||
|
||||
_, err := fs.Stat(builtinTemplates, path.Join("templates", templateRoot))
|
||||
if err != nil {
|
||||
// The given path doesn't appear to be using out built-in templates
|
||||
return templateRoot, nil
|
||||
}
|
||||
|
||||
// We have a built-in template with the same name as templateRoot!
|
||||
// Now we need to make a fully copy of the builtin templates to a real file system
|
||||
// since template.Parse() doesn't support embed.FS.
|
||||
err = fs.WalkDir(builtinTemplates, "templates", func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetPath := filepath.Join(tempDir, path)
|
||||
if entry.IsDir() {
|
||||
return os.Mkdir(targetPath, 0755)
|
||||
} else {
|
||||
content, err := fs.ReadFile(builtinTemplates, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(targetPath, content, 0644)
|
||||
}
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filepath.Join(tempDir, "templates", templateRoot), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package template
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/cli/cmd/root"
|
||||
|
|
@ -19,6 +19,6 @@ func TestMaterializeForNonTemplateDirectory(t *testing.T) {
|
|||
ctx := root.SetWorkspaceClient(context.Background(), w)
|
||||
|
||||
// Try to materialize a non-template directory.
|
||||
err = Materialize(ctx, "", tmpDir, "")
|
||||
assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", filepath.Join(tmpDir, schemaFileName)))
|
||||
err = Materialize(ctx, "", os.DirFS(tmpDir), "")
|
||||
assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", schemaFileName))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,14 +8,12 @@ import (
|
|||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/databricks/cli/libs/filer"
|
||||
"github.com/databricks/cli/libs/log"
|
||||
"github.com/databricks/databricks-sdk-go/logger"
|
||||
)
|
||||
|
|
@ -52,32 +50,42 @@ type renderer struct {
|
|||
// do not match any glob patterns from this list
|
||||
skipPatterns []string
|
||||
|
||||
// Filer rooted at template root. The file tree from this root is walked to
|
||||
// generate the project
|
||||
templateFiler filer.Filer
|
||||
// [fs.FS] that holds the template's file tree.
|
||||
srcFS fs.FS
|
||||
|
||||
// Root directory for the project instantiated from the template
|
||||
instanceRoot string
|
||||
}
|
||||
|
||||
func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
|
||||
func newRenderer(
|
||||
ctx context.Context,
|
||||
config map[string]any,
|
||||
helpers template.FuncMap,
|
||||
templateFS fs.FS,
|
||||
templateDir string,
|
||||
libraryDir string,
|
||||
instanceRoot string,
|
||||
) (*renderer, error) {
|
||||
// Initialize new template, with helper functions loaded
|
||||
tmpl := template.New("").Funcs(helpers)
|
||||
|
||||
// Load user defined associated templates from the library root
|
||||
libraryGlob := filepath.Join(libraryRoot, "*")
|
||||
matches, err := filepath.Glob(libraryGlob)
|
||||
// Find user-defined templates in the library directory
|
||||
matches, err := fs.Glob(templateFS, path.Join(libraryDir, "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse user-defined templates.
|
||||
// Note: we do not call [ParseFS] with the glob directly because
|
||||
// it returns an error if no files match the pattern.
|
||||
if len(matches) != 0 {
|
||||
tmpl, err = tmpl.ParseFiles(matches...)
|
||||
tmpl, err = tmpl.ParseFS(templateFS, matches...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
templateFiler, err := filer.NewLocalClient(templateRoot)
|
||||
srcFS, err := fs.Sub(templateFS, path.Clean(templateDir))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -90,7 +98,7 @@ func newRenderer(ctx context.Context, config map[string]any, helpers template.Fu
|
|||
baseTemplate: tmpl,
|
||||
files: make([]file, 0),
|
||||
skipPatterns: make([]string, 0),
|
||||
templateFiler: templateFiler,
|
||||
srcFS: srcFS,
|
||||
instanceRoot: instanceRoot,
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -141,7 +149,7 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
|
|||
|
||||
func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
||||
// read file permissions
|
||||
info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
|
||||
info, err := fs.Stat(r.srcFS, relPathTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -163,8 +171,8 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
|||
},
|
||||
perm: perm,
|
||||
ctx: r.ctx,
|
||||
srcFS: r.srcFS,
|
||||
srcPath: relPathTemplate,
|
||||
srcFiler: r.templateFiler,
|
||||
}, nil
|
||||
} else {
|
||||
// Trim the .tmpl suffix from file name, if specified in the template
|
||||
|
|
@ -173,7 +181,7 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
|
|||
}
|
||||
|
||||
// read template file's content
|
||||
templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
|
||||
templateReader, err := r.srcFS.Open(relPathTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -263,7 +271,7 @@ func (r *renderer) walk() error {
|
|||
//
|
||||
// 2. For directories: They are appended to a slice, which acts as a queue
|
||||
// allowing BFS traversal of the template file tree
|
||||
entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
|
||||
entries, err := fs.ReadDir(r.srcFS, currentDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ package template
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
|
@ -41,9 +41,8 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
|
|||
func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
|
||||
ctx := context.Background()
|
||||
|
||||
templatePath, err := prepareBuiltinTemplates(template, tempDir)
|
||||
templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
|
||||
require.NoError(t, err)
|
||||
libraryPath := filepath.Join(templatePath, "library")
|
||||
|
||||
w := &databricks.WorkspaceClient{
|
||||
Config: &workspaceConfig.Config{Host: "https://myhost.com"},
|
||||
|
|
@ -58,7 +57,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
|||
ctx = root.SetWorkspaceClient(ctx, w)
|
||||
helpers := loadHelpers(ctx)
|
||||
|
||||
renderer, err := newRenderer(ctx, settings, helpers, templatePath, libraryPath, tempDir)
|
||||
renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName, tempDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Evaluate template
|
||||
|
|
@ -67,7 +66,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
|||
err = renderer.persistToDisk()
|
||||
require.NoError(t, err)
|
||||
|
||||
b, err := bundle.Load(ctx, filepath.Join(tempDir, "template", "my_project"))
|
||||
b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project"))
|
||||
require.NoError(t, err)
|
||||
diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
|
||||
require.NoError(t, diags.Error())
|
||||
|
|
@ -96,18 +95,6 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
|
|||
}
|
||||
}
|
||||
|
||||
func TestPrepareBuiltInTemplatesWithRelativePaths(t *testing.T) {
|
||||
// CWD should not be resolved as a built in template
|
||||
dir, err := prepareBuiltinTemplates(".", t.TempDir())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ".", dir)
|
||||
|
||||
// relative path should not be resolved as a built in template
|
||||
dir, err = prepareBuiltinTemplates("./default-python", t.TempDir())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "./default-python", dir)
|
||||
}
|
||||
|
||||
func TestBuiltinPythonTemplateValid(t *testing.T) {
|
||||
// Test option combinations
|
||||
options := []string{"yes", "no"}
|
||||
|
|
@ -194,7 +181,7 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/email/template", "./testdata/email/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -381,7 +368,7 @@ func TestRendererWalk(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/walk/template", "./testdata/walk/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -392,18 +379,9 @@ func TestRendererWalk(t *testing.T) {
|
|||
if f.DstPath().relPath != path {
|
||||
continue
|
||||
}
|
||||
switch v := f.(type) {
|
||||
case *inMemoryFile:
|
||||
return strings.Trim(string(v.content), "\r\n")
|
||||
case *copyFile:
|
||||
r, err := r.templateFiler.Read(context.Background(), v.srcPath)
|
||||
require.NoError(t, err)
|
||||
b, err := io.ReadAll(r)
|
||||
b, err := f.contents()
|
||||
require.NoError(t, err)
|
||||
return strings.Trim(string(b), "\r\n")
|
||||
default:
|
||||
require.FailNow(t, "execution should not reach here")
|
||||
}
|
||||
}
|
||||
require.FailNow(t, "file is absent: "+path)
|
||||
return ""
|
||||
|
|
@ -422,7 +400,7 @@ func TestRendererFailFunction(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/fail/template", "./testdata/fail/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -435,7 +413,7 @@ func TestRendererSkipsDirsEagerly(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -452,7 +430,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -475,7 +453,7 @@ func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -493,7 +471,7 @@ func TestRendererSkip(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/skip/template", "./testdata/skip/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -525,7 +503,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
|
|||
ctx = root.SetWorkspaceClient(ctx, nil)
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -615,7 +593,7 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
|
|||
tmpDir := t.TempDir()
|
||||
|
||||
helpers := loadHelpers(ctx)
|
||||
r, err := newRenderer(ctx, nil, helpers, "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
|
||||
r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -635,7 +613,7 @@ func TestRendererFileTreeRendering(t *testing.T) {
|
|||
r, err := newRenderer(ctx, map[string]any{
|
||||
"dir_name": "my_directory",
|
||||
"file_name": "my_file",
|
||||
}, helpers, "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
|
||||
}, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
@ -668,7 +646,7 @@ func TestRendererSubTemplateInPath(t *testing.T) {
|
|||
testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}"))
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir)
|
||||
r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library", tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = r.walk()
|
||||
|
|
|
|||
Loading…
Reference in New Issue