Add template renderer for Databricks templates (#589)

## Changes
This PR adds the renderer struct, which is a walker that traverses
templates and generates projects from them

## Tests
Unit tests
This commit is contained in:
shreyas-goenka 2023-07-21 10:59:02 +02:00 committed by GitHub
parent adab9aa5d7
commit 02dbac7b8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 802 additions and 0 deletions

20
libs/template/helpers.go Normal file
View File

@ -0,0 +1,20 @@
package template
import (
"fmt"
"text/template"
)
type ErrFail struct {
msg string
}
func (err ErrFail) Error() string {
return err.msg
}
var helperFuncs = template.FuncMap{
"fail": func(format string, args ...any) (any, error) {
return nil, ErrFail{fmt.Sprintf(format, args...)}
},
}

316
libs/template/renderer.go Normal file
View File

@ -0,0 +1,316 @@
package template
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"text/template"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go/logger"
"golang.org/x/exp/slices"
)
type inMemoryFile struct {
// Root path for the project instance. This path uses the system's default
// file separator. For example /foo/bar on Unix and C:\foo\bar on windows
root string
// Unix like relPath for the file (using '/' as the separator). This path
// is relative to the root. Using unix like relative paths enables skip patterns
// to work across both windows and unix based operating systems.
relPath string
content []byte
perm fs.FileMode
}
func (f *inMemoryFile) fullPath() string {
return filepath.Join(f.root, filepath.FromSlash(f.relPath))
}
func (f *inMemoryFile) persistToDisk() error {
path := f.fullPath()
err := os.MkdirAll(filepath.Dir(path), 0755)
if err != nil {
return err
}
return os.WriteFile(path, f.content, f.perm)
}
// Renders a databricks template as a project
type renderer struct {
ctx context.Context
// A config that is the "dot" value available to any template being rendered.
// Refer to https://pkg.go.dev/text/template for how templates can use
// this "dot" value
config map[string]any
// A base template with helper functions and user defined templates in the
// library directory loaded. This is cloned for each project template computation
// during file tree walk
baseTemplate *template.Template
// List of in memory files generated from template
files []*inMemoryFile
// Glob patterns for files and directories to skip. There are three possible
// outcomes for skip:
//
// 1. File is not generated. This happens if one of the file's parent directories
// match a glob pattern
//
// 2. File is generated but not persisted to disk. This happens if the file itself
// matches a glob pattern, but none of it's parents match a glob pattern from the list
//
// 3. File is persisted to disk. This happens if the file and it's parent directories
// do not match any glob patterns from this list
skipPatterns []string
// Filer rooted at template root. The file tree from this root is walked to
// generate the project
templateFiler filer.Filer
// Root directory for the project instantiated from the template
instanceRoot string
}
func newRenderer(ctx context.Context, config map[string]any, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
// Initialize new template, with helper functions loaded
tmpl := template.New("").Funcs(helperFuncs)
// Load user defined associated templates from the library root
libraryGlob := filepath.Join(libraryRoot, "*")
matches, err := filepath.Glob(libraryGlob)
if err != nil {
return nil, err
}
if len(matches) != 0 {
tmpl, err = tmpl.ParseFiles(matches...)
if err != nil {
return nil, err
}
}
templateFiler, err := filer.NewLocalClient(templateRoot)
if err != nil {
return nil, err
}
ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))
return &renderer{
ctx: ctx,
config: config,
baseTemplate: tmpl,
files: make([]*inMemoryFile, 0),
skipPatterns: make([]string, 0),
templateFiler: templateFiler,
instanceRoot: instanceRoot,
}, nil
}
// Executes the template by applying config on it. Returns the materialized template
// as a string
func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
// Create copy of base template so as to not overwrite it
tmpl, err := r.baseTemplate.Clone()
if err != nil {
return "", err
}
// Parse the template text
tmpl, err = tmpl.Parse(templateDefinition)
if err != nil {
return "", err
}
// Execute template and get result
result := strings.Builder{}
err = tmpl.Execute(&result, r.config)
if err != nil {
return "", err
}
return result.String(), nil
}
func (r *renderer) computeFile(relPathTemplate string) (*inMemoryFile, error) {
// read template file contents
templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
if err != nil {
return nil, err
}
contentTemplate, err := io.ReadAll(templateReader)
if err != nil {
return nil, err
}
// read file permissions
info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
if err != nil {
return nil, err
}
perm := info.Mode().Perm()
// execute the contents of the file as a template
content, err := r.executeTemplate(string(contentTemplate))
// Capture errors caused by the "fail" helper function
if target := (&ErrFail{}); errors.As(err, target) {
return nil, target
}
if err != nil {
return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err)
}
// Execute relative path template to get materialized path for the file
relPath, err := r.executeTemplate(relPathTemplate)
if err != nil {
return nil, err
}
return &inMemoryFile{
root: r.instanceRoot,
relPath: relPath,
content: []byte(content),
perm: perm,
}, nil
}
// This function walks the template file tree to generate an in memory representation
// of a project.
//
// During file tree walk, in the current directory, we would like to determine
// all possible {{skip}} function calls before we process any of the directories
// so that we can skip them eagerly if needed. That is in the current working directory
// we would like to process all files before we process any of the directories.
//
// This is not possible using the std library WalkDir which processes the files in
// lexical order which is why this function implements BFS.
func (r *renderer) walk() error {
directories := []string{"."}
var currentDirectory string
for len(directories) > 0 {
currentDirectory, directories = directories[0], directories[1:]
// Skip current directory if it matches any of accumulated skip patterns
instanceDirectory, err := r.executeTemplate(currentDirectory)
if err != nil {
return err
}
isSkipped, err := r.isSkipped(instanceDirectory)
if err != nil {
return err
}
if isSkipped {
logger.Infof(r.ctx, "skipping directory: %s", instanceDirectory)
continue
}
// Add skip function, which accumulates skip patterns relative to current
// directory
r.baseTemplate.Funcs(template.FuncMap{
"skip": func(relPattern string) string {
// patterns are specified relative to current directory of the file
// the {{skip}} function is called from.
pattern := path.Join(currentDirectory, relPattern)
if !slices.Contains(r.skipPatterns, pattern) {
logger.Infof(r.ctx, "adding skip pattern: %s", pattern)
r.skipPatterns = append(r.skipPatterns, pattern)
}
// return empty string will print nothing at function call site
// when executing the template
return ""
},
})
// Process all entries in current directory
//
// 1. For files: the templates in the file name and content are executed, and
// a in memory representation of the file is generated
//
// 2. For directories: They are appended to a slice, which acts as a queue
// allowing BFS traversal of the template file tree
entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
if err != nil {
return err
}
for _, entry := range entries {
if entry.IsDir() {
// Add to slice, for BFS traversal
directories = append(directories, path.Join(currentDirectory, entry.Name()))
continue
}
// Generate in memory representation of file
f, err := r.computeFile(path.Join(currentDirectory, entry.Name()))
if err != nil {
return err
}
logger.Infof(r.ctx, "added file to list of in memory files: %s", f.relPath)
r.files = append(r.files, f)
}
}
return nil
}
func (r *renderer) persistToDisk() error {
// Accumulate files which we will persist, skipping files whose path matches
// any of the skip patterns
filesToPersist := make([]*inMemoryFile, 0)
for _, file := range r.files {
isSkipped, err := r.isSkipped(file.relPath)
if err != nil {
return err
}
if isSkipped {
log.Infof(r.ctx, "skipping file: %s", file.relPath)
continue
}
filesToPersist = append(filesToPersist, file)
}
// Assert no conflicting files exist
for _, file := range filesToPersist {
path := file.fullPath()
_, err := os.Stat(path)
if err == nil {
return fmt.Errorf("failed to persist to disk, conflict with existing file: %s", path)
}
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err)
}
}
// Persist files to disk
for _, file := range filesToPersist {
err := file.persistToDisk()
if err != nil {
return err
}
}
return nil
}
func (r *renderer) isSkipped(filePath string) (bool, error) {
for _, pattern := range r.skipPatterns {
isMatch, err := path.Match(pattern, filePath)
if err != nil {
return false, err
}
if isMatch {
return true, nil
}
}
return false, nil
}

View File

@ -0,0 +1,438 @@
package template
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"text/template"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func assertFileContent(t *testing.T, path string, content string) {
b, err := os.ReadFile(path)
require.NoError(t, err)
assert.Equal(t, content, string(b))
}
func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
info, err := os.Stat(path)
require.NoError(t, err)
assert.Equal(t, perm, info.Mode().Perm())
}
func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
tmpDir := t.TempDir()
r, err := newRenderer(context.Background(), nil, "./testdata/email/template", "./testdata/email/library", tmpDir)
require.NoError(t, err)
err = r.walk()
require.NoError(t, err)
err = r.persistToDisk()
require.NoError(t, err)
b, err := os.ReadFile(filepath.Join(tmpDir, "my_email"))
require.NoError(t, err)
assert.Equal(t, "shreyas.goenka@databricks.com", strings.Trim(string(b), "\n\r"))
}
func TestRendererExecuteTemplate(t *testing.T) {
templateText :=
`"{{.count}} items are made of {{.Material}}".
{{if eq .Animal "sheep" }}
Sheep wool is the best!
{{else}}
{{.Animal}} wool is not too bad...
{{end}}
My email is {{template "email"}}
`
r := renderer{
config: map[string]any{
"Material": "wool",
"count": 1,
"Animal": "sheep",
},
baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}shreyas.goenka@databricks.com{{end}}`)),
}
statement, err := r.executeTemplate(templateText)
require.NoError(t, err)
assert.Contains(t, statement, `"1 items are made of wool"`)
assert.NotContains(t, statement, `cat wool is not too bad.."`)
assert.Contains(t, statement, "Sheep wool is the best!")
assert.Contains(t, statement, `My email is shreyas.goenka@databricks.com`)
r = renderer{
config: map[string]any{
"Material": "wool",
"count": 1,
"Animal": "cat",
},
baseTemplate: template.Must(template.New("base").Parse(`{{define "email"}}hrithik.roshan@databricks.com{{end}}`)),
}
statement, err = r.executeTemplate(templateText)
require.NoError(t, err)
assert.Contains(t, statement, `"1 items are made of wool"`)
assert.Contains(t, statement, `cat wool is not too bad...`)
assert.NotContains(t, statement, "Sheep wool is the best!")
assert.Contains(t, statement, `My email is hrithik.roshan@databricks.com`)
}
func TestRendererIsSkipped(t *testing.T) {
r := renderer{
skipPatterns: []string{"a*", "*yz", "def", "a/b/*"},
}
// skipped paths
isSkipped, err := r.isSkipped("abc")
require.NoError(t, err)
assert.True(t, isSkipped)
isSkipped, err = r.isSkipped("abcd")
require.NoError(t, err)
assert.True(t, isSkipped)
isSkipped, err = r.isSkipped("a")
require.NoError(t, err)
assert.True(t, isSkipped)
isSkipped, err = r.isSkipped("xxyz")
require.NoError(t, err)
assert.True(t, isSkipped)
isSkipped, err = r.isSkipped("yz")
require.NoError(t, err)
assert.True(t, isSkipped)
isSkipped, err = r.isSkipped("a/b/c")
require.NoError(t, err)
assert.True(t, isSkipped)
// NOT skipped paths
isSkipped, err = r.isSkipped(".")
require.NoError(t, err)
assert.False(t, isSkipped)
isSkipped, err = r.isSkipped("y")
require.NoError(t, err)
assert.False(t, isSkipped)
isSkipped, err = r.isSkipped("z")
require.NoError(t, err)
assert.False(t, isSkipped)
isSkipped, err = r.isSkipped("defg")
require.NoError(t, err)
assert.False(t, isSkipped)
isSkipped, err = r.isSkipped("cat")
require.NoError(t, err)
assert.False(t, isSkipped)
isSkipped, err = r.isSkipped("a/b/c/d")
require.NoError(t, err)
assert.False(t, isSkipped)
}
func TestRendererPersistToDisk(t *testing.T) {
tmpDir := t.TempDir()
ctx := context.Background()
r := &renderer{
ctx: ctx,
instanceRoot: tmpDir,
skipPatterns: []string{"a/b/c", "mn*"},
files: []*inMemoryFile{
{
root: tmpDir,
relPath: "a/b/c",
content: nil,
perm: 0444,
},
{
root: tmpDir,
relPath: "mno",
content: nil,
perm: 0444,
},
{
root: tmpDir,
relPath: "a/b/d",
content: []byte("123"),
perm: 0444,
},
{
root: tmpDir,
relPath: "mmnn",
content: []byte("456"),
perm: 0444,
},
},
}
err := r.persistToDisk()
require.NoError(t, err)
assert.NoFileExists(t, filepath.Join(tmpDir, "a", "b", "c"))
assert.NoFileExists(t, filepath.Join(tmpDir, "mno"))
assertFileContent(t, filepath.Join(tmpDir, "a", "b", "d"), "123")
assertFilePermissions(t, filepath.Join(tmpDir, "a", "b", "d"), 0444)
assertFileContent(t, filepath.Join(tmpDir, "mmnn"), "456")
assertFilePermissions(t, filepath.Join(tmpDir, "mmnn"), 0444)
}
func TestRendererWalk(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/walk/template", "./testdata/walk/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
getContent := func(r *renderer, path string) string {
for _, f := range r.files {
if f.relPath == path {
return strings.Trim(string(f.content), "\r\n")
}
}
require.FailNow(t, "file is absent: "+path)
return ""
}
assert.Len(t, r.files, 4)
assert.Equal(t, "file one", getContent(r, "file1"))
assert.Equal(t, "file two", getContent(r, "file2"))
assert.Equal(t, "file three", getContent(r, "dir1/dir3/file3"))
assert.Equal(t, "file four", getContent(r, "dir2/file4"))
}
func TestRendererFailFunction(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/fail/template", "./testdata/fail/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.Equal(t, "I am an error message", err.Error())
}
func TestRendererSkipsDirsEagerly(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
assert.Len(t, r.files, 1)
content := string(r.files[0].content)
assert.Equal(t, "I should be the only file created", strings.Trim(content, "\r\n"))
}
func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
// All 3 files are executed and have in memory representations
require.Len(t, r.files, 3)
err = r.persistToDisk()
require.NoError(t, err)
entries, err := os.ReadDir(tmpDir)
require.NoError(t, err)
// Assert none of the files are persisted to disk, because of {{skip "*"}}
assert.Len(t, entries, 0)
}
func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
assert.Len(t, r.skipPatterns, 3)
assert.Contains(t, r.skipPatterns, "a")
assert.Contains(t, r.skipPatterns, "dir1/b")
assert.Contains(t, r.skipPatterns, "dir1/dir2/c")
}
func TestRendererSkip(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
r, err := newRenderer(ctx, nil, "./testdata/skip/template", "./testdata/skip/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
// All 6 files are computed, even though "dir2/*" is present as a skip pattern
// This is because "dir2/*" matches the files in dir2, but not dir2 itself
assert.Len(t, r.files, 6)
err = r.persistToDisk()
require.NoError(t, err)
assert.FileExists(t, filepath.Join(tmpDir, "file1"))
assert.FileExists(t, filepath.Join(tmpDir, "file2"))
assert.FileExists(t, filepath.Join(tmpDir, "dir1/file5"))
// These files have been skipped
assert.NoFileExists(t, filepath.Join(tmpDir, "file3"))
assert.NoFileExists(t, filepath.Join(tmpDir, "dir1/file4"))
assert.NoDirExists(t, filepath.Join(tmpDir, "dir2"))
assert.NoFileExists(t, filepath.Join(tmpDir, "dir2/file6"))
}
func TestRendererInMemoryFileFullPathForWindows(t *testing.T) {
if runtime.GOOS != "windows" {
t.SkipNow()
}
f := &inMemoryFile{
root: `c:\a\b\c`,
relPath: "d/e",
}
assert.Equal(t, `c:\a\b\c\d\e`, f.fullPath())
}
func TestRendererInMemoryFilePersistToDiskSetsExecutableBit(t *testing.T) {
if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
t.SkipNow()
}
tmpDir := t.TempDir()
f := &inMemoryFile{
root: tmpDir,
relPath: "a/b/c",
content: []byte("123"),
perm: 0755,
}
err := f.persistToDisk()
assert.NoError(t, err)
assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "123")
assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), 0755)
}
func TestRendererInMemoryFilePersistToDiskForWindows(t *testing.T) {
if runtime.GOOS != "windows" {
t.SkipNow()
}
tmpDir := t.TempDir()
f := &inMemoryFile{
root: tmpDir,
relPath: "a/b/c",
content: []byte("123"),
perm: 0666,
}
err := f.persistToDisk()
assert.NoError(t, err)
assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "123")
assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), 0666)
}
func TestRendererReadsPermissionsBits(t *testing.T) {
if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
t.SkipNow()
}
tmpDir := t.TempDir()
ctx := context.Background()
r, err := newRenderer(ctx, nil, "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
require.NoError(t, err)
err = r.walk()
assert.NoError(t, err)
getPermissions := func(r *renderer, path string) fs.FileMode {
for _, f := range r.files {
if f.relPath == path {
return f.perm
}
}
require.FailNow(t, "file is absent: "+path)
return 0
}
assert.Len(t, r.files, 2)
assert.Equal(t, getPermissions(r, "script.sh"), fs.FileMode(0755))
assert.Equal(t, getPermissions(r, "not-a-script"), fs.FileMode(0644))
}
func TestRendererErrorOnConflictingFile(t *testing.T) {
tmpDir := t.TempDir()
f, err := os.Create(filepath.Join(tmpDir, "a"))
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)
r := renderer{
skipPatterns: []string{},
files: []*inMemoryFile{
{
root: tmpDir,
relPath: "a",
content: []byte("123"),
perm: 0444,
},
},
}
err = r.persistToDisk()
assert.EqualError(t, err, fmt.Sprintf("failed to persist to disk, conflict with existing file: %s", filepath.Join(tmpDir, "a")))
}
func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {
tmpDir := t.TempDir()
ctx := context.Background()
f, err := os.Create(filepath.Join(tmpDir, "a"))
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)
r := renderer{
ctx: ctx,
skipPatterns: []string{"a"},
files: []*inMemoryFile{
{
root: tmpDir,
relPath: "a",
content: []byte("123"),
perm: 0444,
},
},
}
err = r.persistToDisk()
// No error is returned even though a conflicting file exists. This is because
// the generated file is being skipped
assert.NoError(t, err)
assert.Len(t, r.files, 1)
}

View File

@ -0,0 +1 @@
{{define "email"}}shreyas.goenka@databricks.com{{end}}

View File

@ -0,0 +1 @@
{{template "email"}}

View File

@ -0,0 +1 @@
echo "hello"

View File

@ -0,0 +1 @@
{{fail "I am an error message"}}

View File

@ -0,0 +1 @@
a

View File

@ -0,0 +1 @@
b

View File

@ -0,0 +1,3 @@
c
{{skip "*"}}

View File

@ -0,0 +1 @@
{{fail "This template should never be executed"}}

View File

@ -0,0 +1,3 @@
I should be the only file created
{{skip "dir1"}}

View File

@ -0,0 +1 @@
{{skip "c"}}

View File

@ -0,0 +1 @@
{{skip "b"}}

View File

@ -0,0 +1 @@
{{skip "a"}}

View File

View File

View File

View File

@ -0,0 +1 @@
{{skip "file3"}}

View File

@ -0,0 +1,2 @@
{{skip "dir1/file4"}}
{{skip "dir2/*"}}

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@
file three

View File

@ -0,0 +1,5 @@
{{if (eq 1 1)}}
file four
{{else}}
mathematics is a lie
{{end}}

View File

@ -0,0 +1 @@
file one

View File

@ -0,0 +1 @@
file two