databricks-cli/libs/template/renderer.go

343 lines
9.9 KiB
Go

package template
import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"text/template"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go/logger"
)
const templateExtension = ".tmpl"
// Renders a databricks template as a project
type renderer struct {
ctx context.Context
// A config that is the "dot" value available to any template being rendered.
// Refer to https://pkg.go.dev/text/template for how templates can use
// this "dot" value
config map[string]any
// A base template with helper functions and user defined templates in the
// library directory loaded. This is cloned for each project template computation
// during file tree walk
baseTemplate *template.Template
// List of in memory files generated from template
files []file
// Glob patterns for files and directories to skip. There are three possible
// outcomes for skip:
//
// 1. File is not generated. This happens if one of the file's parent directories
// match a glob pattern
//
// 2. File is generated but not persisted to disk. This happens if the file itself
// matches a glob pattern, but none of it's parents match a glob pattern from the list
//
// 3. File is persisted to disk. This happens if the file and it's parent directories
// do not match any glob patterns from this list
skipPatterns []string
// Filer rooted at template root. The file tree from this root is walked to
// generate the project
templateFiler filer.Filer
// Root directory for the project instantiated from the template
instanceRoot string
}
func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
// Initialize new template, with helper functions loaded
tmpl := template.New("").Funcs(helpers)
// Load user defined associated templates from the library root
libraryGlob := filepath.Join(libraryRoot, "*")
matches, err := filepath.Glob(libraryGlob)
if err != nil {
return nil, err
}
if len(matches) != 0 {
tmpl, err = tmpl.ParseFiles(matches...)
if err != nil {
return nil, err
}
}
templateFiler, err := filer.NewLocalClient(templateRoot)
if err != nil {
return nil, err
}
ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))
return &renderer{
ctx: ctx,
config: config,
baseTemplate: tmpl,
files: make([]file, 0),
skipPatterns: make([]string, 0),
templateFiler: templateFiler,
instanceRoot: instanceRoot,
}, nil
}
// Executes the template by applying config on it. Returns the materialized template
// as a string
func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
// Create copy of base template so as to not overwrite it
tmpl, err := r.baseTemplate.Clone()
if err != nil {
return "", err
}
// The template execution will error instead of printing <no value> on unknown
// map keys if the "missingkey=error" option is set.
// We do this here instead of doing this once for r.baseTemplate because
// the Template.Clone() method does not clone options.
tmpl = tmpl.Option("missingkey=error")
// Parse the template text
tmpl, err = tmpl.Parse(templateDefinition)
if err != nil {
return "", fmt.Errorf("error in %s: %w", templateDefinition, err)
}
// Execute template and get result
result := strings.Builder{}
err = tmpl.Execute(&result, r.config)
if err != nil {
// Parse and return a more readable error for missing values that are used
// by the template definition but are not provided in the passed config.
target := &template.ExecError{}
if errors.As(err, target) {
captureRegex := regexp.MustCompile(`map has no entry for key "(.*)"`)
matches := captureRegex.FindStringSubmatch(target.Err.Error())
if len(matches) != 2 {
return "", err
}
return "", template.ExecError{
Name: target.Name,
Err: fmt.Errorf("variable %q not defined", matches[1]),
}
}
return "", err
}
return result.String(), nil
}
func (r *renderer) computeFile(relPathTemplate string) (file, error) {
// read file permissions
info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
if err != nil {
return nil, err
}
perm := info.Mode().Perm()
// Execute relative path template to get destination path for the file
relPath, err := r.executeTemplate(relPathTemplate)
if err != nil {
return nil, err
}
// If file name does not specify the `.tmpl` extension, then it is copied
// over as is, without treating it as a template
if !strings.HasSuffix(relPathTemplate, templateExtension) {
return &copyFile{
dstPath: &destinationPath{
root: r.instanceRoot,
relPath: relPath,
},
perm: perm,
ctx: r.ctx,
srcPath: relPathTemplate,
srcFiler: r.templateFiler,
}, nil
} else {
// Trim the .tmpl suffix from file name, if specified in the template
// path
relPath = strings.TrimSuffix(relPath, templateExtension)
}
// read template file's content
templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
if err != nil {
return nil, err
}
defer templateReader.Close()
// execute the contents of the file as a template
contentTemplate, err := io.ReadAll(templateReader)
if err != nil {
return nil, err
}
content, err := r.executeTemplate(string(contentTemplate))
// Capture errors caused by the "fail" helper function
if target := (&ErrFail{}); errors.As(err, target) {
return nil, target
}
if err != nil {
return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err)
}
return &inMemoryFile{
dstPath: &destinationPath{
root: r.instanceRoot,
relPath: relPath,
},
perm: perm,
content: []byte(content),
}, nil
}
// This function walks the template file tree to generate an in memory representation
// of a project.
//
// During file tree walk, in the current directory, we would like to determine
// all possible {{skip}} function calls before we process any of the directories
// so that we can skip them eagerly if needed. That is in the current working directory
// we would like to process all files before we process any of the directories.
//
// This is not possible using the std library WalkDir which processes the files in
// lexical order which is why this function implements BFS.
func (r *renderer) walk() error {
directories := []string{"."}
var currentDirectory string
for len(directories) > 0 {
currentDirectory, directories = directories[0], directories[1:]
// Skip current directory if it matches any of accumulated skip patterns
instanceDirectory, err := r.executeTemplate(currentDirectory)
if err != nil {
return err
}
match, err := isSkipped(instanceDirectory, r.skipPatterns)
if err != nil {
return err
}
if match {
logger.Infof(r.ctx, "skipping directory: %s", instanceDirectory)
continue
}
// Add skip function, which accumulates skip patterns relative to current
// directory
r.baseTemplate.Funcs(template.FuncMap{
"skip": func(relPattern string) (string, error) {
// patterns are specified relative to current directory of the file
// the {{skip}} function is called from.
patternRaw := path.Join(currentDirectory, relPattern)
pattern, err := r.executeTemplate(patternRaw)
if err != nil {
return "", err
}
if !slices.Contains(r.skipPatterns, pattern) {
logger.Infof(r.ctx, "adding skip pattern: %s", pattern)
r.skipPatterns = append(r.skipPatterns, pattern)
}
// return empty string will print nothing at function call site
// when executing the template
return "", nil
},
})
// Process all entries in current directory
//
// 1. For files: the templates in the file name and content are executed, and
// a in memory representation of the file is generated
//
// 2. For directories: They are appended to a slice, which acts as a queue
// allowing BFS traversal of the template file tree
entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
if err != nil {
return err
}
// Sort by name to ensure deterministic ordering
sort.Slice(entries, func(i, j int) bool {
return entries[i].Name() < entries[j].Name()
})
for _, entry := range entries {
if entry.IsDir() {
// Add to slice, for BFS traversal
directories = append(directories, path.Join(currentDirectory, entry.Name()))
continue
}
// Generate in memory representation of file
f, err := r.computeFile(path.Join(currentDirectory, entry.Name()))
if err != nil {
return err
}
logger.Infof(r.ctx, "added file to list of possible project files: %s", f.DstPath().relPath)
r.files = append(r.files, f)
}
}
return nil
}
func (r *renderer) persistToDisk() error {
// Accumulate files which we will persist, skipping files whose path matches
// any of the skip patterns
filesToPersist := make([]file, 0)
for _, file := range r.files {
match, err := isSkipped(file.DstPath().relPath, r.skipPatterns)
if err != nil {
return err
}
if match {
log.Infof(r.ctx, "skipping file: %s", file.DstPath())
continue
}
filesToPersist = append(filesToPersist, file)
}
// Assert no conflicting files exist
for _, file := range filesToPersist {
path := file.DstPath().absPath()
_, err := os.Stat(path)
if err == nil {
return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path)
}
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err)
}
}
// Persist files to disk
for _, file := range filesToPersist {
err := file.PersistToDisk()
if err != nil {
return err
}
}
return nil
}
func isSkipped(filePath string, patterns []string) (bool, error) {
for _, pattern := range patterns {
isMatch, err := path.Match(pattern, filePath)
if err != nil {
return false, err
}
if isMatch {
return true, nil
}
}
return false, nil
}