databricks-cli/libs/template/renderer.go

package template

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strings"
	"text/template"

	"github.com/databricks/cli/libs/filer"
	"github.com/databricks/cli/libs/log"
	"github.com/databricks/databricks-sdk-go/logger"
)

const templateExtension = ".tmpl"

// Renders a databricks template as a project
type renderer struct {
	ctx context.Context

	// A config that is the "dot" value available to any template being rendered.
	// Refer to https://pkg.go.dev/text/template for how templates can use
	// this "dot" value
	config map[string]any

	// A base template with helper functions and user defined templates in the
	// library directory loaded. This is cloned for each project template computation
	// during file tree walk
	baseTemplate *template.Template

	// List of in memory files generated from template
	files []file

	// Glob patterns for files and directories to skip. There are three possible
	// outcomes for skip:
	//
	// 1. File is not generated. This happens if one of the file's parent directories
	// match a glob pattern
	//
	// 2. File is generated but not persisted to disk. This happens if the file itself
	// matches a glob pattern, but none of it's parents match a glob pattern from the list
	//
	// 3. File is persisted to disk. This happens if the file and it's parent directories
	// do not match any glob patterns from this list
	skipPatterns []string

	// Filer rooted at template root. The file tree from this root is walked to
	// generate the project
	templateFiler filer.Filer

	// Root directory for the project instantiated from the template
	instanceRoot string
}

func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
	// Initialize new template, with helper functions loaded
	tmpl := template.New("").Funcs(helpers)

	// Load user defined associated templates from the library root
	libraryGlob := filepath.Join(libraryRoot, "*")
	matches, err := filepath.Glob(libraryGlob)
	if err != nil {
		return nil, err
	}
	if len(matches) != 0 {
		tmpl, err = tmpl.ParseFiles(matches...)
		if err != nil {
			return nil, err
		}
	}

	templateFiler, err := filer.NewLocalClient(templateRoot)
	if err != nil {
		return nil, err
	}

	ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))

	return &renderer{
		ctx:           ctx,
		config:        config,
		baseTemplate:  tmpl,
		files:         make([]file, 0),
		skipPatterns:  make([]string, 0),
		templateFiler: templateFiler,
		instanceRoot:  instanceRoot,
	}, nil
}

// Executes the template by applying config on it. Returns the materialized template
// as a string
func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
	// Create copy of base template so as to not overwrite it
	tmpl, err := r.baseTemplate.Clone()
	if err != nil {
		return "", err
	}

	// The template execution will error instead of printing <no value> on unknown
	// map keys if the "missingkey=error" option is set.
	// We do this here instead of doing this once for r.baseTemplate because
	// the Template.Clone() method does not clone options.
	tmpl = tmpl.Option("missingkey=error")

	// Parse the template text
	tmpl, err = tmpl.Parse(templateDefinition)
	if err != nil {
		return "", fmt.Errorf("error in %s: %w", templateDefinition, err)
	}

	// Execute template and get result
	result := strings.Builder{}
	err = tmpl.Execute(&result, r.config)
	if err != nil {
		// Parse and return a more readable error for missing values that are used
		// by the template definition but are not provided in the passed config.
		target := &template.ExecError{}
		if errors.As(err, target) {
			captureRegex := regexp.MustCompile(`map has no entry for key "(.*)"`)
			matches := captureRegex.FindStringSubmatch(target.Err.Error())
			if len(matches) != 2 {
				return "", err
			}
			return "", template.ExecError{
				Name: target.Name,
				Err:  fmt.Errorf("variable %q not defined", matches[1]),
			}
		}
		return "", err
	}
	return result.String(), nil
}

func (r *renderer) computeFile(relPathTemplate string) (file, error) {
	// read file permissions
	info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
	if err != nil {
		return nil, err
	}
	perm := info.Mode().Perm()

	// Execute relative path template to get destination path for the file
	relPath, err := r.executeTemplate(relPathTemplate)
	if err != nil {
		return nil, err
	}

	// we need the absolute path in case we need to write notebooks using the REST API
	rootPath, err := filepath.Abs(r.instanceRoot)
	if err != nil {
		return nil, err
	}

	// If file name does not specify the `.tmpl` extension, then it is copied
	// over as is, without treating it as a template
	if !strings.HasSuffix(relPathTemplate, templateExtension) {
		return &copyFile{
			dstPath: &destinationPath{
				root:    rootPath,
				relPath: relPath,
			},
			perm:     perm,
			ctx:      r.ctx,
			srcPath:  relPathTemplate,
			srcFiler: r.templateFiler,
		}, nil
	} else {
		// Trim the .tmpl suffix from file name, if specified in the template
		// path
		relPath = strings.TrimSuffix(relPath, templateExtension)
	}

	// read template file's content
	templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
	if err != nil {
		return nil, err
	}
	defer templateReader.Close()

	// execute the contents of the file as a template
	contentTemplate, err := io.ReadAll(templateReader)
	if err != nil {
		return nil, err
	}
	content, err := r.executeTemplate(string(contentTemplate))
	// Capture errors caused by the "fail" helper function
	if target := (&ErrFail{}); errors.As(err, target) {
		return nil, target
	}
	if err != nil {
		return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err)
	}

	return &inMemoryFile{
		ctx: r.ctx,
		dstPath: &destinationPath{
			root:    rootPath,
			relPath: relPath,
		},
		perm:    perm,
		content: []byte(content),
	}, nil
}

// This function walks the template file tree to generate an in memory representation
// of a project.
//
// During file tree walk, in the current directory, we would like to determine
// all possible {{skip}} function calls before we process any of the directories
// so that we can skip them eagerly if needed. That is in the current working directory
// we would like to process all files before we process any of the directories.
//
// This is not possible using the std library WalkDir which processes the files in
// lexical order which is why this function implements BFS.
func (r *renderer) walk() error {
	directories := []string{"."}
	var currentDirectory string

	for len(directories) > 0 {
		currentDirectory, directories = directories[0], directories[1:]

		// Skip current directory if it matches any of accumulated skip patterns
		instanceDirectory, err := r.executeTemplate(currentDirectory)
		if err != nil {
			return err
		}
		match, err := isSkipped(instanceDirectory, r.skipPatterns)
		if err != nil {
			return err
		}
		if match {
			logger.Infof(r.ctx, "skipping directory: %s", instanceDirectory)
			continue
		}

		// Add skip function, which accumulates skip patterns relative to current
		// directory
		r.baseTemplate.Funcs(template.FuncMap{
			"skip": func(relPattern string) (string, error) {
				// patterns are specified relative to current directory of the file
				// the {{skip}} function is called from.
				patternRaw := path.Join(currentDirectory, relPattern)
				pattern, err := r.executeTemplate(patternRaw)
				if err != nil {
					return "", err
				}

				if !slices.Contains(r.skipPatterns, pattern) {
					logger.Infof(r.ctx, "adding skip pattern: %s", pattern)
					r.skipPatterns = append(r.skipPatterns, pattern)
				}
				// return empty string will print nothing at function call site
				// when executing the template
				return "", nil
			},
		})

		// Process all entries in current directory
		//
		// 1. For files: the templates in the file name and content are executed, and
		//     a in memory representation of the file is generated
		//
		// 2. For directories: They are appended to a slice, which acts as a queue
		//     allowing BFS traversal of the template file tree
		entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
		if err != nil {
			return err
		}
		// Sort by name to ensure deterministic ordering
		sort.Slice(entries, func(i, j int) bool {
			return entries[i].Name() < entries[j].Name()
		})
		for _, entry := range entries {
			if entry.IsDir() {
				// Add to slice, for BFS traversal
				directories = append(directories, path.Join(currentDirectory, entry.Name()))
				continue
			}

			// Generate in memory representation of file
			f, err := r.computeFile(path.Join(currentDirectory, entry.Name()))
			if err != nil {
				return err
			}
			logger.Infof(r.ctx, "added file to list of possible project files: %s", f.DstPath().relPath)
			r.files = append(r.files, f)
		}

	}
	return nil
}

func (r *renderer) persistToDisk() error {
	// Accumulate files which we will persist, skipping files whose path matches
	// any of the skip patterns
	filesToPersist := make([]file, 0)
	for _, file := range r.files {
		match, err := isSkipped(file.DstPath().relPath, r.skipPatterns)
		if err != nil {
			return err
		}
		if match {
			log.Infof(r.ctx, "skipping file: %s", file.DstPath())
			continue
		}
		filesToPersist = append(filesToPersist, file)
	}

	// Assert no conflicting files exist
	for _, file := range filesToPersist {
		path := file.DstPath().absPath()
		_, err := os.Stat(path)
		if err == nil {
			return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path)
		}
		if !errors.Is(err, fs.ErrNotExist) {
			return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err)
		}
	}

	// Persist files to disk
	for _, file := range filesToPersist {
		err := file.PersistToDisk()
		if err != nil {
			return err
		}
	}
	return nil
}

func isSkipped(filePath string, patterns []string) (bool, error) {
	for _, pattern := range patterns {
		isMatch, err := path.Match(pattern, filePath)
		if err != nil {
			return false, err
		}
		if isMatch {
			return true, nil
		}
	}
	return false, nil
}