package template import ( "context" "errors" "fmt" "io" "io/fs" "os" "path" "path/filepath" "strings" "text/template" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/logger" "golang.org/x/exp/slices" ) type inMemoryFile struct { // Root path for the project instance. This path uses the system's default // file separator. For example /foo/bar on Unix and C:\foo\bar on windows root string // Unix like relPath for the file (using '/' as the separator). This path // is relative to the root. Using unix like relative paths enables skip patterns // to work across both windows and unix based operating systems. relPath string content []byte perm fs.FileMode } func (f *inMemoryFile) fullPath() string { return filepath.Join(f.root, filepath.FromSlash(f.relPath)) } func (f *inMemoryFile) persistToDisk() error { path := f.fullPath() err := os.MkdirAll(filepath.Dir(path), 0755) if err != nil { return err } return os.WriteFile(path, f.content, f.perm) } // Renders a databricks template as a project type renderer struct { ctx context.Context // A config that is the "dot" value available to any template being rendered. // Refer to https://pkg.go.dev/text/template for how templates can use // this "dot" value config map[string]any // A base template with helper functions and user defined templates in the // library directory loaded. This is cloned for each project template computation // during file tree walk baseTemplate *template.Template // List of in memory files generated from template files []*inMemoryFile // Glob patterns for files and directories to skip. There are three possible // outcomes for skip: // // 1. File is not generated. This happens if one of the file's parent directories // match a glob pattern // // 2. File is generated but not persisted to disk. This happens if the file itself // matches a glob pattern, but none of it's parents match a glob pattern from the list // // 3. File is persisted to disk. This happens if the file and it's parent directories // do not match any glob patterns from this list skipPatterns []string // Filer rooted at template root. The file tree from this root is walked to // generate the project templateFiler filer.Filer // Root directory for the project instantiated from the template instanceRoot string } func newRenderer(ctx context.Context, config map[string]any, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) { // Initialize new template, with helper functions loaded tmpl := template.New("").Funcs(helperFuncs) // Load user defined associated templates from the library root libraryGlob := filepath.Join(libraryRoot, "*") matches, err := filepath.Glob(libraryGlob) if err != nil { return nil, err } if len(matches) != 0 { tmpl, err = tmpl.ParseFiles(matches...) if err != nil { return nil, err } } templateFiler, err := filer.NewLocalClient(templateRoot) if err != nil { return nil, err } ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template")) return &renderer{ ctx: ctx, config: config, baseTemplate: tmpl, files: make([]*inMemoryFile, 0), skipPatterns: make([]string, 0), templateFiler: templateFiler, instanceRoot: instanceRoot, }, nil } // Executes the template by applying config on it. Returns the materialized template // as a string func (r *renderer) executeTemplate(templateDefinition string) (string, error) { // Create copy of base template so as to not overwrite it tmpl, err := r.baseTemplate.Clone() if err != nil { return "", err } // Parse the template text tmpl, err = tmpl.Parse(templateDefinition) if err != nil { return "", err } // Execute template and get result result := strings.Builder{} err = tmpl.Execute(&result, r.config) if err != nil { return "", err } return result.String(), nil } func (r *renderer) computeFile(relPathTemplate string) (*inMemoryFile, error) { // read template file contents templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate) if err != nil { return nil, err } contentTemplate, err := io.ReadAll(templateReader) if err != nil { return nil, err } // read file permissions info, err := r.templateFiler.Stat(r.ctx, relPathTemplate) if err != nil { return nil, err } perm := info.Mode().Perm() // execute the contents of the file as a template content, err := r.executeTemplate(string(contentTemplate)) // Capture errors caused by the "fail" helper function if target := (&ErrFail{}); errors.As(err, target) { return nil, target } if err != nil { return nil, fmt.Errorf("failed to compute file content for %s. %w", relPathTemplate, err) } // Execute relative path template to get materialized path for the file relPath, err := r.executeTemplate(relPathTemplate) if err != nil { return nil, err } return &inMemoryFile{ root: r.instanceRoot, relPath: relPath, content: []byte(content), perm: perm, }, nil } // This function walks the template file tree to generate an in memory representation // of a project. // // During file tree walk, in the current directory, we would like to determine // all possible {{skip}} function calls before we process any of the directories // so that we can skip them eagerly if needed. That is in the current working directory // we would like to process all files before we process any of the directories. // // This is not possible using the std library WalkDir which processes the files in // lexical order which is why this function implements BFS. func (r *renderer) walk() error { directories := []string{"."} var currentDirectory string for len(directories) > 0 { currentDirectory, directories = directories[0], directories[1:] // Skip current directory if it matches any of accumulated skip patterns instanceDirectory, err := r.executeTemplate(currentDirectory) if err != nil { return err } isSkipped, err := r.isSkipped(instanceDirectory) if err != nil { return err } if isSkipped { logger.Infof(r.ctx, "skipping directory: %s", instanceDirectory) continue } // Add skip function, which accumulates skip patterns relative to current // directory r.baseTemplate.Funcs(template.FuncMap{ "skip": func(relPattern string) string { // patterns are specified relative to current directory of the file // the {{skip}} function is called from. pattern := path.Join(currentDirectory, relPattern) if !slices.Contains(r.skipPatterns, pattern) { logger.Infof(r.ctx, "adding skip pattern: %s", pattern) r.skipPatterns = append(r.skipPatterns, pattern) } // return empty string will print nothing at function call site // when executing the template return "" }, }) // Process all entries in current directory // // 1. For files: the templates in the file name and content are executed, and // a in memory representation of the file is generated // // 2. For directories: They are appended to a slice, which acts as a queue // allowing BFS traversal of the template file tree entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory) if err != nil { return err } for _, entry := range entries { if entry.IsDir() { // Add to slice, for BFS traversal directories = append(directories, path.Join(currentDirectory, entry.Name())) continue } // Generate in memory representation of file f, err := r.computeFile(path.Join(currentDirectory, entry.Name())) if err != nil { return err } logger.Infof(r.ctx, "added file to list of in memory files: %s", f.relPath) r.files = append(r.files, f) } } return nil } func (r *renderer) persistToDisk() error { // Accumulate files which we will persist, skipping files whose path matches // any of the skip patterns filesToPersist := make([]*inMemoryFile, 0) for _, file := range r.files { isSkipped, err := r.isSkipped(file.relPath) if err != nil { return err } if isSkipped { log.Infof(r.ctx, "skipping file: %s", file.relPath) continue } filesToPersist = append(filesToPersist, file) } // Assert no conflicting files exist for _, file := range filesToPersist { path := file.fullPath() _, err := os.Stat(path) if err == nil { return fmt.Errorf("failed to persist to disk, conflict with existing file: %s", path) } if err != nil && !os.IsNotExist(err) { return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err) } } // Persist files to disk for _, file := range filesToPersist { err := file.persistToDisk() if err != nil { return err } } return nil } func (r *renderer) isSkipped(filePath string) (bool, error) { for _, pattern := range r.skipPatterns { isMatch, err := path.Match(pattern, filePath) if err != nil { return false, err } if isMatch { return true, nil } } return false, nil }