From 75b09ff230105eac4ff6a24881289729a8fea64e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 20 Nov 2024 11:11:31 +0100 Subject: [PATCH] Use `filer.Filer` to write template instantiation (#1911) ## Changes Prior to this change, the output directory was part of the `renderer` type and passed down to every `file` it produced. Every file knew its absolute destination path. This is incompatible with the use of a filer, where all operations are automatically anchored to some base path. To make this compatible, this change updates: * the `file` type to only know its own path relative to the instantiation root, * the `renderer` type to no longer require or pass along the output directory, * the `persistToDisk` function to take a context and filer argument, * the `filer.WriteMode` to represent permission bits ## Tests * Existing tests pass. * Manually confirmed template initialization works as expected. --- libs/filer/filer.go | 13 +++- libs/filer/filer_test.go | 12 ++++ libs/filer/local_client.go | 13 +++- libs/template/config_test.go | 4 +- libs/template/file.go | 84 ++++++++----------------- libs/template/file_test.go | 62 +++++++------------ libs/template/helpers_test.go | 24 +++---- libs/template/materialize.go | 10 ++- libs/template/renderer.go | 32 +++------- libs/template/renderer_test.go | 110 +++++++++++++++------------------ 10 files changed, 161 insertions(+), 203 deletions(-) create mode 100644 libs/filer/filer_test.go diff --git a/libs/filer/filer.go b/libs/filer/filer.go index fcfbcea0..b5be4c3c 100644 --- a/libs/filer/filer.go +++ b/libs/filer/filer.go @@ -7,13 +7,24 @@ import ( "io/fs" ) +// WriteMode captures intent when writing a file. +// +// The first 9 bits are reserved for the [fs.FileMode] permission bits. +// These are used only by the local filer implementation and have +// no effect for the other implementations. type WriteMode int +// writeModePerm is a mask to extract permission bits from a WriteMode. +const writeModePerm = WriteMode(fs.ModePerm) + const ( - OverwriteIfExists WriteMode = 1 << iota + // Note: these constants are defined as powers of 2 to support combining them using a bit-wise OR. + // They starts from the 10th bit (permission mask + 1) to avoid conflicts with the permission bits. + OverwriteIfExists WriteMode = (writeModePerm + 1) << iota CreateParentDirectories ) +// DeleteMode captures intent when deleting a file. type DeleteMode int const ( diff --git a/libs/filer/filer_test.go b/libs/filer/filer_test.go new file mode 100644 index 00000000..bacea730 --- /dev/null +++ b/libs/filer/filer_test.go @@ -0,0 +1,12 @@ +package filer + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteMode(t *testing.T) { + assert.Equal(t, 512, int(OverwriteIfExists)) + assert.Equal(t, 1024, int(CreateParentDirectories)) +} diff --git a/libs/filer/local_client.go b/libs/filer/local_client.go index 48e8a05e..8b25345f 100644 --- a/libs/filer/local_client.go +++ b/libs/filer/local_client.go @@ -28,6 +28,15 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader, return err } + // Retrieve permission mask from the [WriteMode], if present. + perm := fs.FileMode(0644) + for _, m := range mode { + bits := m & writeModePerm + if bits != 0 { + perm = fs.FileMode(bits) + } + } + flags := os.O_WRONLY | os.O_CREATE if slices.Contains(mode, OverwriteIfExists) { flags |= os.O_TRUNC @@ -35,7 +44,7 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader, flags |= os.O_EXCL } - f, err := os.OpenFile(absPath, flags, 0644) + f, err := os.OpenFile(absPath, flags, perm) if errors.Is(err, fs.ErrNotExist) && slices.Contains(mode, CreateParentDirectories) { // Create parent directories if they don't exist. err = os.MkdirAll(filepath.Dir(absPath), 0755) @@ -43,7 +52,7 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader, return err } // Try again. - f, err = os.OpenFile(absPath, flags, 0644) + f, err = os.OpenFile(absPath, flags, perm) } if err != nil { diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 49d3423e..a855019b 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -83,7 +83,7 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { c, err := newConfig(ctx, os.DirFS(testDir), "schema.json") require.NoError(t, err) - r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library", t.TempDir()) + r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library") require.NoError(t, err) err = c.assignDefaultValues(r) @@ -102,7 +102,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { c, err := newConfig(ctx, os.DirFS(testDir), "schema.json") require.NoError(t, err) - r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library"), t.TempDir()) + r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library")) require.NoError(t, err) // Note: only the string value is templated. diff --git a/libs/template/file.go b/libs/template/file.go index 5492ebeb..36d079b3 100644 --- a/libs/template/file.go +++ b/libs/template/file.go @@ -1,53 +1,36 @@ package template import ( + "bytes" "context" - "io" "io/fs" - "os" - "path/filepath" "slices" + + "github.com/databricks/cli/libs/filer" ) // Interface representing a file to be materialized from a template into a project // instance type file interface { - // Destination path for file. This is where the file will be created when - // PersistToDisk is called. - DstPath() *destinationPath + // Path of the file relative to the root of the instantiated template. + // This is where the file is written to when persisting the template to disk. + // Must be slash-separated. + RelPath() string // Write file to disk at the destination path. - PersistToDisk() error + Write(ctx context.Context, out filer.Filer) error // contents returns the file contents as a byte slice. // This is used for testing purposes. contents() ([]byte, error) } -type destinationPath struct { - // Root path for the project instance. This path uses the system's default - // file separator. For example /foo/bar on Unix and C:\foo\bar on windows - root string - - // Unix like file path relative to the "root" of the instantiated project. Is used to - // evaluate whether the file should be skipped by comparing it to a list of - // skip glob patterns. - relPath string -} - -// Absolute path of the file, in the os native format. For example /foo/bar on -// Unix and C:\foo\bar on windows -func (f *destinationPath) absPath() string { - return filepath.Join(f.root, filepath.FromSlash(f.relPath)) -} - type copyFile struct { - ctx context.Context - // Permissions bits for the destination file perm fs.FileMode - dstPath *destinationPath + // Destination path for the file. + relPath string // [fs.FS] rooted at template root. Used to read srcPath. srcFS fs.FS @@ -56,28 +39,17 @@ type copyFile struct { srcPath string } -func (f *copyFile) DstPath() *destinationPath { - return f.dstPath +func (f *copyFile) RelPath() string { + return f.relPath } -func (f *copyFile) PersistToDisk() error { - path := f.DstPath().absPath() - err := os.MkdirAll(filepath.Dir(path), 0755) +func (f *copyFile) Write(ctx context.Context, out filer.Filer) error { + src, err := f.srcFS.Open(f.srcPath) if err != nil { return err } - srcFile, err := f.srcFS.Open(f.srcPath) - if err != nil { - return err - } - defer srcFile.Close() - dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, f.perm) - if err != nil { - return err - } - defer dstFile.Close() - _, err = io.Copy(dstFile, srcFile) - return err + defer src.Close() + return out.Write(ctx, f.relPath, src, filer.CreateParentDirectories, filer.WriteMode(f.perm)) } func (f *copyFile) contents() ([]byte, error) { @@ -85,26 +57,22 @@ func (f *copyFile) contents() ([]byte, error) { } type inMemoryFile struct { - dstPath *destinationPath - - content []byte - // Permissions bits for the destination file perm fs.FileMode + + // Destination path for the file. + relPath string + + // Contents of the file. + content []byte } -func (f *inMemoryFile) DstPath() *destinationPath { - return f.dstPath +func (f *inMemoryFile) RelPath() string { + return f.relPath } -func (f *inMemoryFile) PersistToDisk() error { - path := f.DstPath().absPath() - - err := os.MkdirAll(filepath.Dir(path), 0755) - if err != nil { - return err - } - return os.WriteFile(path, f.content, f.perm) +func (f *inMemoryFile) Write(ctx context.Context, out filer.Filer) error { + return out.Write(ctx, f.relPath, bytes.NewReader(f.content), filer.CreateParentDirectories, filer.WriteMode(f.perm)) } func (f *inMemoryFile) contents() ([]byte, error) { diff --git a/libs/template/file_test.go b/libs/template/file_test.go index e1bd5456..bd5f6d63 100644 --- a/libs/template/file_test.go +++ b/libs/template/file_test.go @@ -8,77 +8,56 @@ import ( "runtime" "testing" + "github.com/databricks/cli/libs/filer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testInMemoryFile(t *testing.T, perm fs.FileMode) { +func testInMemoryFile(t *testing.T, ctx context.Context, perm fs.FileMode) { tmpDir := t.TempDir() f := &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a/b/c", - }, perm: perm, + relPath: "a/b/c", content: []byte("123"), } - err := f.PersistToDisk() + + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = f.Write(ctx, out) assert.NoError(t, err) assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "123") assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm) } -func testCopyFile(t *testing.T, perm fs.FileMode) { +func testCopyFile(t *testing.T, ctx context.Context, perm fs.FileMode) { tmpDir := t.TempDir() err := os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm) require.NoError(t, err) f := ©File{ - ctx: context.Background(), - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a/b/c", - }, perm: perm, - srcPath: "source", + relPath: "a/b/c", srcFS: os.DirFS(tmpDir), + srcPath: "source", } - err = f.PersistToDisk() + + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = f.Write(ctx, out) assert.NoError(t, err) assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "qwerty") assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm) } -func TestTemplateFileDestinationPath(t *testing.T) { - if runtime.GOOS == "windows" { - t.SkipNow() - } - f := &destinationPath{ - root: `a/b/c`, - relPath: "d/e", - } - assert.Equal(t, `a/b/c/d/e`, f.absPath()) -} - -func TestTemplateFileDestinationPathForWindows(t *testing.T) { - if runtime.GOOS != "windows" { - t.SkipNow() - } - f := &destinationPath{ - root: `c:\a\b\c`, - relPath: "d/e", - } - assert.Equal(t, `c:\a\b\c\d\e`, f.absPath()) -} - func TestTemplateInMemoryFilePersistToDisk(t *testing.T) { if runtime.GOOS == "windows" { t.SkipNow() } - testInMemoryFile(t, 0755) + ctx := context.Background() + testInMemoryFile(t, ctx, 0755) } func TestTemplateInMemoryFilePersistToDiskForWindows(t *testing.T) { @@ -87,14 +66,16 @@ func TestTemplateInMemoryFilePersistToDiskForWindows(t *testing.T) { } // we have separate tests for windows because of differences in valid // fs.FileMode values we can use for different operating systems. - testInMemoryFile(t, 0666) + ctx := context.Background() + testInMemoryFile(t, ctx, 0666) } func TestTemplateCopyFilePersistToDisk(t *testing.T) { if runtime.GOOS == "windows" { t.SkipNow() } - testCopyFile(t, 0644) + ctx := context.Background() + testCopyFile(t, ctx, 0644) } func TestTemplateCopyFilePersistToDiskForWindows(t *testing.T) { @@ -103,5 +84,6 @@ func TestTemplateCopyFilePersistToDiskForWindows(t *testing.T) { } // we have separate tests for windows because of differences in valid // fs.FileMode values we can use for different operating systems. - testCopyFile(t, 0666) + ctx := context.Background() + testCopyFile(t, ctx, 0666) } diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go index 8a779ecc..9f5804c0 100644 --- a/libs/template/helpers_test.go +++ b/libs/template/helpers_test.go @@ -18,11 +18,10 @@ import ( func TestTemplatePrintStringWithoutProcessing(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library") require.NoError(t, err) err = r.walk() @@ -35,11 +34,10 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) { func TestTemplateRegexpCompileFunction(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library") require.NoError(t, err) err = r.walk() @@ -53,11 +51,10 @@ func TestTemplateRegexpCompileFunction(t *testing.T) { func TestTemplateRandIntFunction(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library") require.NoError(t, err) err = r.walk() @@ -71,11 +68,10 @@ func TestTemplateRandIntFunction(t *testing.T) { func TestTemplateUuidFunction(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library") require.NoError(t, err) err = r.walk() @@ -88,11 +84,10 @@ func TestTemplateUuidFunction(t *testing.T) { func TestTemplateUrlFunction(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library") require.NoError(t, err) @@ -105,11 +100,10 @@ func TestTemplateUrlFunction(t *testing.T) { func TestTemplateMapPairFunction(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library") require.NoError(t, err) @@ -122,7 +116,6 @@ func TestTemplateMapPairFunction(t *testing.T) { func TestWorkspaceHost(t *testing.T) { ctx := context.Background() - tmpDir := t.TempDir() w := &databricks.WorkspaceClient{ Config: &workspaceConfig.Config{ @@ -132,7 +125,7 @@ func TestWorkspaceHost(t *testing.T) { ctx = root.SetWorkspaceClient(ctx, w) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library") require.NoError(t, err) @@ -149,7 +142,6 @@ func TestWorkspaceHostNotConfigured(t *testing.T) { ctx := context.Background() cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "template") ctx = cmdio.InContext(ctx, cmd) - tmpDir := t.TempDir() w := &databricks.WorkspaceClient{ Config: &workspaceConfig.Config{}, @@ -157,7 +149,7 @@ func TestWorkspaceHostNotConfigured(t *testing.T) { ctx = root.SetWorkspaceClient(ctx, w) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library") assert.NoError(t, err) diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 0163eb7d..8338e119 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -7,6 +7,7 @@ import ( "io/fs" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" ) const libraryDirName = "library" @@ -40,7 +41,7 @@ func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, o } helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName, outputDir) + r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName) if err != nil { return err } @@ -72,7 +73,12 @@ func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, o return err } - err = r.persistToDisk() + out, err := filer.NewLocalClient(outputDir) + if err != nil { + return err + } + + err = r.persistToDisk(ctx, out) if err != nil { return err } diff --git a/libs/template/renderer.go b/libs/template/renderer.go index bc865039..0f30a67d 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/fs" - "os" "path" "regexp" "slices" @@ -14,6 +13,7 @@ import ( "strings" "text/template" + "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/logger" ) @@ -52,9 +52,6 @@ type renderer struct { // [fs.FS] that holds the template's file tree. srcFS fs.FS - - // Root directory for the project instantiated from the template - instanceRoot string } func newRenderer( @@ -64,7 +61,6 @@ func newRenderer( templateFS fs.FS, templateDir string, libraryDir string, - instanceRoot string, ) (*renderer, error) { // Initialize new template, with helper functions loaded tmpl := template.New("").Funcs(helpers) @@ -99,7 +95,6 @@ func newRenderer( files: make([]file, 0), skipPatterns: make([]string, 0), srcFS: srcFS, - instanceRoot: instanceRoot, }, nil } @@ -165,12 +160,8 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) { // over as is, without treating it as a template if !strings.HasSuffix(relPathTemplate, templateExtension) { return ©File{ - dstPath: &destinationPath{ - root: r.instanceRoot, - relPath: relPath, - }, perm: perm, - ctx: r.ctx, + relPath: relPath, srcFS: r.srcFS, srcPath: relPathTemplate, }, nil @@ -202,11 +193,8 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) { } return &inMemoryFile{ - dstPath: &destinationPath{ - root: r.instanceRoot, - relPath: relPath, - }, perm: perm, + relPath: relPath, content: []byte(content), }, nil } @@ -291,7 +279,7 @@ func (r *renderer) walk() error { if err != nil { return err } - logger.Infof(r.ctx, "added file to list of possible project files: %s", f.DstPath().relPath) + logger.Infof(r.ctx, "added file to list of possible project files: %s", f.RelPath()) r.files = append(r.files, f) } @@ -299,17 +287,17 @@ func (r *renderer) walk() error { return nil } -func (r *renderer) persistToDisk() error { +func (r *renderer) persistToDisk(ctx context.Context, out filer.Filer) error { // Accumulate files which we will persist, skipping files whose path matches // any of the skip patterns filesToPersist := make([]file, 0) for _, file := range r.files { - match, err := isSkipped(file.DstPath().relPath, r.skipPatterns) + match, err := isSkipped(file.RelPath(), r.skipPatterns) if err != nil { return err } if match { - log.Infof(r.ctx, "skipping file: %s", file.DstPath()) + log.Infof(r.ctx, "skipping file: %s", file.RelPath()) continue } filesToPersist = append(filesToPersist, file) @@ -317,8 +305,8 @@ func (r *renderer) persistToDisk() error { // Assert no conflicting files exist for _, file := range filesToPersist { - path := file.DstPath().absPath() - _, err := os.Stat(path) + path := file.RelPath() + _, err := out.Stat(ctx, path) if err == nil { return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path) } @@ -329,7 +317,7 @@ func (r *renderer) persistToDisk() error { // Persist files to disk for _, file := range filesToPersist { - err := file.PersistToDisk() + err := file.Write(ctx, out) if err != nil { return err } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 9b8861e7..a4b9166d 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go" workspaceConfig "github.com/databricks/databricks-sdk-go/config" @@ -57,13 +58,15 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri ctx = root.SetWorkspaceClient(ctx, w) helpers := loadHelpers(ctx) - renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName, tempDir) + renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName) require.NoError(t, err) // Evaluate template err = renderer.walk() require.NoError(t, err) - err = renderer.persistToDisk() + out, err := filer.NewLocalClient(tempDir) + require.NoError(t, err) + err = renderer.persistToDisk(ctx, out) require.NoError(t, err) b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project")) @@ -181,13 +184,14 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library") require.NoError(t, err) err = r.walk() require.NoError(t, err) - - err = r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) require.NoError(t, err) b, err := os.ReadFile(filepath.Join(tmpDir, "my_email")) @@ -312,45 +316,34 @@ func TestRendererPersistToDisk(t *testing.T) { r := &renderer{ ctx: ctx, - instanceRoot: tmpDir, skipPatterns: []string{"a/b/c", "mn*"}, files: []file{ &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a/b/c", - }, perm: 0444, + relPath: "a/b/c", content: nil, }, &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "mno", - }, perm: 0444, + relPath: "mno", content: nil, }, &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a/b/d", - }, perm: 0444, + relPath: "a/b/d", content: []byte("123"), }, &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "mmnn", - }, perm: 0444, + relPath: "mmnn", content: []byte("456"), }, }, } - err := r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) require.NoError(t, err) assert.NoFileExists(t, filepath.Join(tmpDir, "a", "b", "c")) @@ -365,10 +358,9 @@ func TestRendererPersistToDisk(t *testing.T) { func TestRendererWalk(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library") require.NoError(t, err) err = r.walk() @@ -376,7 +368,7 @@ func TestRendererWalk(t *testing.T) { getContent := func(r *renderer, path string) string { for _, f := range r.files { - if f.DstPath().relPath != path { + if f.RelPath() != path { continue } b, err := f.contents() @@ -397,10 +389,9 @@ func TestRendererWalk(t *testing.T) { func TestRendererFailFunction(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library") require.NoError(t, err) err = r.walk() @@ -410,10 +401,9 @@ func TestRendererFailFunction(t *testing.T) { func TestRendererSkipsDirsEagerly(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library") require.NoError(t, err) err = r.walk() @@ -430,7 +420,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) { tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library") require.NoError(t, err) err = r.walk() @@ -438,7 +428,9 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) { // All 3 files are executed and have in memory representations require.Len(t, r.files, 3) - err = r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) require.NoError(t, err) entries, err := os.ReadDir(tmpDir) @@ -450,10 +442,9 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) { func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library") require.NoError(t, err) err = r.walk() @@ -471,7 +462,7 @@ func TestRendererSkip(t *testing.T) { tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library") require.NoError(t, err) err = r.walk() @@ -480,7 +471,9 @@ func TestRendererSkip(t *testing.T) { // This is because "dir2/*" matches the files in dir2, but not dir2 itself assert.Len(t, r.files, 6) - err = r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) require.NoError(t, err) assert.FileExists(t, filepath.Join(tmpDir, "file1")) @@ -498,12 +491,11 @@ func TestRendererReadsPermissionsBits(t *testing.T) { if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { t.SkipNow() } - tmpDir := t.TempDir() ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library") require.NoError(t, err) err = r.walk() @@ -511,7 +503,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) { getPermissions := func(r *renderer, path string) fs.FileMode { for _, f := range r.files { - if f.DstPath().relPath != path { + if f.RelPath() != path { continue } switch v := f.(type) { @@ -534,6 +526,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) { func TestRendererErrorOnConflictingFile(t *testing.T) { tmpDir := t.TempDir() + ctx := context.Background() f, err := os.Create(filepath.Join(tmpDir, "a")) require.NoError(t, err) @@ -544,17 +537,16 @@ func TestRendererErrorOnConflictingFile(t *testing.T) { skipPatterns: []string{}, files: []file{ &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a", - }, perm: 0444, + relPath: "a", content: []byte("123"), }, }, } - err = r.persistToDisk() - assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", filepath.Join(tmpDir, "a"))) + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) + assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", "a")) } func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) { @@ -571,16 +563,15 @@ func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) { skipPatterns: []string{"a"}, files: []file{ &inMemoryFile{ - dstPath: &destinationPath{ - root: tmpDir, - relPath: "a", - }, perm: 0444, + relPath: "a", content: []byte("123"), }, }, } - err = r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) // No error is returned even though a conflicting file exists. This is because // the generated file is being skipped assert.NoError(t, err) @@ -590,10 +581,9 @@ func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) { func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir) + r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library") require.NoError(t, err) err = r.walk() @@ -601,7 +591,7 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) { assert.Len(t, r.files, 1) assert.Equal(t, r.files[0].(*copyFile).srcPath, "not-a-template") - assert.Equal(t, r.files[0].DstPath().absPath(), filepath.Join(tmpDir, "not-a-template")) + assert.Equal(t, r.files[0].RelPath(), "not-a-template") } func TestRendererFileTreeRendering(t *testing.T) { @@ -613,7 +603,7 @@ func TestRendererFileTreeRendering(t *testing.T) { r, err := newRenderer(ctx, map[string]any{ "dir_name": "my_directory", "file_name": "my_file", - }, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir) + }, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library") require.NoError(t, err) err = r.walk() @@ -621,9 +611,11 @@ func TestRendererFileTreeRendering(t *testing.T) { // Assert in memory representation is created. assert.Len(t, r.files, 1) - assert.Equal(t, r.files[0].DstPath().absPath(), filepath.Join(tmpDir, "my_directory", "my_file")) + assert.Equal(t, r.files[0].RelPath(), "my_directory/my_file") - err = r.persistToDisk() + out, err := filer.NewLocalClient(tmpDir) + require.NoError(t, err) + err = r.persistToDisk(ctx, out) require.NoError(t, err) // Assert files and directories are correctly materialized. @@ -645,8 +637,7 @@ func TestRendererSubTemplateInPath(t *testing.T) { // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file. testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}")) - tmpDir := t.TempDir() - r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library", tmpDir) + r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library") require.NoError(t, err) err = r.walk() @@ -654,7 +645,6 @@ func TestRendererSubTemplateInPath(t *testing.T) { if assert.Len(t, r.files, 2) { f := r.files[1] - assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), f.DstPath().absPath()) - assert.Equal(t, "my_directory/my_file", f.DstPath().relPath) + assert.Equal(t, "my_directory/my_file", f.RelPath()) } }