Merge 977ba12f25 into 756e55fabc

## Changes

This change adds a preset for source-linked deployments. It is enabled
by default for targets in `development` mode **if** the Databricks CLI
is running from the `/Workspace` directory on DBR. It does not have an
effect when running the CLI anywhere else.

Key highlights:
1. Files in this mode won't be uploaded to workspace
2. Created resources will use references to source files instead of
their workspace copies

## Tests
1. Apply preset unit test covering conditional logic
2. High-level process target mode unit test for testing integration
between mutators

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

bundle/config/mutator/apply_presets.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/textutil"
@@ -221,6 +222,15 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 		dashboard.DisplayName = prefix + dashboard.DisplayName
 	}
 
+	if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
+		isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
+		if !isDatabricksWorkspace {
+			disabled := false
+			b.Config.Presets.SourceLinkedDeployment = &disabled
+			diags = diags.Extend(diag.Warningf("source-linked deployment is available only in the Databricks Workspace"))
+		}
+	}
+
 	return diags
 }
 

bundle/config/mutator/apply_presets_test.go

@@ -2,12 +2,14 @@ package mutator_test
 
 import (
 	"context"
+	"runtime"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@@ -364,3 +366,86 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
+	}
+
+	testContext := context.Background()
+	enabled := true
+	disabled := false
+	workspacePath := "/Workspace/user.name@company.com"
+
+	tests := []struct {
+		bundlePath      string
+		ctx             context.Context
+		name            string
+		initialValue    *bool
+		expectedValue   *bool
+		expectedWarning string
+	}{
+		{
+			name:          "preset enabled, bundle in Workspace, databricks runtime",
+			bundlePath:    workspacePath,
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &enabled,
+			expectedValue: &enabled,
+		},
+		{
+			name:            "preset enabled, bundle not in Workspace, databricks runtime",
+			bundlePath:      "/Users/user.name@company.com",
+			ctx:             dbr.MockRuntime(testContext, true),
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:            "preset enabled, bundle in Workspace, not databricks runtime",
+			bundlePath:      workspacePath,
+			ctx:             dbr.MockRuntime(testContext, false),
+			initialValue:    &enabled,
+			expectedValue:   &disabled,
+			expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
+		},
+		{
+			name:          "preset disabled, bundle in Workspace, databricks runtime",
+			bundlePath:    workspacePath,
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  &disabled,
+			expectedValue: &disabled,
+		},
+		{
+			name:          "preset nil, bundle in Workspace, databricks runtime",
+			bundlePath:    workspacePath,
+			ctx:           dbr.MockRuntime(testContext, true),
+			initialValue:  nil,
+			expectedValue: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b := &bundle.Bundle{
+				SyncRootPath: tt.bundlePath,
+				Config: config.Root{
+					Presets: config.Presets{
+						SourceLinkedDeployment: tt.initialValue,
+					},
+				},
+			}
+
+			diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
+			if diags.HasError() {
+				t.Fatalf("unexpected error: %v", diags)
+			}
+
+			if tt.expectedWarning != "" {
+				require.Equal(t, tt.expectedWarning, diags[0].Summary)
+			}
+
+			require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
+		})
+	}
+
+}

bundle/config/mutator/process_target_mode.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/iamutil"
@@ -57,6 +58,14 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
 		t.TriggerPauseStatus = config.Paused
 	}
 
+	if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
+		isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
+		if isInWorkspace && dbr.RunsOnRuntime(ctx) {
+			enabled := true
+			t.SourceLinkedDeployment = &enabled
+		}
+	}
+
 	if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
 		enabled := true
 		t.PipelinesDevelopment = &enabled

bundle/config/mutator/process_target_mode_test.go

@@ -3,14 +3,17 @@ package mutator
 import (
 	"context"
 	"reflect"
+	"runtime"
 	"strings"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/tags"
+	"github.com/databricks/cli/libs/vfs"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/compute"
@@ -140,6 +143,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 				},
 			},
 		},
+		SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
 		// Use AWS implementation for testing.
 		Tagging: tags.ForCloud(&sdkconfig.Config{
 			Host: "https://company.cloud.databricks.com",
@@ -522,3 +526,32 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
 
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 }
+
+func TestSourceLinkedDeploymentEnabled(t *testing.T) {
+	b, diags := processSourceLinkedBundle(t, true)
+	require.NoError(t, diags.Error())
+	assert.True(t, *b.Config.Presets.SourceLinkedDeployment)
+}
+
+func TestSourceLinkedDeploymentDisabled(t *testing.T) {
+	b, diags := processSourceLinkedBundle(t, false)
+	require.NoError(t, diags.Error())
+	assert.False(t, *b.Config.Presets.SourceLinkedDeployment)
+}
+
+func processSourceLinkedBundle(t *testing.T, presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
+	}
+
+	b := mockBundle(config.Development)
+
+	workspacePath := "/Workspace/lennart@company.com/"
+	b.SyncRootPath = workspacePath
+	b.Config.Presets.SourceLinkedDeployment = &presetEnabled
+
+	ctx := dbr.MockRuntime(context.Background(), true)
+	m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
+	diags := bundle.Apply(ctx, b, m)
+	return b, diags
+}

bundle/config/mutator/translate_paths.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/notebook"
@@ -103,8 +104,13 @@ func (t *translateContext) rewritePath(
 		return fmt.Errorf("path %s is not contained in sync root path", localPath)
 	}
 
-	// Prefix remote path with its remote root path.
-	remotePath := path.Join(t.b.Config.Workspace.FilePath, filepath.ToSlash(localRelPath))
+	var workspacePath string
+	if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) {
+		workspacePath = t.b.SyncRootPath
+	} else {
+		workspacePath = t.b.Config.Workspace.FilePath
+	}
+	remotePath := path.Join(workspacePath, filepath.ToSlash(localRelPath))
 
 	// Convert local path into workspace path via specified function.
 	interp, err := fn(*p, localPath, localRelPath, remotePath)

bundle/config/mutator/translate_paths_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -787,3 +788,163 @@ func TestTranslatePathWithComplexVariables(t *testing.T) {
 		b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
 	)
 }
+
+func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
+	}
+
+	dir := t.TempDir()
+	touchNotebookFile(t, filepath.Join(dir, "my_job_notebook.py"))
+	touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
+	touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
+	touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
+	touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
+
+	enabled := true
+	b := &bundle.Bundle{
+		SyncRootPath: dir,
+		SyncRoot:     vfs.MustNew(dir),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath: "/bundle",
+			},
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "my_job_notebook.py",
+									},
+									Libraries: []compute.Library{
+										{Whl: "./dist/task.whl"},
+									},
+								},
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "/Users/jane.doe@databricks.com/absolute_remote.py",
+									},
+								},
+								{
+									NotebookTask: &jobs.NotebookTask{
+										NotebookPath: "my_job_notebook.py",
+									},
+									Libraries: []compute.Library{
+										{Requirements: "requirements.txt"},
+									},
+								},
+								{
+									SparkPythonTask: &jobs.SparkPythonTask{
+										PythonFile: "my_python_file.py",
+									},
+								},
+								{
+									SparkJarTask: &jobs.SparkJarTask{
+										MainClassName: "HelloWorld",
+									},
+									Libraries: []compute.Library{
+										{Jar: "./dist/task.jar"},
+									},
+								},
+								{
+									SparkJarTask: &jobs.SparkJarTask{
+										MainClassName: "HelloWorldRemote",
+									},
+									Libraries: []compute.Library{
+										{Jar: "dbfs:/bundle/dist/task_remote.jar"},
+									},
+								},
+							},
+						},
+					},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					"pipeline": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Libraries: []pipelines.PipelineLibrary{
+								{
+									Notebook: &pipelines.NotebookLibrary{
+										Path: "my_pipeline_notebook.py",
+									},
+								},
+								{
+									Notebook: &pipelines.NotebookLibrary{
+										Path: "/Users/jane.doe@databricks.com/absolute_remote.py",
+									},
+								},
+								{
+									File: &pipelines.FileLibrary{
+										Path: "my_python_file.py",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			Presets: config.Presets{
+				SourceLinkedDeployment: &enabled,
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}})
+	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// updated to source path
+	assert.Equal(
+		t,
+		filepath.Join(dir, "my_job_notebook"),
+		b.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath,
+	)
+	assert.Equal(
+		t,
+		filepath.Join(dir, "requirements.txt"),
+		b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
+	)
+	assert.Equal(
+		t,
+		filepath.Join(dir, "my_python_file.py"),
+		b.Config.Resources.Jobs["job"].Tasks[3].SparkPythonTask.PythonFile,
+	)
+	assert.Equal(
+		t,
+		filepath.Join(dir, "my_pipeline_notebook"),
+		b.Config.Resources.Pipelines["pipeline"].Libraries[0].Notebook.Path,
+	)
+	assert.Equal(
+		t,
+		filepath.Join(dir, "my_python_file.py"),
+		b.Config.Resources.Pipelines["pipeline"].Libraries[2].File.Path,
+	)
+
+	// left as is
+	assert.Equal(
+		t,
+		filepath.Join("dist", "task.whl"),
+		b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
+	)
+	assert.Equal(
+		t,
+		"/Users/jane.doe@databricks.com/absolute_remote.py",
+		b.Config.Resources.Jobs["job"].Tasks[1].NotebookTask.NotebookPath,
+	)
+	assert.Equal(
+		t,
+		filepath.Join("dist", "task.jar"),
+		b.Config.Resources.Jobs["job"].Tasks[4].Libraries[0].Jar,
+	)
+	assert.Equal(
+		t,
+		"dbfs:/bundle/dist/task_remote.jar",
+		b.Config.Resources.Jobs["job"].Tasks[5].Libraries[0].Jar,
+	)
+	assert.Equal(
+		t,
+		"/Users/jane.doe@databricks.com/absolute_remote.py",
+		b.Config.Resources.Pipelines["pipeline"].Libraries[1].Notebook.Path,
+	)
+}

bundle/config/presets.go

@@ -17,6 +17,11 @@ type Presets struct {
 	// JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs.
 	JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"`
 
+	// SourceLinkedDeployment indicates whether source-linked deployment is enabled. Works only in Databricks Workspace
+	// When set to true, resources created during deployment will point to source files in the workspace instead of their workspace copies.
+	// File synchronization to ${workspace.file_path} is skipped.
+	SourceLinkedDeployment *bool `json:"source_linked_deployment,omitempty"`
+
 	// Tags to add to all resources.
 	Tags map[string]string `json:"tags,omitempty"`
 }

bundle/deploy/files/upload.go

@@ -7,6 +7,7 @@ import (
 	"io/fs"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/permissions"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/diag"
@@ -23,6 +24,11 @@ func (m *upload) Name() string {
 }
 
 func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+		cmdio.LogString(ctx, "Source-linked deployment is enabled. Deployed resources reference the source files in your working tree instead of separate copies.")
+		return nil
+	}
+
 	cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath))
 	opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b))
 	if err != nil {

bundle/trampoline/python_dbr_warning.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/log"
@@ -22,6 +23,9 @@ func WrapperWarning() bundle.Mutator {
 
 func (m *wrapperWarning) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if isPythonWheelWrapperOn(b) {
+		if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
+			return diag.Warningf("Python wheel notebook wrapper is not available when using source-linked deployment mode. You can disable this mode by setting 'presets.source_linked_deployment: false'")
+		}
 		return nil
 	}
 

cmd/bundle/init.go

@@ -1,8 +1,10 @@
 package bundle
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"slices"
@@ -10,6 +12,8 @@ import (
 
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/template"
 	"github.com/spf13/cobra"
@@ -109,6 +113,24 @@ func getUrlForNativeTemplate(name string) string {
 	return ""
 }
 
+func getFsForNativeTemplate(name string) (fs.FS, error) {
+	builtin, err := template.Builtin()
+	if err != nil {
+		return nil, err
+	}
+
+	// If this is a built-in template, the return value will be non-nil.
+	var templateFS fs.FS
+	for _, entry := range builtin {
+		if entry.Name == name {
+			templateFS = entry.FS
+			break
+		}
+	}
+
+	return templateFS, nil
+}
+
 func isRepoUrl(url string) bool {
 	result := false
 	for _, prefix := range gitUrlPrefixes {
@@ -128,6 +150,26 @@ func repoName(url string) string {
 	return parts[len(parts)-1]
 }
 
+func constructOutputFiler(ctx context.Context, outputDir string) (filer.Filer, error) {
+	outputDir, err := filepath.Abs(outputDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the CLI is running on DBR and we're writing to the workspace file system,
+	// use the extension-aware workspace filesystem filer to instantiate the template.
+	//
+	// It is not possible to write notebooks through the workspace filesystem's FUSE mount.
+	// Therefore this is the only way we can initialize templates that contain notebooks
+	// when running the CLI on DBR and initializing a template to the workspace.
+	//
+	if strings.HasPrefix(outputDir, "/Workspace/") && dbr.RunsOnRuntime(ctx) {
+		return filer.NewWorkspaceFilesExtensionsClient(root.WorkspaceClient(ctx), outputDir)
+	}
+
+	return filer.NewLocalClient(outputDir)
+}
+
 func newInitCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init [TEMPLATE_PATH]",
@@ -182,6 +224,11 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 			templatePath = getNativeTemplateByDescription(description)
 		}
 
+		outputFiler, err := constructOutputFiler(ctx, outputDir)
+		if err != nil {
+			return err
+		}
+
 		if templatePath == customTemplate {
 			cmdio.LogString(ctx, "Please specify a path or Git repository to use a custom template.")
 			cmdio.LogString(ctx, "See https://docs.databricks.com/en/dev-tools/bundles/templates.html to learn more about custom templates.")
@@ -198,9 +245,20 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 			if templateDir != "" {
 				return errors.New("--template-dir can only be used with a Git repository URL")
 			}
+
+			templateFS, err := getFsForNativeTemplate(templatePath)
+			if err != nil {
+				return err
+			}
+
+			// If this is not a built-in template, then it must be a local file system path.
+			if templateFS == nil {
+				templateFS = os.DirFS(templatePath)
+			}
+
 			// skip downloading the repo because input arg is not a URL. We assume
 			// it's a path on the local file system in that case
-			return template.Materialize(ctx, configFile, templatePath, outputDir)
+			return template.Materialize(ctx, configFile, templateFS, outputFiler)
 		}
 
 		// Create a temporary directory with the name of the repository.  The '*'
@@ -224,7 +282,8 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 
 		// Clean up downloaded repository once the template is materialized.
 		defer os.RemoveAll(repoDir)
-		return template.Materialize(ctx, configFile, filepath.Join(repoDir, templateDir), outputDir)
+		templateFS := os.DirFS(filepath.Join(repoDir, templateDir))
+		return template.Materialize(ctx, configFile, templateFS, outputFiler)
 	}
 	return cmd
 }

internal/bundle/helpers.go

@@ -16,6 +16,7 @@ import (
 	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/template"
 	"github.com/databricks/cli/libs/vfs"
@@ -42,7 +43,9 @@ func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateN
 	cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
 	ctx = cmdio.InContext(ctx, cmd)
 
-	err = template.Materialize(ctx, configFilePath, templateRoot, bundleRoot)
+	out, err := filer.NewLocalClient(bundleRoot)
+	require.NoError(t, err)
+	err = template.Materialize(ctx, configFilePath, os.DirFS(templateRoot), out)
 	return bundleRoot, err
 }
 

internal/filer_test.go

@@ -723,6 +723,63 @@ func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) {
 	assert.ErrorIs(t, err, fs.ErrNotExist)
 }
 
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotReadAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Reading foo should fail. Even though the WSFS name for the notebook is foo
+	// reading the notebook should only work with the .ipynb extension.
+	_, err = wf.Read(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = wf.Read(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotStatAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Stating foo should fail. Even though the WSFS name for the notebook is foo
+	// stating the notebook should only work with the .ipynb extension.
+	_, err = wf.Stat(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = wf.Stat(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotDeletedAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Deleting foo should fail. Even though the WSFS name for the notebook is foo
+	// deleting the notebook should only work with the .ipynb extension.
+	err = wf.Delete(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	err = wf.Delete(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
 func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) {
 	t.Parallel()
 

libs/filer/filer.go

@@ -7,13 +7,24 @@ import (
 	"io/fs"
 )
 
+// WriteMode captures intent when writing a file.
+//
+// The first 9 bits are reserved for the [fs.FileMode] permission bits.
+// These are used only by the local filer implementation and have
+// no effect for the other implementations.
 type WriteMode int
 
+// writeModePerm is a mask to extract permission bits from a WriteMode.
+const writeModePerm = WriteMode(fs.ModePerm)
+
 const (
-	OverwriteIfExists WriteMode = 1 << iota
+	// Note: these constants are defined as powers of 2 to support combining them using a bit-wise OR.
+	// They starts from the 10th bit (permission mask + 1) to avoid conflicts with the permission bits.
+	OverwriteIfExists WriteMode = (writeModePerm + 1) << iota
 	CreateParentDirectories
 )
 
+// DeleteMode captures intent when deleting a file.
 type DeleteMode int
 
 const (

libs/filer/filer_test.go

@@ -0,0 +1,12 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriteMode(t *testing.T) {
+	assert.Equal(t, 512, int(OverwriteIfExists))
+	assert.Equal(t, 1024, int(CreateParentDirectories))
+}

libs/filer/local_client.go

@@ -28,6 +28,15 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader,
 		return err
 	}
 
+	// Retrieve permission mask from the [WriteMode], if present.
+	perm := fs.FileMode(0644)
+	for _, m := range mode {
+		bits := m & writeModePerm
+		if bits != 0 {
+			perm = fs.FileMode(bits)
+		}
+	}
+
 	flags := os.O_WRONLY | os.O_CREATE
 	if slices.Contains(mode, OverwriteIfExists) {
 		flags |= os.O_TRUNC
@@ -35,7 +44,7 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader,
 		flags |= os.O_EXCL
 	}
 
-	f, err := os.OpenFile(absPath, flags, 0644)
+	f, err := os.OpenFile(absPath, flags, perm)
 	if errors.Is(err, fs.ErrNotExist) && slices.Contains(mode, CreateParentDirectories) {
 		// Create parent directories if they don't exist.
 		err = os.MkdirAll(filepath.Dir(absPath), 0755)
@@ -43,7 +52,7 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader,
 			return err
 		}
 		// Try again.
-		f, err = os.OpenFile(absPath, flags, 0644)
+		f, err = os.OpenFile(absPath, flags, perm)
 	}
 
 	if err != nil {

libs/filer/workspace_files_extensions_client.go

@@ -244,6 +244,17 @@ func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string,
 
 // Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
 func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
+	// Ensure that the file / notebook exists. We do this check here to avoid reading
+	// the content of a notebook called `foo` when the user actually wanted
+	// to read the content of a file called `foo`.
+	//
+	// To read the content of a notebook called `foo` in the workspace the user
+	// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
+	_, err := w.Stat(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
 	r, err := w.wsfs.Read(ctx, name)
 
 	// If the file is not found, it might be a notebook.
@@ -276,7 +287,18 @@ func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string
 		return ReadOnlyError{"delete"}
 	}
 
-	err := w.wsfs.Delete(ctx, name, mode...)
+	// Ensure that the file / notebook exists. We do this check here to avoid
+	// deleting the a notebook called `foo` when the user actually wanted to
+	// delete a file called `foo`.
+	//
+	// To delete a notebook called `foo` in the workspace the user should use the
+	// name with the extension included like `foo.ipynb` or `foo.sql`.
+	_, err := w.Stat(ctx, name)
+	if err != nil {
+		return err
+	}
+
+	err = w.wsfs.Delete(ctx, name, mode...)
 
 	// If the file is not found, it might be a notebook.
 	if errors.As(err, &FileDoesNotExistError{}) {
@@ -315,7 +337,24 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string)
 		return wsfsFileInfo{ObjectInfo: stat.ObjectInfo}, nil
 	}
 
-	return info, err
+	if err != nil {
+		return nil, err
+	}
+
+	// If an object is found and it is a notebook, return a FileDoesNotExistError.
+	// If a notebook is found by the workspace files client, without having stripped
+	// the extension, this implies that no file with the same name exists.
+	//
+	// This check is done to avoid returning the stat for a notebook called `foo`
+	// when the user actually wanted to stat a file called `foo`.
+	//
+	// To stat the metadata of a notebook called `foo` in the workspace the user
+	// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
+	if info.Sys().(workspace.ObjectInfo).ObjectType == workspace.ObjectTypeNotebook {
+		return nil, FileDoesNotExistError{name}
+	}
+
+	return info, nil
 }
 
 // Note: The import API returns opaque internal errors for namespace clashes

libs/jsonschema/schema.go

@@ -3,7 +3,9 @@ package jsonschema
 import (
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"regexp"
 	"slices"
 
@@ -255,7 +257,12 @@ func (schema *Schema) validate() error {
 }
 
 func Load(path string) (*Schema, error) {
-	b, err := os.ReadFile(path)
+	dir, file := filepath.Split(path)
+	return LoadFS(os.DirFS(dir), file)
+}
+
+func LoadFS(fsys fs.FS, path string) (*Schema, error) {
+	b, err := fs.ReadFile(fsys, path)
 	if err != nil {
 		return nil, err
 	}

libs/jsonschema/schema_test.go

@@ -1,6 +1,7 @@
 package jsonschema
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -305,3 +306,9 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) {
 	err = s.validate()
 	assert.NoError(t, err)
 }
+
+func TestSchema_LoadFS(t *testing.T) {
+	fsys := os.DirFS("./testdata/schema-load-int")
+	_, err := LoadFS(fsys, "schema-valid.json")
+	assert.NoError(t, err)
+}

libs/template/builtin.go

@@ -0,0 +1,47 @@
+package template
+
+import (
+	"embed"
+	"io/fs"
+)
+
+//go:embed all:templates
+var builtinTemplates embed.FS
+
+// BuiltinTemplate represents a template that is built into the CLI.
+type BuiltinTemplate struct {
+	Name string
+	FS   fs.FS
+}
+
+// Builtin returns the list of all built-in templates.
+func Builtin() ([]BuiltinTemplate, error) {
+	templates, err := fs.Sub(builtinTemplates, "templates")
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err := fs.ReadDir(templates, ".")
+	if err != nil {
+		return nil, err
+	}
+
+	var out []BuiltinTemplate
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		templateFS, err := fs.Sub(templates, entry.Name())
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, BuiltinTemplate{
+			Name: entry.Name(),
+			FS:   templateFS,
+		})
+	}
+
+	return out, nil
+}

libs/template/builtin_test.go

@@ -0,0 +1,28 @@
+package template
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuiltin(t *testing.T) {
+	out, err := Builtin()
+	require.NoError(t, err)
+	assert.Len(t, out, 3)
+
+	// Confirm names.
+	assert.Equal(t, "dbt-sql", out[0].Name)
+	assert.Equal(t, "default-python", out[1].Name)
+	assert.Equal(t, "default-sql", out[2].Name)
+
+	// Confirm that the filesystems work.
+	_, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
+	assert.NoError(t, err)
+	_, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
+	assert.NoError(t, err)
+	_, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
+	assert.NoError(t, err)
+}

libs/template/config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/jsonschema"
@@ -28,9 +29,8 @@ type config struct {
 	schema *jsonschema.Schema
 }
 
-func newConfig(ctx context.Context, schemaPath string) (*config, error) {
-	// Read config schema
-	schema, err := jsonschema.Load(schemaPath)
+func newConfig(ctx context.Context, templateFS fs.FS, schemaPath string) (*config, error) {
+	schema, err := jsonschema.LoadFS(templateFS, schemaPath)
 	if err != nil {
 		return nil, err
 	}

libs/template/config_test.go

@@ -3,6 +3,8 @@ package template
 import (
 	"context"
 	"fmt"
+	"os"
+	"path"
 	"path/filepath"
 	"testing"
 	"text/template"
@@ -16,7 +18,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) {
 	testDir := "./testdata/config-assign-from-file"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -32,7 +34,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
 	testDir := "./testdata/config-assign-from-file"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -52,7 +54,7 @@ func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T)
 	testDir := "./testdata/config-assign-from-file-invalid-int"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -63,7 +65,7 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te
 	testDir := "./testdata/config-assign-from-file-unknown-property"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -78,10 +80,10 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) {
 	testDir := "./testdata/config-assign-from-default-value"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
-	r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
+	r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library")
 	require.NoError(t, err)
 
 	err = c.assignDefaultValues(r)
@@ -97,10 +99,10 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
 	testDir := "./testdata/config-assign-from-templated-default-value"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
-	r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir())
+	r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library"))
 	require.NoError(t, err)
 
 	// Note: only the string value is templated.
@@ -116,7 +118,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
 
 func TestTemplateConfigValidateValuesDefined(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -131,7 +133,7 @@ func TestTemplateConfigValidateValuesDefined(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -147,7 +149,7 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -164,7 +166,7 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -271,7 +273,8 @@ func TestTemplateEnumValidation(t *testing.T) {
 }
 
 func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) {
-	_, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json")
+	ctx := context.Background()
+	_, err := newConfig(ctx, os.DirFS("./testdata/config-test-schema"), "invalid-test-schema.json")
 	assert.EqualError(t, err, "template property property-without-description is missing a description")
 }
 

libs/template/file.go

@@ -1,11 +1,10 @@
 package template
 
 import (
+	"bytes"
 	"context"
-	"io"
 	"io/fs"
-	"os"
-	"path/filepath"
+	"slices"
 
 	"github.com/databricks/cli/libs/filer"
 )
@@ -13,89 +12,69 @@ import (
 // Interface representing a file to be materialized from a template into a project
 // instance
 type file interface {
-	// Destination path for file. This is where the file will be created when
-	// PersistToDisk is called.
-	DstPath() *destinationPath
+	// Path of the file relative to the root of the instantiated template.
+	// This is where the file is written to when persisting the template to disk.
+	// Must be slash-separated.
+	RelPath() string
 
 	// Write file to disk at the destination path.
-	PersistToDisk() error
-}
+	Write(ctx context.Context, out filer.Filer) error
 
-type destinationPath struct {
-	// Root path for the project instance. This path uses the system's default
-	// file separator. For example /foo/bar on Unix and C:\foo\bar on windows
-	root string
-
-	// Unix like file path relative to the "root" of the instantiated project. Is used to
-	// evaluate whether the file should be skipped by comparing it to a list of
-	// skip glob patterns.
-	relPath string
-}
-
-// Absolute path of the file, in the os native format. For example /foo/bar on
-// Unix and C:\foo\bar on windows
-func (f *destinationPath) absPath() string {
-	return filepath.Join(f.root, filepath.FromSlash(f.relPath))
+	// contents returns the file contents as a byte slice.
+	// This is used for testing purposes.
+	contents() ([]byte, error)
 }
 
 type copyFile struct {
-	ctx context.Context
-
 	// Permissions bits for the destination file
 	perm fs.FileMode
 
-	dstPath *destinationPath
+	// Destination path for the file.
+	relPath string
 
-	// Filer rooted at template root. Used to read srcPath.
-	srcFiler filer.Filer
+	// [fs.FS] rooted at template root. Used to read srcPath.
+	srcFS fs.FS
 
 	// Relative path from template root for file to be copied.
 	srcPath string
 }
 
-func (f *copyFile) DstPath() *destinationPath {
-	return f.dstPath
+func (f *copyFile) RelPath() string {
+	return f.relPath
 }
 
-func (f *copyFile) PersistToDisk() error {
-	path := f.DstPath().absPath()
-	err := os.MkdirAll(filepath.Dir(path), 0755)
+func (f *copyFile) Write(ctx context.Context, out filer.Filer) error {
+	src, err := f.srcFS.Open(f.srcPath)
 	if err != nil {
 		return err
 	}
-	srcFile, err := f.srcFiler.Read(f.ctx, f.srcPath)
-	if err != nil {
-		return err
-	}
-	defer srcFile.Close()
-	dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, f.perm)
-	if err != nil {
-		return err
-	}
-	defer dstFile.Close()
-	_, err = io.Copy(dstFile, srcFile)
-	return err
+	defer src.Close()
+	return out.Write(ctx, f.relPath, src, filer.CreateParentDirectories, filer.WriteMode(f.perm))
+}
+
+func (f *copyFile) contents() ([]byte, error) {
+	return fs.ReadFile(f.srcFS, f.srcPath)
 }
 
 type inMemoryFile struct {
-	dstPath *destinationPath
-
-	content []byte
-
 	// Permissions bits for the destination file
 	perm fs.FileMode
+
+	// Destination path for the file.
+	relPath string
+
+	// Contents of the file.
+	content []byte
 }
 
-func (f *inMemoryFile) DstPath() *destinationPath {
-	return f.dstPath
+func (f *inMemoryFile) RelPath() string {
+	return f.relPath
 }
 
-func (f *inMemoryFile) PersistToDisk() error {
-	path := f.DstPath().absPath()
-
-	err := os.MkdirAll(filepath.Dir(path), 0755)
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(path, f.content, f.perm)
+func (f *inMemoryFile) Write(ctx context.Context, out filer.Filer) error {
+	return out.Write(ctx, f.relPath, bytes.NewReader(f.content), filer.CreateParentDirectories, filer.WriteMode(f.perm))
+}
+
+func (f *inMemoryFile) contents() ([]byte, error) {
+	return slices.Clone(f.content), nil
 }

libs/template/file_test.go

@@ -13,76 +13,51 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func testInMemoryFile(t *testing.T, perm fs.FileMode) {
+func testInMemoryFile(t *testing.T, ctx context.Context, perm fs.FileMode) {
 	tmpDir := t.TempDir()
 
 	f := &inMemoryFile{
-		dstPath: &destinationPath{
-			root:    tmpDir,
-			relPath: "a/b/c",
-		},
 		perm:    perm,
+		relPath: "a/b/c",
 		content: []byte("123"),
 	}
-	err := f.PersistToDisk()
+
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = f.Write(ctx, out)
 	assert.NoError(t, err)
 
 	assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "123")
 	assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm)
 }
 
-func testCopyFile(t *testing.T, perm fs.FileMode) {
+func testCopyFile(t *testing.T, ctx context.Context, perm fs.FileMode) {
 	tmpDir := t.TempDir()
-
-	templateFiler, err := filer.NewLocalClient(tmpDir)
-	require.NoError(t, err)
-	err = os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
+	err := os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
 	require.NoError(t, err)
 
 	f := &copyFile{
-		ctx: context.Background(),
-		dstPath: &destinationPath{
-			root:    tmpDir,
-			relPath: "a/b/c",
-		},
-		perm:     perm,
-		srcPath:  "source",
-		srcFiler: templateFiler,
+		perm:    perm,
+		relPath: "a/b/c",
+		srcFS:   os.DirFS(tmpDir),
+		srcPath: "source",
 	}
-	err = f.PersistToDisk()
+
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = f.Write(ctx, out)
 	assert.NoError(t, err)
 
 	assertFileContent(t, filepath.Join(tmpDir, "a/b/c"), "qwerty")
 	assertFilePermissions(t, filepath.Join(tmpDir, "a/b/c"), perm)
 }
 
-func TestTemplateFileDestinationPath(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.SkipNow()
-	}
-	f := &destinationPath{
-		root:    `a/b/c`,
-		relPath: "d/e",
-	}
-	assert.Equal(t, `a/b/c/d/e`, f.absPath())
-}
-
-func TestTemplateFileDestinationPathForWindows(t *testing.T) {
-	if runtime.GOOS != "windows" {
-		t.SkipNow()
-	}
-	f := &destinationPath{
-		root:    `c:\a\b\c`,
-		relPath: "d/e",
-	}
-	assert.Equal(t, `c:\a\b\c\d\e`, f.absPath())
-}
-
 func TestTemplateInMemoryFilePersistToDisk(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.SkipNow()
 	}
-	testInMemoryFile(t, 0755)
+	ctx := context.Background()
+	testInMemoryFile(t, ctx, 0755)
 }
 
 func TestTemplateInMemoryFilePersistToDiskForWindows(t *testing.T) {
@@ -91,14 +66,16 @@ func TestTemplateInMemoryFilePersistToDiskForWindows(t *testing.T) {
 	}
 	// we have separate tests for windows because of differences in valid
 	// fs.FileMode values we can use for different operating systems.
-	testInMemoryFile(t, 0666)
+	ctx := context.Background()
+	testInMemoryFile(t, ctx, 0666)
 }
 
 func TestTemplateCopyFilePersistToDisk(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.SkipNow()
 	}
-	testCopyFile(t, 0644)
+	ctx := context.Background()
+	testCopyFile(t, ctx, 0644)
 }
 
 func TestTemplateCopyFilePersistToDiskForWindows(t *testing.T) {
@@ -107,5 +84,6 @@ func TestTemplateCopyFilePersistToDiskForWindows(t *testing.T) {
 	}
 	// we have separate tests for windows because of differences in valid
 	// fs.FileMode values we can use for different operating systems.
-	testCopyFile(t, 0666)
+	ctx := context.Background()
+	testCopyFile(t, ctx, 0666)
 }

libs/template/helpers_test.go

@@ -18,11 +18,10 @@ import (
 
 func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -35,11 +34,10 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
 
 func TestTemplateRegexpCompileFunction(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -53,11 +51,10 @@ func TestTemplateRegexpCompileFunction(t *testing.T) {
 
 func TestTemplateRandIntFunction(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -71,11 +68,10 @@ func TestTemplateRandIntFunction(t *testing.T) {
 
 func TestTemplateUuidFunction(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -88,11 +84,10 @@ func TestTemplateUuidFunction(t *testing.T) {
 
 func TestTemplateUrlFunction(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library")
 
 	require.NoError(t, err)
 
@@ -105,11 +100,10 @@ func TestTemplateUrlFunction(t *testing.T) {
 
 func TestTemplateMapPairFunction(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library")
 
 	require.NoError(t, err)
 
@@ -122,7 +116,6 @@ func TestTemplateMapPairFunction(t *testing.T) {
 
 func TestWorkspaceHost(t *testing.T) {
 	ctx := context.Background()
-	tmpDir := t.TempDir()
 
 	w := &databricks.WorkspaceClient{
 		Config: &workspaceConfig.Config{
@@ -132,7 +125,7 @@ func TestWorkspaceHost(t *testing.T) {
 	ctx = root.SetWorkspaceClient(ctx, w)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library")
 
 	require.NoError(t, err)
 
@@ -149,7 +142,6 @@ func TestWorkspaceHostNotConfigured(t *testing.T) {
 	ctx := context.Background()
 	cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "template")
 	ctx = cmdio.InContext(ctx, cmd)
-	tmpDir := t.TempDir()
 
 	w := &databricks.WorkspaceClient{
 		Config: &workspaceConfig.Config{},
@@ -157,7 +149,7 @@ func TestWorkspaceHostNotConfigured(t *testing.T) {
 	ctx = root.SetWorkspaceClient(ctx, w)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library")
 
 	assert.NoError(t, err)
 

libs/template/materialize.go

@@ -2,54 +2,32 @@ package template
 
 import (
 	"context"
-	"embed"
 	"errors"
 	"fmt"
 	"io/fs"
-	"os"
-	"path"
-	"path/filepath"
 
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/filer"
 )
 
 const libraryDirName = "library"
 const templateDirName = "template"
 const schemaFileName = "databricks_template_schema.json"
 
-//go:embed all:templates
-var builtinTemplates embed.FS
-
 // This function materializes the input templates as a project, using user defined
 // configurations.
 // Parameters:
 //
 //	ctx: 			context containing a cmdio object. This is used to prompt the user
 //	configFilePath: file path containing user defined config values
-//	templateRoot: 	root of the template definition
-//	outputDir: 	root of directory where to initialize the template
-func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir string) error {
-	// Use a temporary directory in case any builtin templates like default-python are used
-	tempDir, err := os.MkdirTemp("", "templates")
-	defer os.RemoveAll(tempDir)
-	if err != nil {
-		return err
-	}
-	templateRoot, err = prepareBuiltinTemplates(templateRoot, tempDir)
-	if err != nil {
-		return err
+//	templateFS: 	root of the template definition
+//	outputFiler: 	filer to use for writing the initialized template
+func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, outputFiler filer.Filer) error {
+	if _, err := fs.Stat(templateFS, schemaFileName); errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
 	}
 
-	templatePath := filepath.Join(templateRoot, templateDirName)
-	libraryPath := filepath.Join(templateRoot, libraryDirName)
-	schemaPath := filepath.Join(templateRoot, schemaFileName)
-	helpers := loadHelpers(ctx)
-
-	if _, err := os.Stat(schemaPath); errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaPath)
-	}
-
-	config, err := newConfig(ctx, schemaPath)
+	config, err := newConfig(ctx, templateFS, schemaFileName)
 	if err != nil {
 		return err
 	}
@@ -62,7 +40,8 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
 		}
 	}
 
-	r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
+	helpers := loadHelpers(ctx)
+	r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName)
 	if err != nil {
 		return err
 	}
@@ -94,7 +73,7 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
 		return err
 	}
 
-	err = r.persistToDisk()
+	err = r.persistToDisk(ctx, outputFiler)
 	if err != nil {
 		return err
 	}
@@ -111,44 +90,3 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
 	}
 	return nil
 }
-
-// If the given templateRoot matches
-func prepareBuiltinTemplates(templateRoot string, tempDir string) (string, error) {
-	// Check that `templateRoot` is a clean basename, i.e. `some_path` and not `./some_path` or "."
-	// Return early if that's not the case.
-	if templateRoot == "." || path.Base(templateRoot) != templateRoot {
-		return templateRoot, nil
-	}
-
-	_, err := fs.Stat(builtinTemplates, path.Join("templates", templateRoot))
-	if err != nil {
-		// The given path doesn't appear to be using out built-in templates
-		return templateRoot, nil
-	}
-
-	// We have a built-in template with the same name as templateRoot!
-	// Now we need to make a fully copy of the builtin templates to a real file system
-	// since template.Parse() doesn't support embed.FS.
-	err = fs.WalkDir(builtinTemplates, "templates", func(path string, entry fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		targetPath := filepath.Join(tempDir, path)
-		if entry.IsDir() {
-			return os.Mkdir(targetPath, 0755)
-		} else {
-			content, err := fs.ReadFile(builtinTemplates, path)
-			if err != nil {
-				return err
-			}
-			return os.WriteFile(targetPath, content, 0644)
-		}
-	})
-
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Join(tempDir, "templates", templateRoot), nil
-}

libs/template/materialize_test.go

@@ -3,7 +3,7 @@ package template
 import (
 	"context"
 	"fmt"
-	"path/filepath"
+	"os"
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
@@ -19,6 +19,6 @@ func TestMaterializeForNonTemplateDirectory(t *testing.T) {
 	ctx := root.SetWorkspaceClient(context.Background(), w)
 
 	// Try to materialize a non-template directory.
-	err = Materialize(ctx, "", tmpDir, "")
-	assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", filepath.Join(tmpDir, schemaFileName)))
+	err = Materialize(ctx, "", os.DirFS(tmpDir), nil)
+	assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", schemaFileName))
 }

libs/template/renderer.go

@@ -6,9 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"os"
 	"path"
-	"path/filepath"
 	"regexp"
 	"slices"
 	"sort"
@@ -52,32 +50,38 @@ type renderer struct {
 	// do not match any glob patterns from this list
 	skipPatterns []string
 
-	// Filer rooted at template root. The file tree from this root is walked to
-	// generate the project
-	templateFiler filer.Filer
-
-	// Root directory for the project instantiated from the template
-	instanceRoot string
+	// [fs.FS] that holds the template's file tree.
+	srcFS fs.FS
 }
 
-func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
+func newRenderer(
+	ctx context.Context,
+	config map[string]any,
+	helpers template.FuncMap,
+	templateFS fs.FS,
+	templateDir string,
+	libraryDir string,
+) (*renderer, error) {
 	// Initialize new template, with helper functions loaded
 	tmpl := template.New("").Funcs(helpers)
 
-	// Load user defined associated templates from the library root
-	libraryGlob := filepath.Join(libraryRoot, "*")
-	matches, err := filepath.Glob(libraryGlob)
+	// Find user-defined templates in the library directory
+	matches, err := fs.Glob(templateFS, path.Join(libraryDir, "*"))
 	if err != nil {
 		return nil, err
 	}
+
+	// Parse user-defined templates.
+	// Note: we do not call [ParseFS] with the glob directly because
+	// it returns an error if no files match the pattern.
 	if len(matches) != 0 {
-		tmpl, err = tmpl.ParseFiles(matches...)
+		tmpl, err = tmpl.ParseFS(templateFS, matches...)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	templateFiler, err := filer.NewLocalClient(templateRoot)
+	srcFS, err := fs.Sub(templateFS, path.Clean(templateDir))
 	if err != nil {
 		return nil, err
 	}
@@ -85,13 +89,12 @@ func newRenderer(ctx context.Context, config map[string]any, helpers template.Fu
 	ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))
 
 	return &renderer{
-		ctx:           ctx,
-		config:        config,
-		baseTemplate:  tmpl,
-		files:         make([]file, 0),
-		skipPatterns:  make([]string, 0),
-		templateFiler: templateFiler,
-		instanceRoot:  instanceRoot,
+		ctx:          ctx,
+		config:       config,
+		baseTemplate: tmpl,
+		files:        make([]file, 0),
+		skipPatterns: make([]string, 0),
+		srcFS:        srcFS,
 	}, nil
 }
 
@@ -141,7 +144,7 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
 
 func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	// read file permissions
-	info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
+	info, err := fs.Stat(r.srcFS, relPathTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -157,14 +160,10 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	// over as is, without treating it as a template
 	if !strings.HasSuffix(relPathTemplate, templateExtension) {
 		return &copyFile{
-			dstPath: &destinationPath{
-				root:    r.instanceRoot,
-				relPath: relPath,
-			},
-			perm:     perm,
-			ctx:      r.ctx,
-			srcPath:  relPathTemplate,
-			srcFiler: r.templateFiler,
+			perm:    perm,
+			relPath: relPath,
+			srcFS:   r.srcFS,
+			srcPath: relPathTemplate,
 		}, nil
 	} else {
 		// Trim the .tmpl suffix from file name, if specified in the template
@@ -173,7 +172,7 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	}
 
 	// read template file's content
-	templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
+	templateReader, err := r.srcFS.Open(relPathTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -194,11 +193,8 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	}
 
 	return &inMemoryFile{
-		dstPath: &destinationPath{
-			root:    r.instanceRoot,
-			relPath: relPath,
-		},
 		perm:    perm,
+		relPath: relPath,
 		content: []byte(content),
 	}, nil
 }
@@ -263,7 +259,7 @@ func (r *renderer) walk() error {
 		//
 		// 2. For directories: They are appended to a slice, which acts as a queue
 		//     allowing BFS traversal of the template file tree
-		entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
+		entries, err := fs.ReadDir(r.srcFS, currentDirectory)
 		if err != nil {
 			return err
 		}
@@ -283,7 +279,7 @@ func (r *renderer) walk() error {
 			if err != nil {
 				return err
 			}
-			logger.Infof(r.ctx, "added file to list of possible project files: %s", f.DstPath().relPath)
+			logger.Infof(r.ctx, "added file to list of possible project files: %s", f.RelPath())
 			r.files = append(r.files, f)
 		}
 
@@ -291,17 +287,17 @@ func (r *renderer) walk() error {
 	return nil
 }
 
-func (r *renderer) persistToDisk() error {
+func (r *renderer) persistToDisk(ctx context.Context, out filer.Filer) error {
 	// Accumulate files which we will persist, skipping files whose path matches
 	// any of the skip patterns
 	filesToPersist := make([]file, 0)
 	for _, file := range r.files {
-		match, err := isSkipped(file.DstPath().relPath, r.skipPatterns)
+		match, err := isSkipped(file.RelPath(), r.skipPatterns)
 		if err != nil {
 			return err
 		}
 		if match {
-			log.Infof(r.ctx, "skipping file: %s", file.DstPath())
+			log.Infof(r.ctx, "skipping file: %s", file.RelPath())
 			continue
 		}
 		filesToPersist = append(filesToPersist, file)
@@ -309,8 +305,8 @@ func (r *renderer) persistToDisk() error {
 
 	// Assert no conflicting files exist
 	for _, file := range filesToPersist {
-		path := file.DstPath().absPath()
-		_, err := os.Stat(path)
+		path := file.RelPath()
+		_, err := out.Stat(ctx, path)
 		if err == nil {
 			return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path)
 		}
@@ -321,7 +317,7 @@ func (r *renderer) persistToDisk() error {
 
 	// Persist files to disk
 	for _, file := range filesToPersist {
-		err := file.PersistToDisk()
+		err := file.Write(ctx, out)
 		if err != nil {
 			return err
 		}

libs/template/renderer_test.go

@@ -3,9 +3,9 @@ package template
 import (
 	"context"
 	"fmt"
-	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -18,6 +18,7 @@ import (
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/tags"
 	"github.com/databricks/databricks-sdk-go"
 	workspaceConfig "github.com/databricks/databricks-sdk-go/config"
@@ -41,9 +42,8 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
 func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
 	ctx := context.Background()
 
-	templatePath, err := prepareBuiltinTemplates(template, tempDir)
+	templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
 	require.NoError(t, err)
-	libraryPath := filepath.Join(templatePath, "library")
 
 	w := &databricks.WorkspaceClient{
 		Config: &workspaceConfig.Config{Host: "https://myhost.com"},
@@ -58,16 +58,18 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	ctx = root.SetWorkspaceClient(ctx, w)
 	helpers := loadHelpers(ctx)
 
-	renderer, err := newRenderer(ctx, settings, helpers, templatePath, libraryPath, tempDir)
+	renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName)
 	require.NoError(t, err)
 
 	// Evaluate template
 	err = renderer.walk()
 	require.NoError(t, err)
-	err = renderer.persistToDisk()
+	out, err := filer.NewLocalClient(tempDir)
+	require.NoError(t, err)
+	err = renderer.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
-	b, err := bundle.Load(ctx, filepath.Join(tempDir, "template", "my_project"))
+	b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project"))
 	require.NoError(t, err)
 	diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
 	require.NoError(t, diags.Error())
@@ -96,18 +98,6 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	}
 }
 
-func TestPrepareBuiltInTemplatesWithRelativePaths(t *testing.T) {
-	// CWD should not be resolved as a built in template
-	dir, err := prepareBuiltinTemplates(".", t.TempDir())
-	assert.NoError(t, err)
-	assert.Equal(t, ".", dir)
-
-	// relative path should not be resolved as a built in template
-	dir, err = prepareBuiltinTemplates("./default-python", t.TempDir())
-	assert.NoError(t, err)
-	assert.Equal(t, "./default-python", dir)
-}
-
 func TestBuiltinPythonTemplateValid(t *testing.T) {
 	// Test option combinations
 	options := []string{"yes", "no"}
@@ -194,13 +184,14 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/email/template", "./testdata/email/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library")
 	require.NoError(t, err)
 
 	err = r.walk()
 	require.NoError(t, err)
-
-	err = r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
 	b, err := os.ReadFile(filepath.Join(tmpDir, "my_email"))
@@ -325,45 +316,34 @@ func TestRendererPersistToDisk(t *testing.T) {
 
 	r := &renderer{
 		ctx:          ctx,
-		instanceRoot: tmpDir,
 		skipPatterns: []string{"a/b/c", "mn*"},
 		files: []file{
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "a/b/c",
-				},
 				perm:    0444,
+				relPath: "a/b/c",
 				content: nil,
 			},
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "mno",
-				},
 				perm:    0444,
+				relPath: "mno",
 				content: nil,
 			},
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "a/b/d",
-				},
 				perm:    0444,
+				relPath: "a/b/d",
 				content: []byte("123"),
 			},
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "mmnn",
-				},
 				perm:    0444,
+				relPath: "mmnn",
 				content: []byte("456"),
 			},
 		},
 	}
 
-	err := r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
 	assert.NoFileExists(t, filepath.Join(tmpDir, "a", "b", "c"))
@@ -378,10 +358,9 @@ func TestRendererPersistToDisk(t *testing.T) {
 func TestRendererWalk(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
-	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/walk/template", "./testdata/walk/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -389,21 +368,12 @@ func TestRendererWalk(t *testing.T) {
 
 	getContent := func(r *renderer, path string) string {
 		for _, f := range r.files {
-			if f.DstPath().relPath != path {
+			if f.RelPath() != path {
 				continue
 			}
-			switch v := f.(type) {
-			case *inMemoryFile:
-				return strings.Trim(string(v.content), "\r\n")
-			case *copyFile:
-				r, err := r.templateFiler.Read(context.Background(), v.srcPath)
-				require.NoError(t, err)
-				b, err := io.ReadAll(r)
-				require.NoError(t, err)
-				return strings.Trim(string(b), "\r\n")
-			default:
-				require.FailNow(t, "execution should not reach here")
-			}
+			b, err := f.contents()
+			require.NoError(t, err)
+			return strings.Trim(string(b), "\r\n")
 		}
 		require.FailNow(t, "file is absent: "+path)
 		return ""
@@ -419,10 +389,9 @@ func TestRendererWalk(t *testing.T) {
 func TestRendererFailFunction(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
-	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/fail/template", "./testdata/fail/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -432,10 +401,9 @@ func TestRendererFailFunction(t *testing.T) {
 func TestRendererSkipsDirsEagerly(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
-	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -452,7 +420,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -460,7 +428,9 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
 	// All 3 files are executed and have in memory representations
 	require.Len(t, r.files, 3)
 
-	err = r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
 	entries, err := os.ReadDir(tmpDir)
@@ -472,10 +442,9 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
 func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
-	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -493,7 +462,7 @@ func TestRendererSkip(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip/template", "./testdata/skip/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -502,7 +471,9 @@ func TestRendererSkip(t *testing.T) {
 	// This is because "dir2/*" matches the files in dir2, but not dir2 itself
 	assert.Len(t, r.files, 6)
 
-	err = r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
 	assert.FileExists(t, filepath.Join(tmpDir, "file1"))
@@ -520,12 +491,11 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
 	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
 		t.SkipNow()
 	}
-	tmpDir := t.TempDir()
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -533,7 +503,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
 
 	getPermissions := func(r *renderer, path string) fs.FileMode {
 		for _, f := range r.files {
-			if f.DstPath().relPath != path {
+			if f.RelPath() != path {
 				continue
 			}
 			switch v := f.(type) {
@@ -556,6 +526,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
 
 func TestRendererErrorOnConflictingFile(t *testing.T) {
 	tmpDir := t.TempDir()
+	ctx := context.Background()
 
 	f, err := os.Create(filepath.Join(tmpDir, "a"))
 	require.NoError(t, err)
@@ -566,17 +537,16 @@ func TestRendererErrorOnConflictingFile(t *testing.T) {
 		skipPatterns: []string{},
 		files: []file{
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "a",
-				},
 				perm:    0444,
+				relPath: "a",
 				content: []byte("123"),
 			},
 		},
 	}
-	err = r.persistToDisk()
-	assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", filepath.Join(tmpDir, "a")))
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
+	assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", "a"))
 }
 
 func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {
@@ -593,16 +563,15 @@ func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {
 		skipPatterns: []string{"a"},
 		files: []file{
 			&inMemoryFile{
-				dstPath: &destinationPath{
-					root:    tmpDir,
-					relPath: "a",
-				},
 				perm:    0444,
+				relPath: "a",
 				content: []byte("123"),
 			},
 		},
 	}
-	err = r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	// No error is returned even though a conflicting file exists. This is because
 	// the generated file is being skipped
 	assert.NoError(t, err)
@@ -612,10 +581,9 @@ func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {
 func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
-	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -623,7 +591,7 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
 
 	assert.Len(t, r.files, 1)
 	assert.Equal(t, r.files[0].(*copyFile).srcPath, "not-a-template")
-	assert.Equal(t, r.files[0].DstPath().absPath(), filepath.Join(tmpDir, "not-a-template"))
+	assert.Equal(t, r.files[0].RelPath(), "not-a-template")
 }
 
 func TestRendererFileTreeRendering(t *testing.T) {
@@ -635,7 +603,7 @@ func TestRendererFileTreeRendering(t *testing.T) {
 	r, err := newRenderer(ctx, map[string]any{
 		"dir_name":  "my_directory",
 		"file_name": "my_file",
-	}, helpers, "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
+	}, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -643,9 +611,11 @@ func TestRendererFileTreeRendering(t *testing.T) {
 
 	// Assert in memory representation is created.
 	assert.Len(t, r.files, 1)
-	assert.Equal(t, r.files[0].DstPath().absPath(), filepath.Join(tmpDir, "my_directory", "my_file"))
+	assert.Equal(t, r.files[0].RelPath(), "my_directory/my_file")
 
-	err = r.persistToDisk()
+	out, err := filer.NewLocalClient(tmpDir)
+	require.NoError(t, err)
+	err = r.persistToDisk(ctx, out)
 	require.NoError(t, err)
 
 	// Assert files and directories are correctly materialized.
@@ -667,8 +637,7 @@ func TestRendererSubTemplateInPath(t *testing.T) {
 	// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file.
 	testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}"))
 
-	tmpDir := t.TempDir()
-	r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir)
+	r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library")
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -676,7 +645,6 @@ func TestRendererSubTemplateInPath(t *testing.T) {
 
 	if assert.Len(t, r.files, 2) {
 		f := r.files[1]
-		assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), f.DstPath().absPath())
-		assert.Equal(t, "my_directory/my_file", f.DstPath().relPath)
+		assert.Equal(t, "my_directory/my_file", f.RelPath())
 	}
 }