Merge 6ea53065cf into 4fea0219fd

## Changes

While working on the v2 of #1744, I found that:
* Template initialization first copies built-in templates to a temporary
directory before initializing them
* Reading a template's contents goes through a `filer.Filer` but is
hardcoded to a local one

This change updates the interface for reading templates to be `fs.FS`.
This is compatible with the `embed.FS` type for the built-in templates,
so they no longer have to be copied to a temporary directory before
being used.

The alternative is to use a `filer.Filer` throughout, but this would
have required even more plumbing, and we don't need to _read_ templates,
including notebooks, from the workspace filesystem (yet?).

As part of making `template.Materialize` take an `fs.FS` argument, the
logic to match a given argument to a particular built-in template in the
`init` command has moved to sit next to its implementation.

## Tests

Existing tests pass.

bundle/config/bundle.go

@@ -18,6 +18,9 @@ type Bundle struct {
 	// Target is set by the mutator that selects the target.
 	Target string `json:"target,omitempty" bundle:"readonly"`
 
+	// TargetConfig stores a snapshot of the target configuration when it was selected by SelectTarget.
+	TargetConfig *Target `json:"target_config,omitempty" bundle:"internal"`
+
 	// DEPRECATED. Left for backward compatibility with Target
 	Environment string `json:"environment,omitempty" bundle:"readonly"`
 

bundle/config/mutator/configure_wsfs.go

@@ -5,14 +5,12 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/vfs"
 )
 
-const envDatabricksRuntimeVersion = "DATABRICKS_RUNTIME_VERSION"
-
 type configureWSFS struct{}
 
 func ConfigureWSFS() bundle.Mutator {
@@ -32,7 +30,7 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno
 	}
 
 	// The executable must be running on DBR.
-	if _, ok := env.Lookup(ctx, envDatabricksRuntimeVersion); !ok {
+	if !dbr.RunsOnRuntime(ctx) {
 		return nil
 	}
 

bundle/config/mutator/configure_wsfs_test.go

@@ -0,0 +1,65 @@
+package mutator_test
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/libs/dbr"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/config"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/stretchr/testify/assert"
+)
+
+func mockBundleForConfigureWSFS(t *testing.T, syncRootPath string) *bundle.Bundle {
+	// The native path of the sync root on Windows will never match the /Workspace prefix,
+	// so the test case for nominal behavior will always fail.
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not applicable on Windows")
+	}
+
+	b := &bundle.Bundle{
+		SyncRoot: vfs.MustNew(syncRootPath),
+	}
+
+	w := mocks.NewMockWorkspaceClient(t)
+	w.WorkspaceClient.Config = &config.Config{}
+	b.SetWorkpaceClient(w.WorkspaceClient)
+
+	return b
+}
+
+func TestConfigureWSFS_SkipsIfNotWorkspacePrefix(t *testing.T) {
+	b := mockBundleForConfigureWSFS(t, "/foo")
+	originalSyncRoot := b.SyncRoot
+
+	ctx := context.Background()
+	diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
+	assert.Empty(t, diags)
+	assert.Equal(t, originalSyncRoot, b.SyncRoot)
+}
+
+func TestConfigureWSFS_SkipsIfNotRunningOnRuntime(t *testing.T) {
+	b := mockBundleForConfigureWSFS(t, "/Workspace/foo")
+	originalSyncRoot := b.SyncRoot
+
+	ctx := context.Background()
+	ctx = dbr.MockRuntime(ctx, false)
+	diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
+	assert.Empty(t, diags)
+	assert.Equal(t, originalSyncRoot, b.SyncRoot)
+}
+
+func TestConfigureWSFS_SwapSyncRoot(t *testing.T) {
+	b := mockBundleForConfigureWSFS(t, "/Workspace/foo")
+	originalSyncRoot := b.SyncRoot
+
+	ctx := context.Background()
+	ctx = dbr.MockRuntime(ctx, true)
+	diags := bundle.Apply(ctx, b, mutator.ConfigureWSFS())
+	assert.Empty(t, diags)
+	assert.NotEqual(t, originalSyncRoot, b.SyncRoot)
+}

bundle/config/mutator/initialize_urls_test.go

@@ -65,9 +65,8 @@ func TestInitializeURLs(t *testing.T) {
 				},
 				QualityMonitors: map[string]*resources.QualityMonitor{
 					"qualityMonitor1": {
-						CreateMonitor: &catalog.CreateMonitor{
-							TableName: "catalog.schema.qualityMonitor1",
-						},
+						TableName:     "catalog.schema.qualityMonitor1",
+						CreateMonitor: &catalog.CreateMonitor{},
 					},
 				},
 				Schemas: map[string]*resources.Schema{

bundle/config/mutator/prepend_workspace_prefix.go

@@ -44,6 +44,11 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
 					return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind())
 				}
 
+				// Skip prefixing if the path does not start with /, it might be variable reference or smth else.
+				if !strings.HasPrefix(path, "/") {
+					return pv, nil
+				}
+
 				for _, prefix := range skipPrefixes {
 					if strings.HasPrefix(path, prefix) {
 						return pv, nil

bundle/config/mutator/prepend_workspace_prefix_test.go

@@ -31,6 +31,14 @@ func TestPrependWorkspacePrefix(t *testing.T) {
 			path:     "/Volumes/Users/test",
 			expected: "/Volumes/Users/test",
 		},
+		{
+			path:     "~/test",
+			expected: "~/test",
+		},
+		{
+			path:     "${workspace.file_path}/test",
+			expected: "${workspace.file_path}/test",
+		},
 	}
 
 	for _, tc := range testCases {

bundle/config/mutator/process_target_mode.go

@@ -2,6 +2,7 @@ package mutator
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/databricks/cli/bundle"
@@ -146,8 +147,21 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
 		}
 	}
 
-	if !isPrincipalUsed && !isRunAsSet(r) {
-		return diag.Errorf("'run_as' must be set for all jobs when using 'mode: production'")
+	// We need to verify that there is only a single deployment of the current target.
+	// The best way to enforce this is to explicitly set root_path.
+	advice := fmt.Sprintf(
+		"set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Users/%s/.bundle/${bundle.name}/${bundle.target}",
+		b.Config.Workspace.CurrentUser.UserName,
+	)
+	if !isExplicitRootSet(b) {
+		if isRunAsSet(r) || isPrincipalUsed {
+			// Just setting run_as is not enough to guarantee a single deployment,
+			// and neither is setting a principal.
+			// We only show a warning for these cases since we didn't historically
+			// report an error for them.
+			return diag.Warningf("target with 'mode: production' should " + advice)
+		}
+		return diag.Errorf("target with 'mode: production' must " + advice)
 	}
 	return nil
 }
@@ -164,6 +178,17 @@ func isRunAsSet(r config.Resources) bool {
 	return true
 }
 
+func isExplicitRootSet(b *bundle.Bundle) bool {
+	if b.Config.Bundle.TargetConfig == nil {
+		return false
+	}
+	targetConfig := b.Config.Bundle.TargetConfig
+	if targetConfig.Workspace == nil {
+		return false
+	}
+	return targetConfig.Workspace.RootPath != ""
+}
+
 func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	switch b.Config.Bundle.Mode {
 	case config.Development:

bundle/config/mutator/process_target_mode_test.go

@@ -102,16 +102,23 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 					"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
 				},
 				QualityMonitors: map[string]*resources.QualityMonitor{
-					"qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}},
-					"qualityMonitor2": {
+					"qualityMonitor1": {
+						TableName: "qualityMonitor1",
 						CreateMonitor: &catalog.CreateMonitor{
-							TableName: "qualityMonitor2",
-							Schedule:  &catalog.MonitorCronSchedule{},
+							OutputSchemaName: "catalog.schema",
+						},
+					},
+					"qualityMonitor2": {
+						TableName: "qualityMonitor2",
+						CreateMonitor: &catalog.CreateMonitor{
+							OutputSchemaName: "catalog.schema",
+							Schedule:         &catalog.MonitorCronSchedule{},
 						},
 					},
 					"qualityMonitor3": {
+						TableName: "qualityMonitor3",
 						CreateMonitor: &catalog.CreateMonitor{
-							TableName: "qualityMonitor3",
+							OutputSchemaName: "catalog.schema",
 							Schedule: &catalog.MonitorCronSchedule{
 								PauseStatus: catalog.MonitorCronSchedulePauseStatusUnpaused,
 							},
@@ -307,7 +314,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	b := mockBundle(config.Production)
 
 	diags := validateProductionMode(context.Background(), b, false)
-	require.ErrorContains(t, diags.Error(), "run_as")
+	require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}")
 
 	b.Config.Workspace.StatePath = "/Shared/.bundle/x/y/state"
 	b.Config.Workspace.ArtifactPath = "/Shared/.bundle/x/y/artifacts"
@@ -315,7 +322,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	b.Config.Workspace.ResourcePath = "/Shared/.bundle/x/y/resources"
 
 	diags = validateProductionMode(context.Background(), b, false)
-	require.ErrorContains(t, diags.Error(), "production")
+	require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Users/lennart@company.com/.bundle/${bundle.name}/${bundle.target}")
 
 	permissions := []resources.Permission{
 		{
@@ -359,6 +366,23 @@ func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
 	require.NoError(t, diags.Error())
 }
 
+func TestProcessTargetModeProductionOkWithRootPath(t *testing.T) {
+	b := mockBundle(config.Production)
+
+	// Our target has all kinds of problems when not using service principals ...
+	diags := validateProductionMode(context.Background(), b, false)
+	require.Error(t, diags.Error())
+
+	// ... but we're okay if we specify a root path
+	b.Config.Bundle.TargetConfig = &config.Target{
+		Workspace: &config.Workspace{
+			RootPath: "some-root-path",
+		},
+	}
+	diags = validateProductionMode(context.Background(), b, false)
+	require.NoError(t, diags.Error())
+}
+
 // Make sure that we have test coverage for all resource types
 func TestAllResourcesMocked(t *testing.T) {
 	b := mockBundle(config.Development)

bundle/config/mutator/select_target.go

@@ -15,6 +15,7 @@ type selectTarget struct {
 }
 
 // SelectTarget merges the specified target into the root configuration.
+// After merging, it removes the 'Targets' section from the configuration.
 func SelectTarget(name string) bundle.Mutator {
 	return &selectTarget{
 		name: name,
@@ -31,7 +32,7 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
 	}
 
 	// Get specified target
-	_, ok := b.Config.Targets[m.name]
+	targetConfig, ok := b.Config.Targets[m.name]
 	if !ok {
 		return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(maps.Keys(b.Config.Targets), ", "))
 	}
@@ -43,13 +44,15 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
 	}
 
 	// Store specified target in configuration for reference.
+	b.Config.Bundle.TargetConfig = targetConfig
 	b.Config.Bundle.Target = m.name
 
 	// We do this for backward compatibility.
 	// TODO: remove when Environments section is not supported anymore.
 	b.Config.Bundle.Environment = b.Config.Bundle.Target
 
-	// Clear targets after loading.
+	// Cleanup the original targets and environments sections since they
+	// show up in the JSON output of the 'summary' and 'validate' commands.
 	b.Config.Targets = nil
 	b.Config.Environments = nil
 

bundle/config/resources/quality_monitor.go

@@ -13,17 +13,15 @@ import (
 )
 
 type QualityMonitor struct {
-	// Represents the Input Arguments for Terraform and will get
-	// converted to a HCL representation for CRUD
-	*catalog.CreateMonitor
-
-	// This represents the id which is the full name of the monitor
-	// (catalog_name.schema_name.table_name) that can be used
-	// as a reference in other resources. This value is returned by terraform.
-	ID string `json:"id,omitempty" bundle:"readonly"`
-
+	ID             string         `json:"id,omitempty" bundle:"readonly"`
 	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
 	URL            string         `json:"url,omitempty" bundle:"internal"`
+
+	// The table name is a required field but not included as a JSON field in [catalog.CreateMonitor].
+	TableName string `json:"table_name"`
+
+	// This struct defines the creation payload for a monitor.
+	*catalog.CreateMonitor
 }
 
 func (s *QualityMonitor) UnmarshalJSON(b []byte) error {

bundle/config/root.go

@@ -47,8 +47,8 @@ type Root struct {
 
 	// Targets can be used to differentiate settings and resources between
 	// bundle deployment targets (e.g. development, staging, production).
-	// If not specified, the code below initializes this field with a
-	// single default-initialized target called "default".
+	// Note that this field is set to 'nil' by the SelectTarget mutator;
+	// use Bundle.TargetConfig to access the selected target configuration.
 	Targets map[string]*Target `json:"targets,omitempty"`
 
 	// DEPRECATED. Left for backward compatibility with Targets

bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go

@@ -15,8 +15,8 @@ import (
 
 func TestConvertQualityMonitor(t *testing.T) {
 	var src = resources.QualityMonitor{
+		TableName: "test_table_name",
 		CreateMonitor: &catalog.CreateMonitor{
-			TableName:        "test_table_name",
 			AssetsDir:        "assets_dir",
 			OutputSchemaName: "output_schema_name",
 			InferenceLog: &catalog.MonitorInferenceLog{

bundle/internal/schema/testdata/pass/quality_monitor.yml

@@ -4,6 +4,7 @@ bundle:
 resources:
   quality_monitors:
     myqualitymonitor:
+      table_name: catalog.schema.quality_monitor
       inference_log:
         granularities:
           - a

bundle/schema/jsonschema.json

@@ -684,6 +684,9 @@
                         "description": "Configuration for monitoring snapshot tables.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot"
                       },
+                      "table_name": {
+                        "$ref": "#/$defs/string"
+                      },
                       "time_series": {
                         "description": "Configuration for monitoring time series tables.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries"
@@ -695,6 +698,7 @@
                     },
                     "additionalProperties": false,
                     "required": [
+                      "table_name",
                       "assets_dir",
                       "output_schema_name"
                     ]

cmd/bundle/init.go

@@ -3,6 +3,7 @@ package bundle
 import (
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"slices"
@@ -109,6 +110,24 @@ func getUrlForNativeTemplate(name string) string {
 	return ""
 }
 
+func getFsForNativeTemplate(name string) (fs.FS, error) {
+	builtin, err := template.Builtin()
+	if err != nil {
+		return nil, err
+	}
+
+	// If this is a built-in template, the return value will be non-nil.
+	var templateFS fs.FS
+	for _, entry := range builtin {
+		if entry.Name == name {
+			templateFS = entry.FS
+			break
+		}
+	}
+
+	return templateFS, nil
+}
+
 func isRepoUrl(url string) bool {
 	result := false
 	for _, prefix := range gitUrlPrefixes {
@@ -198,9 +217,20 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 			if templateDir != "" {
 				return errors.New("--template-dir can only be used with a Git repository URL")
 			}
+
+			templateFS, err := getFsForNativeTemplate(templatePath)
+			if err != nil {
+				return err
+			}
+
+			// If this is not a built-in template, then it must be a local file system path.
+			if templateFS == nil {
+				templateFS = os.DirFS(templatePath)
+			}
+
 			// skip downloading the repo because input arg is not a URL. We assume
 			// it's a path on the local file system in that case
-			return template.Materialize(ctx, configFile, templatePath, outputDir)
+			return template.Materialize(ctx, configFile, templateFS, outputDir)
 		}
 
 		// Create a temporary directory with the name of the repository.  The '*'
@@ -224,7 +254,8 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf
 
 		// Clean up downloaded repository once the template is materialized.
 		defer os.RemoveAll(repoDir)
-		return template.Materialize(ctx, configFile, filepath.Join(repoDir, templateDir), outputDir)
+		templateFS := os.DirFS(filepath.Join(repoDir, templateDir))
+		return template.Materialize(ctx, configFile, templateFS, outputDir)
 	}
 	return cmd
 }

cmd/fs/helpers_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/fakefs"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/spf13/cobra"
@@ -84,7 +85,7 @@ func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceCl
 	cmd, m := setupCommand(t)
 
 	fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) {
-		fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
+		fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
 			"dir":       {FakeName: "root", FakeDir: true},
 			"dir/dirA":  {FakeDir: true},
 			"dir/dirB":  {FakeDir: true},

cmd/root/root.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/databricks/cli/internal/build"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/log"
 	"github.com/spf13/cobra"
 )
@@ -73,6 +74,9 @@ func New(ctx context.Context) *cobra.Command {
 		// get the context back
 		ctx = cmd.Context()
 
+		// Detect if the CLI is running on DBR and store this on the context.
+		ctx = dbr.DetectRuntime(ctx)
+
 		// Configure our user agent with the command that's about to be executed.
 		ctx = withCommandInUserAgent(ctx, cmd)
 		ctx = withCommandExecIdInUserAgent(ctx)

internal/bundle/helpers.go

@@ -42,7 +42,7 @@ func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateN
 	cmd := cmdio.NewIO(flags.OutputJSON, strings.NewReader(""), os.Stdout, os.Stderr, "", "bundles")
 	ctx = cmdio.InContext(ctx, cmd)
 
-	err = template.Materialize(ctx, configFilePath, templateRoot, bundleRoot)
+	err = template.Materialize(ctx, configFilePath, os.DirFS(templateRoot), bundleRoot)
 	return bundleRoot, err
 }
 

internal/filer_test.go

@@ -723,6 +723,63 @@ func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) {
 	assert.ErrorIs(t, err, fs.ErrNotExist)
 }
 
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotReadAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Reading foo should fail. Even though the WSFS name for the notebook is foo
+	// reading the notebook should only work with the .ipynb extension.
+	_, err = wf.Read(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = wf.Read(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotStatAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Stating foo should fail. Even though the WSFS name for the notebook is foo
+	// stating the notebook should only work with the .ipynb extension.
+	_, err = wf.Stat(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = wf.Stat(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
+func TestAccWorkspaceFilesExtensionsNotebooksAreNotDeletedAsFiles(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a notebook
+	err := wf.Write(ctx, "foo.ipynb", strings.NewReader(readFile(t, "testdata/notebooks/py1.ipynb")))
+	require.NoError(t, err)
+
+	// Deleting foo should fail. Even though the WSFS name for the notebook is foo
+	// deleting the notebook should only work with the .ipynb extension.
+	err = wf.Delete(ctx, "foo")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	err = wf.Delete(ctx, "foo.ipynb")
+	assert.NoError(t, err)
+}
+
 func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) {
 	t.Parallel()
 

libs/dbr/context.go

@@ -0,0 +1,49 @@
+package dbr
+
+import "context"
+
+// key is a package-local type to use for context keys.
+//
+// Using an unexported type for context keys prevents key collisions across
+// packages since external packages cannot create values of this type.
+type key int
+
+const (
+	// dbrKey is the context key for the detection result.
+	// The value of 1 is arbitrary and can be any number.
+	// Other keys in the same package must have different values.
+	dbrKey = key(1)
+)
+
+// DetectRuntime detects whether or not the current
+// process is running inside a Databricks Runtime environment.
+// It return a new context with the detection result set.
+func DetectRuntime(ctx context.Context) context.Context {
+	if v := ctx.Value(dbrKey); v != nil {
+		panic("dbr.DetectRuntime called twice on the same context")
+	}
+	return context.WithValue(ctx, dbrKey, detect(ctx))
+}
+
+// MockRuntime is a helper function to mock the detection result.
+// It returns a new context with the detection result set.
+func MockRuntime(ctx context.Context, b bool) context.Context {
+	if v := ctx.Value(dbrKey); v != nil {
+		panic("dbr.MockRuntime called twice on the same context")
+	}
+	return context.WithValue(ctx, dbrKey, b)
+}
+
+// RunsOnRuntime returns the detection result from the context.
+// It expects a context returned by [DetectRuntime] or [MockRuntime].
+//
+// We store this value in a context to avoid having to use either
+// a global variable, passing a boolean around everywhere, or
+// performing the same detection multiple times.
+func RunsOnRuntime(ctx context.Context) bool {
+	v := ctx.Value(dbrKey)
+	if v == nil {
+		panic("dbr.RunsOnRuntime called without calling dbr.DetectRuntime first")
+	}
+	return v.(bool)
+}

libs/dbr/context_test.go

@@ -0,0 +1,59 @@
+package dbr
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestContext_DetectRuntimePanics(t *testing.T) {
+	ctx := context.Background()
+
+	// Run detection.
+	ctx = DetectRuntime(ctx)
+
+	// Expect a panic if the detection is run twice.
+	assert.Panics(t, func() {
+		ctx = DetectRuntime(ctx)
+	})
+}
+
+func TestContext_MockRuntimePanics(t *testing.T) {
+	ctx := context.Background()
+
+	// Run detection.
+	ctx = MockRuntime(ctx, true)
+
+	// Expect a panic if the mock function is run twice.
+	assert.Panics(t, func() {
+		MockRuntime(ctx, true)
+	})
+}
+
+func TestContext_RunsOnRuntimePanics(t *testing.T) {
+	ctx := context.Background()
+
+	// Expect a panic if the detection is not run.
+	assert.Panics(t, func() {
+		RunsOnRuntime(ctx)
+	})
+}
+
+func TestContext_RunsOnRuntime(t *testing.T) {
+	ctx := context.Background()
+
+	// Run detection.
+	ctx = DetectRuntime(ctx)
+
+	// Expect no panic because detection has run.
+	assert.NotPanics(t, func() {
+		RunsOnRuntime(ctx)
+	})
+}
+
+func TestContext_RunsOnRuntimeWithMock(t *testing.T) {
+	ctx := context.Background()
+	assert.True(t, RunsOnRuntime(MockRuntime(ctx, true)))
+	assert.False(t, RunsOnRuntime(MockRuntime(ctx, false)))
+}

libs/dbr/detect.go

@@ -0,0 +1,35 @@
+package dbr
+
+import (
+	"context"
+	"os"
+	"runtime"
+
+	"github.com/databricks/cli/libs/env"
+)
+
+// Dereference [os.Stat] to allow mocking in tests.
+var statFunc = os.Stat
+
+// detect returns true if the current process is running on a Databricks Runtime.
+// Its return value is meant to be cached in the context.
+func detect(ctx context.Context) bool {
+	// Databricks Runtime implies Linux.
+	// Return early on other operating systems.
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	// Databricks Runtime always has the DATABRICKS_RUNTIME_VERSION environment variable set.
+	if value, ok := env.Lookup(ctx, "DATABRICKS_RUNTIME_VERSION"); !ok || value == "" {
+		return false
+	}
+
+	// Expect to see a "/databricks" directory.
+	if fi, err := statFunc("/databricks"); err != nil || !fi.IsDir() {
+		return false
+	}
+
+	// All checks passed.
+	return true
+}

libs/dbr/detect_test.go

@@ -0,0 +1,83 @@
+package dbr
+
+import (
+	"context"
+	"io/fs"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/cli/libs/fakefs"
+	"github.com/stretchr/testify/assert"
+)
+
+func requireLinux(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skipf("skipping test on %s", runtime.GOOS)
+	}
+}
+
+func configureStatFunc(t *testing.T, fi fs.FileInfo, err error) {
+	originalFunc := statFunc
+	statFunc = func(name string) (fs.FileInfo, error) {
+		assert.Equal(t, "/databricks", name)
+		return fi, err
+	}
+
+	t.Cleanup(func() {
+		statFunc = originalFunc
+	})
+}
+
+func TestDetect_NotLinux(t *testing.T) {
+	if runtime.GOOS == "linux" {
+		t.Skip("skipping test on Linux OS")
+	}
+
+	ctx := context.Background()
+	assert.False(t, detect(ctx))
+}
+
+func TestDetect_Env(t *testing.T) {
+	requireLinux(t)
+
+	// Configure other checks to pass.
+	configureStatFunc(t, fakefs.FileInfo{FakeDir: true}, nil)
+
+	t.Run("empty", func(t *testing.T) {
+		ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "")
+		assert.False(t, detect(ctx))
+	})
+
+	t.Run("non-empty cluster", func(t *testing.T) {
+		ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "15.4")
+		assert.True(t, detect(ctx))
+	})
+
+	t.Run("non-empty serverless", func(t *testing.T) {
+		ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "client.1.13")
+		assert.True(t, detect(ctx))
+	})
+}
+
+func TestDetect_Stat(t *testing.T) {
+	requireLinux(t)
+
+	// Configure other checks to pass.
+	ctx := env.Set(context.Background(), "DATABRICKS_RUNTIME_VERSION", "non-empty")
+
+	t.Run("error", func(t *testing.T) {
+		configureStatFunc(t, nil, fs.ErrNotExist)
+		assert.False(t, detect(ctx))
+	})
+
+	t.Run("not a directory", func(t *testing.T) {
+		configureStatFunc(t, fakefs.FileInfo{}, nil)
+		assert.False(t, detect(ctx))
+	})
+
+	t.Run("directory", func(t *testing.T) {
+		configureStatFunc(t, fakefs.FileInfo{FakeDir: true}, nil)
+		assert.True(t, detect(ctx))
+	})
+}

libs/dyn/convert/struct_info.go

@@ -6,7 +6,6 @@ import (
 	"sync"
 
 	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/textutil"
 )
 
 // structInfo holds the type information we need to efficiently
@@ -85,14 +84,6 @@ func buildStructInfo(typ reflect.Type) structInfo {
 			}
 
 			name, _, _ := strings.Cut(sf.Tag.Get("json"), ",")
-			if typ.Name() == "QualityMonitor" && name == "-" {
-				urlName, _, _ := strings.Cut(sf.Tag.Get("url"), ",")
-				if urlName == "" || urlName == "-" {
-					name = textutil.CamelToSnakeCase(sf.Name)
-				} else {
-					name = urlName
-				}
-			}
 			if name == "" || name == "-" {
 				continue
 			}

libs/fakefs/fakefs.go

@@ -0,0 +1,87 @@
+package fakefs
+
+import (
+	"fmt"
+	"io/fs"
+	"time"
+)
+
+var ErrNotImplemented = fmt.Errorf("not implemented")
+
+// DirEntry is a fake implementation of [fs.DirEntry].
+type DirEntry struct {
+	fs.FileInfo
+}
+
+func (entry DirEntry) Type() fs.FileMode {
+	typ := fs.ModePerm
+	if entry.IsDir() {
+		typ |= fs.ModeDir
+	}
+	return typ
+}
+
+func (entry DirEntry) Info() (fs.FileInfo, error) {
+	return entry.FileInfo, nil
+}
+
+// FileInfo is a fake implementation of [fs.FileInfo].
+type FileInfo struct {
+	FakeName string
+	FakeSize int64
+	FakeDir  bool
+	FakeMode fs.FileMode
+}
+
+func (info FileInfo) Name() string {
+	return info.FakeName
+}
+
+func (info FileInfo) Size() int64 {
+	return info.FakeSize
+}
+
+func (info FileInfo) Mode() fs.FileMode {
+	return info.FakeMode
+}
+
+func (info FileInfo) ModTime() time.Time {
+	return time.Now()
+}
+
+func (info FileInfo) IsDir() bool {
+	return info.FakeDir
+}
+
+func (info FileInfo) Sys() any {
+	return nil
+}
+
+// File is a fake implementation of [fs.File].
+type File struct {
+	FileInfo fs.FileInfo
+}
+
+func (f File) Close() error {
+	return nil
+}
+
+func (f File) Read(p []byte) (n int, err error) {
+	return 0, ErrNotImplemented
+}
+
+func (f File) Stat() (fs.FileInfo, error) {
+	return f.FileInfo, nil
+}
+
+// FS is a fake implementation of [fs.FS].
+type FS map[string]fs.File
+
+func (f FS) Open(name string) (fs.File, error) {
+	e, ok := f[name]
+	if !ok {
+		return nil, fs.ErrNotExist
+	}
+
+	return e, nil
+}

libs/fakefs/fakefs_test.go

@@ -0,0 +1,38 @@
+package fakefs
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFile(t *testing.T) {
+	var fakefile fs.File = File{
+		FileInfo: FileInfo{
+			FakeName: "file",
+		},
+	}
+
+	_, err := fakefile.Read([]byte{})
+	assert.ErrorIs(t, err, ErrNotImplemented)
+
+	fi, err := fakefile.Stat()
+	assert.NoError(t, err)
+	assert.Equal(t, "file", fi.Name())
+
+	err = fakefile.Close()
+	assert.NoError(t, err)
+}
+
+func TestFS(t *testing.T) {
+	var fakefs fs.FS = FS{
+		"file": File{},
+	}
+
+	_, err := fakefs.Open("doesntexist")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = fakefs.Open("file")
+	assert.NoError(t, err)
+}

libs/filer/completer/completer_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/fakefs"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/spf13/cobra"
@@ -17,7 +18,7 @@ func setupCompleter(t *testing.T, onlyDirs bool) *completer {
 	// Needed to make type context.valueCtx for mockFilerForPath
 	ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient)
 
-	fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{
+	fakeFiler := filer.NewFakeFiler(map[string]fakefs.FileInfo{
 		"dir":       {FakeName: "root", FakeDir: true},
 		"dir/dirA":  {FakeDir: true},
 		"dir/dirB":  {FakeDir: true},

libs/filer/fake_filer.go

@@ -8,58 +8,12 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"time"
+
+	"github.com/databricks/cli/libs/fakefs"
 )
 
-type FakeDirEntry struct {
-	FakeFileInfo
-}
-
-func (entry FakeDirEntry) Type() fs.FileMode {
-	typ := fs.ModePerm
-	if entry.FakeDir {
-		typ |= fs.ModeDir
-	}
-	return typ
-}
-
-func (entry FakeDirEntry) Info() (fs.FileInfo, error) {
-	return entry.FakeFileInfo, nil
-}
-
-type FakeFileInfo struct {
-	FakeName string
-	FakeSize int64
-	FakeDir  bool
-	FakeMode fs.FileMode
-}
-
-func (info FakeFileInfo) Name() string {
-	return info.FakeName
-}
-
-func (info FakeFileInfo) Size() int64 {
-	return info.FakeSize
-}
-
-func (info FakeFileInfo) Mode() fs.FileMode {
-	return info.FakeMode
-}
-
-func (info FakeFileInfo) ModTime() time.Time {
-	return time.Now()
-}
-
-func (info FakeFileInfo) IsDir() bool {
-	return info.FakeDir
-}
-
-func (info FakeFileInfo) Sys() any {
-	return nil
-}
-
 type FakeFiler struct {
-	entries map[string]FakeFileInfo
+	entries map[string]fakefs.FileInfo
 }
 
 func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error {
@@ -97,7 +51,7 @@ func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error
 			continue
 		}
 
-		out = append(out, FakeDirEntry{v})
+		out = append(out, fakefs.DirEntry{FileInfo: v})
 	}
 
 	sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() })
@@ -117,7 +71,11 @@ func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error)
 	return entry, nil
 }
 
-func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler {
+// NewFakeFiler creates a new fake [Filer] instance with the given entries.
+// It sets the [Name] field of each entry to the base name of the path.
+//
+// This is meant to be used in tests.
+func NewFakeFiler(entries map[string]fakefs.FileInfo) *FakeFiler {
 	fakeFiler := &FakeFiler{
 		entries: entries,
 	}

libs/filer/fake_filer_test.go

@@ -0,0 +1,98 @@
+package filer
+
+import (
+	"context"
+	"io"
+	"io/fs"
+	"testing"
+
+	"github.com/databricks/cli/libs/fakefs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFakeFiler_Read(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"file": {},
+	})
+
+	ctx := context.Background()
+	r, err := f.Read(ctx, "file")
+	require.NoError(t, err)
+	contents, err := io.ReadAll(r)
+	require.NoError(t, err)
+
+	// Contents of every file is "foo".
+	assert.Equal(t, "foo", string(contents))
+}
+
+func TestFakeFiler_Read_NotFound(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"foo": {},
+	})
+
+	ctx := context.Background()
+	_, err := f.Read(ctx, "bar")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestFakeFiler_ReadDir_NotFound(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"dir1": {FakeDir: true},
+	})
+
+	ctx := context.Background()
+	_, err := f.ReadDir(ctx, "dir2")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestFakeFiler_ReadDir_NotADirectory(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"file": {},
+	})
+
+	ctx := context.Background()
+	_, err := f.ReadDir(ctx, "file")
+	assert.ErrorIs(t, err, fs.ErrInvalid)
+}
+
+func TestFakeFiler_ReadDir(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"dir1":       {FakeDir: true},
+		"dir1/file2": {},
+		"dir1/dir2":  {FakeDir: true},
+	})
+
+	ctx := context.Background()
+	entries, err := f.ReadDir(ctx, "dir1/")
+	require.NoError(t, err)
+	require.Len(t, entries, 2)
+
+	// The entries are sorted by name.
+	assert.Equal(t, "dir2", entries[0].Name())
+	assert.True(t, entries[0].IsDir())
+	assert.Equal(t, "file2", entries[1].Name())
+	assert.False(t, entries[1].IsDir())
+}
+
+func TestFakeFiler_Stat(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"file": {},
+	})
+
+	ctx := context.Background()
+	info, err := f.Stat(ctx, "file")
+	require.NoError(t, err)
+
+	assert.Equal(t, "file", info.Name())
+}
+
+func TestFakeFiler_Stat_NotFound(t *testing.T) {
+	f := NewFakeFiler(map[string]fakefs.FileInfo{
+		"foo": {},
+	})
+
+	ctx := context.Background()
+	_, err := f.Stat(ctx, "bar")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}

libs/filer/fs_test.go

@@ -6,6 +6,7 @@ import (
 	"io/fs"
 	"testing"
 
+	"github.com/databricks/cli/libs/fakefs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,7 +36,7 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) {
 }
 
 func fakeFS() fs.FS {
-	fakeFiler := NewFakeFiler(map[string]FakeFileInfo{
+	fakeFiler := NewFakeFiler(map[string]fakefs.FileInfo{
 		".":     {FakeName: "root", FakeDir: true},
 		"dirA":  {FakeDir: true},
 		"dirB":  {FakeDir: true},

libs/filer/workspace_files_extensions_client.go

@@ -244,6 +244,17 @@ func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string,
 
 // Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
 func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
+	// Ensure that the file / notebook exists. We do this check here to avoid reading
+	// the content of a notebook called `foo` when the user actually wanted
+	// to read the content of a file called `foo`.
+	//
+	// To read the content of a notebook called `foo` in the workspace the user
+	// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
+	_, err := w.Stat(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
 	r, err := w.wsfs.Read(ctx, name)
 
 	// If the file is not found, it might be a notebook.
@@ -276,7 +287,18 @@ func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string
 		return ReadOnlyError{"delete"}
 	}
 
-	err := w.wsfs.Delete(ctx, name, mode...)
+	// Ensure that the file / notebook exists. We do this check here to avoid
+	// deleting the a notebook called `foo` when the user actually wanted to
+	// delete a file called `foo`.
+	//
+	// To delete a notebook called `foo` in the workspace the user should use the
+	// name with the extension included like `foo.ipynb` or `foo.sql`.
+	_, err := w.Stat(ctx, name)
+	if err != nil {
+		return err
+	}
+
+	err = w.wsfs.Delete(ctx, name, mode...)
 
 	// If the file is not found, it might be a notebook.
 	if errors.As(err, &FileDoesNotExistError{}) {
@@ -315,7 +337,24 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string)
 		return wsfsFileInfo{ObjectInfo: stat.ObjectInfo}, nil
 	}
 
-	return info, err
+	if err != nil {
+		return nil, err
+	}
+
+	// If an object is found and it is a notebook, return a FileDoesNotExistError.
+	// If a notebook is found by the workspace files client, without having stripped
+	// the extension, this implies that no file with the same name exists.
+	//
+	// This check is done to avoid returning the stat for a notebook called `foo`
+	// when the user actually wanted to stat a file called `foo`.
+	//
+	// To stat the metadata of a notebook called `foo` in the workspace the user
+	// should use the name with the extension included like `foo.ipynb` or `foo.sql`.
+	if info.Sys().(workspace.ObjectInfo).ObjectType == workspace.ObjectTypeNotebook {
+		return nil, FileDoesNotExistError{name}
+	}
+
+	return info, nil
 }
 
 // Note: The import API returns opaque internal errors for namespace clashes

libs/jsonschema/schema.go

@@ -3,7 +3,9 @@ package jsonschema
 import (
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"regexp"
 	"slices"
 
@@ -255,7 +257,12 @@ func (schema *Schema) validate() error {
 }
 
 func Load(path string) (*Schema, error) {
-	b, err := os.ReadFile(path)
+	dir, file := filepath.Split(path)
+	return LoadFS(os.DirFS(dir), file)
+}
+
+func LoadFS(fsys fs.FS, path string) (*Schema, error) {
+	b, err := fs.ReadFile(fsys, path)
 	if err != nil {
 		return nil, err
 	}

libs/jsonschema/schema_test.go

@@ -1,6 +1,7 @@
 package jsonschema
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -305,3 +306,9 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) {
 	err = s.validate()
 	assert.NoError(t, err)
 }
+
+func TestSchema_LoadFS(t *testing.T) {
+	fsys := os.DirFS("./testdata/schema-load-int")
+	_, err := LoadFS(fsys, "schema-valid.json")
+	assert.NoError(t, err)
+}

libs/notebook/detect_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/libs/fakefs"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -100,11 +101,21 @@ func TestDetectFileWithLongHeader(t *testing.T) {
 	assert.False(t, nb)
 }
 
+type fileInfoWithWorkspaceInfo struct {
+	fakefs.FileInfo
+
+	oi workspace.ObjectInfo
+}
+
+func (f fileInfoWithWorkspaceInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
+	return f.oi
+}
+
 func TestDetectWithObjectInfo(t *testing.T) {
-	fakeFS := &fakeFS{
-		fakeFile{
-			fakeFileInfo{
-				workspace.ObjectInfo{
+	fakefs := fakefs.FS{
+		"file.py": fakefs.File{
+			FileInfo: fileInfoWithWorkspaceInfo{
+				oi: workspace.ObjectInfo{
 					ObjectType: workspace.ObjectTypeNotebook,
 					Language:   workspace.LanguagePython,
 				},
@@ -112,7 +123,7 @@ func TestDetectWithObjectInfo(t *testing.T) {
 		},
 	}
 
-	nb, lang, err := DetectWithFS(fakeFS, "doesntmatter")
+	nb, lang, err := DetectWithFS(fakefs, "file.py")
 	require.NoError(t, err)
 	assert.True(t, nb)
 	assert.Equal(t, workspace.LanguagePython, lang)

libs/notebook/fakefs_test.go

@@ -1,77 +0,0 @@
-package notebook
-
-import (
-	"fmt"
-	"io/fs"
-	"time"
-
-	"github.com/databricks/databricks-sdk-go/service/workspace"
-)
-
-type fakeFS struct {
-	fakeFile
-}
-
-type fakeFile struct {
-	fakeFileInfo
-}
-
-func (f fakeFile) Close() error {
-	return nil
-}
-
-func (f fakeFile) Read(p []byte) (n int, err error) {
-	return 0, fmt.Errorf("not implemented")
-}
-
-func (f fakeFile) Stat() (fs.FileInfo, error) {
-	return f.fakeFileInfo, nil
-}
-
-type fakeFileInfo struct {
-	oi workspace.ObjectInfo
-}
-
-func (f fakeFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo {
-	return f.oi
-}
-
-func (f fakeFileInfo) Name() string {
-	return ""
-}
-
-func (f fakeFileInfo) Size() int64 {
-	return 0
-}
-
-func (f fakeFileInfo) Mode() fs.FileMode {
-	return 0
-}
-
-func (f fakeFileInfo) ModTime() time.Time {
-	return time.Time{}
-}
-
-func (f fakeFileInfo) IsDir() bool {
-	return false
-}
-
-func (f fakeFileInfo) Sys() any {
-	return nil
-}
-
-func (f fakeFS) Open(name string) (fs.File, error) {
-	return f.fakeFile, nil
-}
-
-func (f fakeFS) Stat(name string) (fs.FileInfo, error) {
-	panic("not implemented")
-}
-
-func (f fakeFS) ReadDir(name string) ([]fs.DirEntry, error) {
-	panic("not implemented")
-}
-
-func (f fakeFS) ReadFile(name string) ([]byte, error) {
-	panic("not implemented")
-}

libs/template/builtin.go

@@ -0,0 +1,47 @@
+package template
+
+import (
+	"embed"
+	"io/fs"
+)
+
+//go:embed all:templates
+var builtinTemplates embed.FS
+
+// BuiltinTemplate represents a template that is built into the CLI.
+type BuiltinTemplate struct {
+	Name string
+	FS   fs.FS
+}
+
+// Builtin returns the list of all built-in templates.
+func Builtin() ([]BuiltinTemplate, error) {
+	templates, err := fs.Sub(builtinTemplates, "templates")
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err := fs.ReadDir(templates, ".")
+	if err != nil {
+		return nil, err
+	}
+
+	var out []BuiltinTemplate
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		templateFS, err := fs.Sub(templates, entry.Name())
+		if err != nil {
+			return nil, err
+		}
+
+		out = append(out, BuiltinTemplate{
+			Name: entry.Name(),
+			FS:   templateFS,
+		})
+	}
+
+	return out, nil
+}

libs/template/builtin_test.go

@@ -0,0 +1,28 @@
+package template
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuiltin(t *testing.T) {
+	out, err := Builtin()
+	require.NoError(t, err)
+	assert.Len(t, out, 3)
+
+	// Confirm names.
+	assert.Equal(t, "dbt-sql", out[0].Name)
+	assert.Equal(t, "default-python", out[1].Name)
+	assert.Equal(t, "default-sql", out[2].Name)
+
+	// Confirm that the filesystems work.
+	_, err = fs.Stat(out[0].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
+	assert.NoError(t, err)
+	_, err = fs.Stat(out[1].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
+	assert.NoError(t, err)
+	_, err = fs.Stat(out[2].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
+	assert.NoError(t, err)
+}

libs/template/config.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/jsonschema"
@@ -28,9 +29,8 @@ type config struct {
 	schema *jsonschema.Schema
 }
 
-func newConfig(ctx context.Context, schemaPath string) (*config, error) {
-	// Read config schema
-	schema, err := jsonschema.Load(schemaPath)
+func newConfig(ctx context.Context, templateFS fs.FS, schemaPath string) (*config, error) {
+	schema, err := jsonschema.LoadFS(templateFS, schemaPath)
 	if err != nil {
 		return nil, err
 	}

libs/template/config_test.go

@@ -3,6 +3,8 @@ package template
 import (
 	"context"
 	"fmt"
+	"os"
+	"path"
 	"path/filepath"
 	"testing"
 	"text/template"
@@ -16,7 +18,7 @@ func TestTemplateConfigAssignValuesFromFile(t *testing.T) {
 	testDir := "./testdata/config-assign-from-file"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -32,7 +34,7 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
 	testDir := "./testdata/config-assign-from-file"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -52,7 +54,7 @@ func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T)
 	testDir := "./testdata/config-assign-from-file-invalid-int"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -63,7 +65,7 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te
 	testDir := "./testdata/config-assign-from-file-unknown-property"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
 	err = c.assignValuesFromFile(filepath.Join(testDir, "config.json"))
@@ -78,10 +80,10 @@ func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) {
 	testDir := "./testdata/config-assign-from-default-value"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
-	r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
+	r, err := newRenderer(ctx, nil, nil, os.DirFS("."), "./testdata/empty/template", "./testdata/empty/library", t.TempDir())
 	require.NoError(t, err)
 
 	err = c.assignDefaultValues(r)
@@ -97,10 +99,10 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
 	testDir := "./testdata/config-assign-from-templated-default-value"
 
 	ctx := context.Background()
-	c, err := newConfig(ctx, filepath.Join(testDir, "schema.json"))
+	c, err := newConfig(ctx, os.DirFS(testDir), "schema.json")
 	require.NoError(t, err)
 
-	r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir())
+	r, err := newRenderer(ctx, nil, nil, os.DirFS("."), path.Join(testDir, "template/template"), path.Join(testDir, "template/library"), t.TempDir())
 	require.NoError(t, err)
 
 	// Note: only the string value is templated.
@@ -116,7 +118,7 @@ func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) {
 
 func TestTemplateConfigValidateValuesDefined(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -131,7 +133,7 @@ func TestTemplateConfigValidateValuesDefined(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -147,7 +149,7 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -164,7 +166,7 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) {
 
 func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) {
 	ctx := context.Background()
-	c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json")
+	c, err := newConfig(ctx, os.DirFS("testdata/config-test-schema"), "test-schema.json")
 	require.NoError(t, err)
 
 	c.values = map[string]any{
@@ -271,7 +273,8 @@ func TestTemplateEnumValidation(t *testing.T) {
 }
 
 func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) {
-	_, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json")
+	ctx := context.Background()
+	_, err := newConfig(ctx, os.DirFS("./testdata/config-test-schema"), "invalid-test-schema.json")
 	assert.EqualError(t, err, "template property property-without-description is missing a description")
 }
 

libs/template/file.go

@@ -6,8 +6,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-
-	"github.com/databricks/cli/libs/filer"
+	"slices"
 )
 
 // Interface representing a file to be materialized from a template into a project
@@ -19,6 +18,10 @@ type file interface {
 
 	// Write file to disk at the destination path.
 	PersistToDisk() error
+
+	// contents returns the file contents as a byte slice.
+	// This is used for testing purposes.
+	contents() ([]byte, error)
 }
 
 type destinationPath struct {
@@ -46,8 +49,8 @@ type copyFile struct {
 
 	dstPath *destinationPath
 
-	// Filer rooted at template root. Used to read srcPath.
-	srcFiler filer.Filer
+	// [fs.FS] rooted at template root. Used to read srcPath.
+	srcFS fs.FS
 
 	// Relative path from template root for file to be copied.
 	srcPath string
@@ -63,7 +66,7 @@ func (f *copyFile) PersistToDisk() error {
 	if err != nil {
 		return err
 	}
-	srcFile, err := f.srcFiler.Read(f.ctx, f.srcPath)
+	srcFile, err := f.srcFS.Open(f.srcPath)
 	if err != nil {
 		return err
 	}
@@ -77,6 +80,10 @@ func (f *copyFile) PersistToDisk() error {
 	return err
 }
 
+func (f *copyFile) contents() ([]byte, error) {
+	return fs.ReadFile(f.srcFS, f.srcPath)
+}
+
 type inMemoryFile struct {
 	dstPath *destinationPath
 
@@ -99,3 +106,7 @@ func (f *inMemoryFile) PersistToDisk() error {
 	}
 	return os.WriteFile(path, f.content, f.perm)
 }
+
+func (f *inMemoryFile) contents() ([]byte, error) {
+	return slices.Clone(f.content), nil
+}

libs/template/file_test.go

@@ -8,7 +8,6 @@ import (
 	"runtime"
 	"testing"
 
-	"github.com/databricks/cli/libs/filer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,10 +32,7 @@ func testInMemoryFile(t *testing.T, perm fs.FileMode) {
 
 func testCopyFile(t *testing.T, perm fs.FileMode) {
 	tmpDir := t.TempDir()
-
-	templateFiler, err := filer.NewLocalClient(tmpDir)
-	require.NoError(t, err)
-	err = os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
+	err := os.WriteFile(filepath.Join(tmpDir, "source"), []byte("qwerty"), perm)
 	require.NoError(t, err)
 
 	f := &copyFile{
@@ -45,9 +41,9 @@ func testCopyFile(t *testing.T, perm fs.FileMode) {
 			root:    tmpDir,
 			relPath: "a/b/c",
 		},
-		perm:     perm,
-		srcPath:  "source",
-		srcFiler: templateFiler,
+		perm:    perm,
+		srcPath: "source",
+		srcFS:   os.DirFS(tmpDir),
 	}
 	err = f.PersistToDisk()
 	assert.NoError(t, err)

libs/template/helpers_test.go

@@ -22,7 +22,7 @@ func TestTemplatePrintStringWithoutProcessing(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/print-without-processing/template", "./testdata/print-without-processing/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -39,7 +39,7 @@ func TestTemplateRegexpCompileFunction(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/regexp-compile/template", "./testdata/regexp-compile/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -57,7 +57,7 @@ func TestTemplateRandIntFunction(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/random-int/template", "./testdata/random-int/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -75,7 +75,7 @@ func TestTemplateUuidFunction(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/uuid/template", "./testdata/uuid/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -92,7 +92,7 @@ func TestTemplateUrlFunction(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/urlparse-function/template", "./testdata/urlparse-function/library", tmpDir)
 
 	require.NoError(t, err)
 
@@ -109,7 +109,7 @@ func TestTemplateMapPairFunction(t *testing.T) {
 
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/map-pair/template", "./testdata/map-pair/library", tmpDir)
 
 	require.NoError(t, err)
 
@@ -132,7 +132,7 @@ func TestWorkspaceHost(t *testing.T) {
 	ctx = root.SetWorkspaceClient(ctx, w)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
 
 	require.NoError(t, err)
 
@@ -157,7 +157,7 @@ func TestWorkspaceHostNotConfigured(t *testing.T) {
 	ctx = root.SetWorkspaceClient(ctx, w)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/workspace-host/template", "./testdata/map-pair/library", tmpDir)
 
 	assert.NoError(t, err)
 

libs/template/materialize.go

@@ -2,13 +2,9 @@ package template
 
 import (
 	"context"
-	"embed"
 	"errors"
 	"fmt"
 	"io/fs"
-	"os"
-	"path"
-	"path/filepath"
 
 	"github.com/databricks/cli/libs/cmdio"
 )
@@ -17,39 +13,20 @@ const libraryDirName = "library"
 const templateDirName = "template"
 const schemaFileName = "databricks_template_schema.json"
 
-//go:embed all:templates
-var builtinTemplates embed.FS
-
 // This function materializes the input templates as a project, using user defined
 // configurations.
 // Parameters:
 //
 //	ctx: 			context containing a cmdio object. This is used to prompt the user
 //	configFilePath: file path containing user defined config values
-//	templateRoot: 	root of the template definition
+//	templateFS: 	root of the template definition
 //	outputDir: 	root of directory where to initialize the template
-func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir string) error {
-	// Use a temporary directory in case any builtin templates like default-python are used
-	tempDir, err := os.MkdirTemp("", "templates")
-	defer os.RemoveAll(tempDir)
-	if err != nil {
-		return err
-	}
-	templateRoot, err = prepareBuiltinTemplates(templateRoot, tempDir)
-	if err != nil {
-		return err
+func Materialize(ctx context.Context, configFilePath string, templateFS fs.FS, outputDir string) error {
+	if _, err := fs.Stat(templateFS, schemaFileName); errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaFileName)
 	}
 
-	templatePath := filepath.Join(templateRoot, templateDirName)
-	libraryPath := filepath.Join(templateRoot, libraryDirName)
-	schemaPath := filepath.Join(templateRoot, schemaFileName)
-	helpers := loadHelpers(ctx)
-
-	if _, err := os.Stat(schemaPath); errors.Is(err, fs.ErrNotExist) {
-		return fmt.Errorf("not a bundle template: expected to find a template schema file at %s", schemaPath)
-	}
-
-	config, err := newConfig(ctx, schemaPath)
+	config, err := newConfig(ctx, templateFS, schemaFileName)
 	if err != nil {
 		return err
 	}
@@ -62,7 +39,8 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
 		}
 	}
 
-	r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
+	helpers := loadHelpers(ctx)
+	r, err := newRenderer(ctx, config.values, helpers, templateFS, templateDirName, libraryDirName, outputDir)
 	if err != nil {
 		return err
 	}
@@ -111,44 +89,3 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
 	}
 	return nil
 }
-
-// If the given templateRoot matches
-func prepareBuiltinTemplates(templateRoot string, tempDir string) (string, error) {
-	// Check that `templateRoot` is a clean basename, i.e. `some_path` and not `./some_path` or "."
-	// Return early if that's not the case.
-	if templateRoot == "." || path.Base(templateRoot) != templateRoot {
-		return templateRoot, nil
-	}
-
-	_, err := fs.Stat(builtinTemplates, path.Join("templates", templateRoot))
-	if err != nil {
-		// The given path doesn't appear to be using out built-in templates
-		return templateRoot, nil
-	}
-
-	// We have a built-in template with the same name as templateRoot!
-	// Now we need to make a fully copy of the builtin templates to a real file system
-	// since template.Parse() doesn't support embed.FS.
-	err = fs.WalkDir(builtinTemplates, "templates", func(path string, entry fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		targetPath := filepath.Join(tempDir, path)
-		if entry.IsDir() {
-			return os.Mkdir(targetPath, 0755)
-		} else {
-			content, err := fs.ReadFile(builtinTemplates, path)
-			if err != nil {
-				return err
-			}
-			return os.WriteFile(targetPath, content, 0644)
-		}
-	})
-
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Join(tempDir, "templates", templateRoot), nil
-}

libs/template/materialize_test.go

@@ -3,7 +3,7 @@ package template
 import (
 	"context"
 	"fmt"
-	"path/filepath"
+	"os"
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
@@ -19,6 +19,6 @@ func TestMaterializeForNonTemplateDirectory(t *testing.T) {
 	ctx := root.SetWorkspaceClient(context.Background(), w)
 
 	// Try to materialize a non-template directory.
-	err = Materialize(ctx, "", tmpDir, "")
-	assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", filepath.Join(tmpDir, schemaFileName)))
+	err = Materialize(ctx, "", os.DirFS(tmpDir), "")
+	assert.EqualError(t, err, fmt.Sprintf("not a bundle template: expected to find a template schema file at %s", schemaFileName))
 }

libs/template/renderer.go

@@ -8,14 +8,12 @@ import (
 	"io/fs"
 	"os"
 	"path"
-	"path/filepath"
 	"regexp"
 	"slices"
 	"sort"
 	"strings"
 	"text/template"
 
-	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/databricks-sdk-go/logger"
 )
@@ -52,32 +50,42 @@ type renderer struct {
 	// do not match any glob patterns from this list
 	skipPatterns []string
 
-	// Filer rooted at template root. The file tree from this root is walked to
-	// generate the project
-	templateFiler filer.Filer
+	// [fs.FS] that holds the template's file tree.
+	srcFS fs.FS
 
 	// Root directory for the project instantiated from the template
 	instanceRoot string
 }
 
-func newRenderer(ctx context.Context, config map[string]any, helpers template.FuncMap, templateRoot, libraryRoot, instanceRoot string) (*renderer, error) {
+func newRenderer(
+	ctx context.Context,
+	config map[string]any,
+	helpers template.FuncMap,
+	templateFS fs.FS,
+	templateDir string,
+	libraryDir string,
+	instanceRoot string,
+) (*renderer, error) {
 	// Initialize new template, with helper functions loaded
 	tmpl := template.New("").Funcs(helpers)
 
-	// Load user defined associated templates from the library root
-	libraryGlob := filepath.Join(libraryRoot, "*")
-	matches, err := filepath.Glob(libraryGlob)
+	// Find user-defined templates in the library directory
+	matches, err := fs.Glob(templateFS, path.Join(libraryDir, "*"))
 	if err != nil {
 		return nil, err
 	}
+
+	// Parse user-defined templates.
+	// Note: we do not call [ParseFS] with the glob directly because
+	// it returns an error if no files match the pattern.
 	if len(matches) != 0 {
-		tmpl, err = tmpl.ParseFiles(matches...)
+		tmpl, err = tmpl.ParseFS(templateFS, matches...)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	templateFiler, err := filer.NewLocalClient(templateRoot)
+	srcFS, err := fs.Sub(templateFS, path.Clean(templateDir))
 	if err != nil {
 		return nil, err
 	}
@@ -85,13 +93,13 @@ func newRenderer(ctx context.Context, config map[string]any, helpers template.Fu
 	ctx = log.NewContext(ctx, log.GetLogger(ctx).With("action", "initialize-template"))
 
 	return &renderer{
-		ctx:           ctx,
-		config:        config,
-		baseTemplate:  tmpl,
-		files:         make([]file, 0),
-		skipPatterns:  make([]string, 0),
-		templateFiler: templateFiler,
-		instanceRoot:  instanceRoot,
+		ctx:          ctx,
+		config:       config,
+		baseTemplate: tmpl,
+		files:        make([]file, 0),
+		skipPatterns: make([]string, 0),
+		srcFS:        srcFS,
+		instanceRoot: instanceRoot,
 	}, nil
 }
 
@@ -141,7 +149,7 @@ func (r *renderer) executeTemplate(templateDefinition string) (string, error) {
 
 func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	// read file permissions
-	info, err := r.templateFiler.Stat(r.ctx, relPathTemplate)
+	info, err := fs.Stat(r.srcFS, relPathTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -161,10 +169,10 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 				root:    r.instanceRoot,
 				relPath: relPath,
 			},
-			perm:     perm,
-			ctx:      r.ctx,
-			srcPath:  relPathTemplate,
-			srcFiler: r.templateFiler,
+			perm:    perm,
+			ctx:     r.ctx,
+			srcFS:   r.srcFS,
+			srcPath: relPathTemplate,
 		}, nil
 	} else {
 		// Trim the .tmpl suffix from file name, if specified in the template
@@ -173,7 +181,7 @@ func (r *renderer) computeFile(relPathTemplate string) (file, error) {
 	}
 
 	// read template file's content
-	templateReader, err := r.templateFiler.Read(r.ctx, relPathTemplate)
+	templateReader, err := r.srcFS.Open(relPathTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -263,7 +271,7 @@ func (r *renderer) walk() error {
 		//
 		// 2. For directories: They are appended to a slice, which acts as a queue
 		//     allowing BFS traversal of the template file tree
-		entries, err := r.templateFiler.ReadDir(r.ctx, currentDirectory)
+		entries, err := fs.ReadDir(r.srcFS, currentDirectory)
 		if err != nil {
 			return err
 		}

libs/template/renderer_test.go

@@ -3,9 +3,9 @@ package template
 import (
 	"context"
 	"fmt"
-	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -41,9 +41,8 @@ func assertFilePermissions(t *testing.T, path string, perm fs.FileMode) {
 func assertBuiltinTemplateValid(t *testing.T, template string, settings map[string]any, target string, isServicePrincipal bool, build bool, tempDir string) {
 	ctx := context.Background()
 
-	templatePath, err := prepareBuiltinTemplates(template, tempDir)
+	templateFS, err := fs.Sub(builtinTemplates, path.Join("templates", template))
 	require.NoError(t, err)
-	libraryPath := filepath.Join(templatePath, "library")
 
 	w := &databricks.WorkspaceClient{
 		Config: &workspaceConfig.Config{Host: "https://myhost.com"},
@@ -58,7 +57,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	ctx = root.SetWorkspaceClient(ctx, w)
 	helpers := loadHelpers(ctx)
 
-	renderer, err := newRenderer(ctx, settings, helpers, templatePath, libraryPath, tempDir)
+	renderer, err := newRenderer(ctx, settings, helpers, templateFS, templateDirName, libraryDirName, tempDir)
 	require.NoError(t, err)
 
 	// Evaluate template
@@ -67,7 +66,7 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	err = renderer.persistToDisk()
 	require.NoError(t, err)
 
-	b, err := bundle.Load(ctx, filepath.Join(tempDir, "template", "my_project"))
+	b, err := bundle.Load(ctx, filepath.Join(tempDir, "my_project"))
 	require.NoError(t, err)
 	diags := bundle.Apply(ctx, b, phases.LoadNamedTarget(target))
 	require.NoError(t, diags.Error())
@@ -96,18 +95,6 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri
 	}
 }
 
-func TestPrepareBuiltInTemplatesWithRelativePaths(t *testing.T) {
-	// CWD should not be resolved as a built in template
-	dir, err := prepareBuiltinTemplates(".", t.TempDir())
-	assert.NoError(t, err)
-	assert.Equal(t, ".", dir)
-
-	// relative path should not be resolved as a built in template
-	dir, err = prepareBuiltinTemplates("./default-python", t.TempDir())
-	assert.NoError(t, err)
-	assert.Equal(t, "./default-python", dir)
-}
-
 func TestBuiltinPythonTemplateValid(t *testing.T) {
 	// Test option combinations
 	options := []string{"yes", "no"}
@@ -194,7 +181,7 @@ func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) {
 	ctx := context.Background()
 	ctx = root.SetWorkspaceClient(ctx, nil)
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/email/template", "./testdata/email/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/email/template", "./testdata/email/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -381,7 +368,7 @@ func TestRendererWalk(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/walk/template", "./testdata/walk/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/walk/template", "./testdata/walk/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -392,18 +379,9 @@ func TestRendererWalk(t *testing.T) {
 			if f.DstPath().relPath != path {
 				continue
 			}
-			switch v := f.(type) {
-			case *inMemoryFile:
-				return strings.Trim(string(v.content), "\r\n")
-			case *copyFile:
-				r, err := r.templateFiler.Read(context.Background(), v.srcPath)
-				require.NoError(t, err)
-				b, err := io.ReadAll(r)
-				require.NoError(t, err)
-				return strings.Trim(string(b), "\r\n")
-			default:
-				require.FailNow(t, "execution should not reach here")
-			}
+			b, err := f.contents()
+			require.NoError(t, err)
+			return strings.Trim(string(b), "\r\n")
 		}
 		require.FailNow(t, "file is absent: "+path)
 		return ""
@@ -422,7 +400,7 @@ func TestRendererFailFunction(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/fail/template", "./testdata/fail/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/fail/template", "./testdata/fail/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -435,7 +413,7 @@ func TestRendererSkipsDirsEagerly(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-dir-eagerly/template", "./testdata/skip-dir-eagerly/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -452,7 +430,7 @@ func TestRendererSkipAllFilesInCurrentDirectory(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-all-files-in-cwd/template", "./testdata/skip-all-files-in-cwd/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -475,7 +453,7 @@ func TestRendererSkipPatternsAreRelativeToFileDirectory(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip-is-relative/template", "./testdata/skip-is-relative/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -493,7 +471,7 @@ func TestRendererSkip(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/skip/template", "./testdata/skip/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/skip/template", "./testdata/skip/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -525,7 +503,7 @@ func TestRendererReadsPermissionsBits(t *testing.T) {
 	ctx = root.SetWorkspaceClient(ctx, nil)
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/executable-bit-read/template", "./testdata/executable-bit-read/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -615,7 +593,7 @@ func TestRendererNonTemplatesAreCreatedAsCopyFiles(t *testing.T) {
 	tmpDir := t.TempDir()
 
 	helpers := loadHelpers(ctx)
-	r, err := newRenderer(ctx, nil, helpers, "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
+	r, err := newRenderer(ctx, nil, helpers, os.DirFS("."), "./testdata/copy-file-walk/template", "./testdata/copy-file-walk/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -635,7 +613,7 @@ func TestRendererFileTreeRendering(t *testing.T) {
 	r, err := newRenderer(ctx, map[string]any{
 		"dir_name":  "my_directory",
 		"file_name": "my_file",
-	}, helpers, "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
+	}, helpers, os.DirFS("."), "./testdata/file-tree-rendering/template", "./testdata/file-tree-rendering/library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()
@@ -668,7 +646,7 @@ func TestRendererSubTemplateInPath(t *testing.T) {
 	testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}"))
 
 	tmpDir := t.TempDir()
-	r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir)
+	r, err := newRenderer(ctx, nil, nil, os.DirFS(templateDir), "template", "library", tmpDir)
 	require.NoError(t, err)
 
 	err = r.walk()