Add Mkdir and ReadDir functions to filer.Filer interface (#414)

## Changes

This cherry-picks the filer changes from #408.

## Tests

Manually ran integration tests.

internal/filer_test.go

@@ -65,7 +65,7 @@ func temporaryWorkspaceDir(t *testing.T, w *databricks.WorkspaceClient) string {
 	return path
 }
 
-func TestAccFilerWorkspaceFiles(t *testing.T) {
+func setupWorkspaceFilesTest(t *testing.T) (context.Context, filer.Filer) {
 	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
 
 	ctx := context.Background()
@@ -81,6 +81,14 @@ func TestAccFilerWorkspaceFiles(t *testing.T) {
 		t.Skip(aerr.Message)
 	}
 
+	return ctx, f
+}
+
+func TestAccFilerWorkspaceFilesReadWrite(t *testing.T) {
+	var err error
+
+	ctx, f := setupWorkspaceFilesTest(t)
+
 	// Write should fail because the root path doesn't yet exist.
 	err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello world`))
 	assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}))
@@ -111,3 +119,56 @@ func TestAccFilerWorkspaceFiles(t *testing.T) {
 	err = f.Delete(ctx, "/foo/bar")
 	assert.NoError(t, err)
 }
+
+func TestAccFilerWorkspaceFilesReadDir(t *testing.T) {
+	var err error
+
+	ctx, f := setupWorkspaceFilesTest(t)
+
+	// We start with an empty directory.
+	entries, err := f.ReadDir(ctx, ".")
+	require.NoError(t, err)
+	assert.Len(t, entries, 0)
+
+	// Write a file.
+	err = f.Write(ctx, "/hello.txt", strings.NewReader(`hello world`))
+	require.NoError(t, err)
+
+	// Create a directory.
+	err = f.Mkdir(ctx, "/dir")
+	require.NoError(t, err)
+
+	// Write a file.
+	err = f.Write(ctx, "/dir/world.txt", strings.NewReader(`hello world`))
+	require.NoError(t, err)
+
+	// Create a nested directory (check that it creates intermediate directories).
+	err = f.Mkdir(ctx, "/dir/a/b/c")
+	require.NoError(t, err)
+
+	// Expect an error if the path doesn't exist.
+	_, err = f.ReadDir(ctx, "/dir/a/b/c/d/e")
+	assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{}))
+
+	// Expect two entries in the root.
+	entries, err = f.ReadDir(ctx, ".")
+	require.NoError(t, err)
+	assert.Len(t, entries, 2)
+	assert.Equal(t, "dir", entries[0].Name)
+	assert.Equal(t, "hello.txt", entries[1].Name)
+	assert.Greater(t, entries[1].ModTime.Unix(), int64(0))
+
+	// Expect two entries in the directory.
+	entries, err = f.ReadDir(ctx, "/dir")
+	require.NoError(t, err)
+	assert.Len(t, entries, 2)
+	assert.Equal(t, "a", entries[0].Name)
+	assert.Equal(t, "world.txt", entries[1].Name)
+	assert.Greater(t, entries[1].ModTime.Unix(), int64(0))
+
+	// Expect a single entry in the nested path.
+	entries, err = f.ReadDir(ctx, "/dir/a/b")
+	require.NoError(t, err)
+	assert.Len(t, entries, 1)
+	assert.Equal(t, "c", entries[0].Name)
+}

libs/filer/filer.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"time"
 )
 
 type WriteMode int
@@ -13,6 +14,22 @@ const (
 	CreateParentDirectories           = iota << 1
 )
 
+// FileInfo abstracts over file information from different file systems.
+// Inspired by https://pkg.go.dev/io/fs#FileInfo.
+type FileInfo struct {
+	// The type of the file in workspace.
+	Type string
+
+	// Base name.
+	Name string
+
+	// Size in bytes.
+	Size int64
+
+	// Modification time.
+	ModTime time.Time
+}
+
 type FileAlreadyExistsError struct {
 	path string
 }
@@ -41,4 +58,10 @@ type Filer interface {
 
 	// Delete file at `path`.
 	Delete(ctx context.Context, path string) error
+
+	// Return contents of directory at `path`.
+	ReadDir(ctx context.Context, path string) ([]FileInfo, error)
+
+	// Creates directory at `path`, creating any intermediate directories as required.
+	Mkdir(ctx context.Context, path string) error
 }

libs/filer/root_path.go

@@ -30,10 +30,5 @@ func (p *RootPath) Join(name string) (string, error) {
 		return "", fmt.Errorf("relative path escapes root: %s", name)
 	}
 
-	// Don't allow name to resolve to the root path.
-	if strings.TrimPrefix(absPath, p.rootPath) == "" {
-		return "", fmt.Errorf("relative path resolves to root: %s", name)
-	}
-
 	return absPath, nil
 }

libs/filer/root_path_test.go

@@ -31,6 +31,26 @@ func testRootPath(t *testing.T, uncleanRoot string) {
 	assert.NoError(t, err)
 	assert.Equal(t, cleanRoot+"/a/b/f/g", remotePath)
 
+	remotePath, err = rp.Join(".//a/..//./b/..")
+	assert.NoError(t, err)
+	assert.Equal(t, cleanRoot, remotePath)
+
+	remotePath, err = rp.Join("a/b/../..")
+	assert.NoError(t, err)
+	assert.Equal(t, cleanRoot, remotePath)
+
+	remotePath, err = rp.Join("")
+	assert.NoError(t, err)
+	assert.Equal(t, cleanRoot, remotePath)
+
+	remotePath, err = rp.Join(".")
+	assert.NoError(t, err)
+	assert.Equal(t, cleanRoot, remotePath)
+
+	remotePath, err = rp.Join("/")
+	assert.NoError(t, err)
+	assert.Equal(t, cleanRoot, remotePath)
+
 	_, err = rp.Join("..")
 	assert.ErrorContains(t, err, `relative path escapes root: ..`)
 
@@ -57,21 +77,6 @@ func testRootPath(t *testing.T, uncleanRoot string) {
 
 	_, err = rp.Join("../..")
 	assert.ErrorContains(t, err, `relative path escapes root: ../..`)
-
-	_, err = rp.Join(".//a/..//./b/..")
-	assert.ErrorContains(t, err, `relative path resolves to root: .//a/..//./b/..`)
-
-	_, err = rp.Join("a/b/../..")
-	assert.ErrorContains(t, err, "relative path resolves to root: a/b/../..")
-
-	_, err = rp.Join("")
-	assert.ErrorContains(t, err, "relative path resolves to root: ")
-
-	_, err = rp.Join(".")
-	assert.ErrorContains(t, err, "relative path resolves to root: .")
-
-	_, err = rp.Join("/")
-	assert.ErrorContains(t, err, "relative path resolves to root: /")
 }
 
 func TestRootPathClean(t *testing.T) {

libs/filer/workspace_files_client.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"sort"
 	"strings"
+	"time"
 
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/apierr"
@@ -128,3 +130,52 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string) error {
 		Recursive: false,
 	})
 }
+
+func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) {
+	absPath, err := w.root.Join(name)
+	if err != nil {
+		return nil, err
+	}
+
+	objects, err := w.workspaceClient.Workspace.ListAll(ctx, workspace.ListWorkspaceRequest{
+		Path: absPath,
+	})
+	if err != nil {
+		// If we got an API error we deal with it below.
+		var aerr *apierr.APIError
+		if !errors.As(err, &aerr) {
+			return nil, err
+		}
+
+		// This API returns a 404 if the specified path does not exist.
+		if aerr.StatusCode == http.StatusNotFound {
+			return nil, NoSuchDirectoryError{path.Dir(absPath)}
+		}
+
+		return nil, err
+	}
+
+	info := make([]FileInfo, len(objects))
+	for i, v := range objects {
+		info[i] = FileInfo{
+			Type:    string(v.ObjectType),
+			Name:    path.Base(v.Path),
+			Size:    v.Size,
+			ModTime: time.UnixMilli(v.ModifiedAt),
+		}
+	}
+
+	// Sort by name for parity with os.ReadDir.
+	sort.Slice(info, func(i, j int) bool { return info[i].Name < info[j].Name })
+	return info, nil
+}
+
+func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error {
+	dirPath, err := w.root.Join(name)
+	if err != nil {
+		return err
+	}
+	return w.workspaceClient.Workspace.Mkdirs(ctx, workspace.Mkdirs{
+		Path: dirPath,
+	})
+}