diff --git a/internal/filer_test.go b/internal/filer_test.go index 2dbb8ae66..45227b225 100644 --- a/internal/filer_test.go +++ b/internal/filer_test.go @@ -65,7 +65,7 @@ func temporaryWorkspaceDir(t *testing.T, w *databricks.WorkspaceClient) string { return path } -func TestAccFilerWorkspaceFiles(t *testing.T) { +func setupWorkspaceFilesTest(t *testing.T) (context.Context, filer.Filer) { t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) ctx := context.Background() @@ -81,6 +81,14 @@ func TestAccFilerWorkspaceFiles(t *testing.T) { t.Skip(aerr.Message) } + return ctx, f +} + +func TestAccFilerWorkspaceFilesReadWrite(t *testing.T) { + var err error + + ctx, f := setupWorkspaceFilesTest(t) + // Write should fail because the root path doesn't yet exist. err = f.Write(ctx, "/foo/bar", strings.NewReader(`hello world`)) assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{})) @@ -111,3 +119,56 @@ func TestAccFilerWorkspaceFiles(t *testing.T) { err = f.Delete(ctx, "/foo/bar") assert.NoError(t, err) } + +func TestAccFilerWorkspaceFilesReadDir(t *testing.T) { + var err error + + ctx, f := setupWorkspaceFilesTest(t) + + // We start with an empty directory. + entries, err := f.ReadDir(ctx, ".") + require.NoError(t, err) + assert.Len(t, entries, 0) + + // Write a file. + err = f.Write(ctx, "/hello.txt", strings.NewReader(`hello world`)) + require.NoError(t, err) + + // Create a directory. + err = f.Mkdir(ctx, "/dir") + require.NoError(t, err) + + // Write a file. + err = f.Write(ctx, "/dir/world.txt", strings.NewReader(`hello world`)) + require.NoError(t, err) + + // Create a nested directory (check that it creates intermediate directories). + err = f.Mkdir(ctx, "/dir/a/b/c") + require.NoError(t, err) + + // Expect an error if the path doesn't exist. + _, err = f.ReadDir(ctx, "/dir/a/b/c/d/e") + assert.True(t, errors.As(err, &filer.NoSuchDirectoryError{})) + + // Expect two entries in the root. + entries, err = f.ReadDir(ctx, ".") + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Equal(t, "dir", entries[0].Name) + assert.Equal(t, "hello.txt", entries[1].Name) + assert.Greater(t, entries[1].ModTime.Unix(), int64(0)) + + // Expect two entries in the directory. + entries, err = f.ReadDir(ctx, "/dir") + require.NoError(t, err) + assert.Len(t, entries, 2) + assert.Equal(t, "a", entries[0].Name) + assert.Equal(t, "world.txt", entries[1].Name) + assert.Greater(t, entries[1].ModTime.Unix(), int64(0)) + + // Expect a single entry in the nested path. + entries, err = f.ReadDir(ctx, "/dir/a/b") + require.NoError(t, err) + assert.Len(t, entries, 1) + assert.Equal(t, "c", entries[0].Name) +} diff --git a/libs/filer/filer.go b/libs/filer/filer.go index 92de6e128..841d1b831 100644 --- a/libs/filer/filer.go +++ b/libs/filer/filer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "time" ) type WriteMode int @@ -13,6 +14,22 @@ const ( CreateParentDirectories = iota << 1 ) +// FileInfo abstracts over file information from different file systems. +// Inspired by https://pkg.go.dev/io/fs#FileInfo. +type FileInfo struct { + // The type of the file in workspace. + Type string + + // Base name. + Name string + + // Size in bytes. + Size int64 + + // Modification time. + ModTime time.Time +} + type FileAlreadyExistsError struct { path string } @@ -41,4 +58,10 @@ type Filer interface { // Delete file at `path`. Delete(ctx context.Context, path string) error + + // Return contents of directory at `path`. + ReadDir(ctx context.Context, path string) ([]FileInfo, error) + + // Creates directory at `path`, creating any intermediate directories as required. + Mkdir(ctx context.Context, path string) error } diff --git a/libs/filer/root_path.go b/libs/filer/root_path.go index 65b26d531..bdeff5d73 100644 --- a/libs/filer/root_path.go +++ b/libs/filer/root_path.go @@ -30,10 +30,5 @@ func (p *RootPath) Join(name string) (string, error) { return "", fmt.Errorf("relative path escapes root: %s", name) } - // Don't allow name to resolve to the root path. - if strings.TrimPrefix(absPath, p.rootPath) == "" { - return "", fmt.Errorf("relative path resolves to root: %s", name) - } - return absPath, nil } diff --git a/libs/filer/root_path_test.go b/libs/filer/root_path_test.go index 3787ef36b..965842d03 100644 --- a/libs/filer/root_path_test.go +++ b/libs/filer/root_path_test.go @@ -31,6 +31,26 @@ func testRootPath(t *testing.T, uncleanRoot string) { assert.NoError(t, err) assert.Equal(t, cleanRoot+"/a/b/f/g", remotePath) + remotePath, err = rp.Join(".//a/..//./b/..") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("a/b/../..") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join(".") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + + remotePath, err = rp.Join("/") + assert.NoError(t, err) + assert.Equal(t, cleanRoot, remotePath) + _, err = rp.Join("..") assert.ErrorContains(t, err, `relative path escapes root: ..`) @@ -57,21 +77,6 @@ func testRootPath(t *testing.T, uncleanRoot string) { _, err = rp.Join("../..") assert.ErrorContains(t, err, `relative path escapes root: ../..`) - - _, err = rp.Join(".//a/..//./b/..") - assert.ErrorContains(t, err, `relative path resolves to root: .//a/..//./b/..`) - - _, err = rp.Join("a/b/../..") - assert.ErrorContains(t, err, "relative path resolves to root: a/b/../..") - - _, err = rp.Join("") - assert.ErrorContains(t, err, "relative path resolves to root: ") - - _, err = rp.Join(".") - assert.ErrorContains(t, err, "relative path resolves to root: .") - - _, err = rp.Join("/") - assert.ErrorContains(t, err, "relative path resolves to root: /") } func TestRootPathClean(t *testing.T) { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index ff813f091..33fefc3d2 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -9,7 +9,9 @@ import ( "net/http" "net/url" "path" + "sort" "strings" + "time" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -128,3 +130,52 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string) error { Recursive: false, }) } + +func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) { + absPath, err := w.root.Join(name) + if err != nil { + return nil, err + } + + objects, err := w.workspaceClient.Workspace.ListAll(ctx, workspace.ListWorkspaceRequest{ + Path: absPath, + }) + if err != nil { + // If we got an API error we deal with it below. + var aerr *apierr.APIError + if !errors.As(err, &aerr) { + return nil, err + } + + // This API returns a 404 if the specified path does not exist. + if aerr.StatusCode == http.StatusNotFound { + return nil, NoSuchDirectoryError{path.Dir(absPath)} + } + + return nil, err + } + + info := make([]FileInfo, len(objects)) + for i, v := range objects { + info[i] = FileInfo{ + Type: string(v.ObjectType), + Name: path.Base(v.Path), + Size: v.Size, + ModTime: time.UnixMilli(v.ModifiedAt), + } + } + + // Sort by name for parity with os.ReadDir. + sort.Slice(info, func(i, j int) bool { return info[i].Name < info[j].Name }) + return info, nil +} + +func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { + dirPath, err := w.root.Join(name) + if err != nil { + return err + } + return w.workspaceClient.Workspace.Mkdirs(ctx, workspace.Mkdirs{ + Path: dirPath, + }) +}