From 1c0d67f66c24306ca3a9162b600aa4cbbad2e6c3 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 2 Jun 2023 14:49:59 +0200 Subject: [PATCH] Add fs.FS adapter for the filer interface (#422) ## Changes This enables the use of `io/fs` functions `fs.Glob` and `fs.WalkDir` with filers. We can't use `fs.FS` as the standard interface instead of `filer.Filer` because: 1. It was made for reading from filesystems only, not writing 2. It doesn't take a context for the core functions Therefore a wrapper will do. ## Tests * Added unit tests to cover the adapter through a fake filer. * Manually ran `fs.WalkDir` against both WSFS and DBFS filers. --- libs/filer/fs.go | 151 ++++++++++++++++++++++ libs/filer/fs_test.go | 288 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 libs/filer/fs.go create mode 100644 libs/filer/fs_test.go diff --git a/libs/filer/fs.go b/libs/filer/fs.go new file mode 100644 index 00000000..c6dd21de --- /dev/null +++ b/libs/filer/fs.go @@ -0,0 +1,151 @@ +package filer + +import ( + "bytes" + "context" + "io" + "io/fs" +) + +// filerFS implements the fs.FS interface for a filer. +type filerFS struct { + ctx context.Context + filer Filer +} + +// NewFS returns an fs.FS backed by a filer. +func NewFS(ctx context.Context, filer Filer) fs.FS { + return &filerFS{ctx: ctx, filer: filer} +} + +func (fs *filerFS) Open(name string) (fs.File, error) { + stat, err := fs.filer.Stat(fs.ctx, name) + if err != nil { + return nil, err + } + + if stat.IsDir() { + return &fsDir{fs: fs, name: name, stat: stat}, nil + } + + return &fsFile{fs: fs, name: name, stat: stat}, nil +} + +func (fs *filerFS) ReadDir(name string) ([]fs.DirEntry, error) { + return fs.filer.ReadDir(fs.ctx, name) +} + +func (fs *filerFS) ReadFile(name string) ([]byte, error) { + reader, err := fs.filer.Read(fs.ctx, name) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, reader) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (fs *filerFS) Stat(name string) (fs.FileInfo, error) { + return fs.filer.Stat(fs.ctx, name) +} + +// Type that implements fs.File for a filer-backed fs.FS. +type fsFile struct { + fs *filerFS + name string + stat fs.FileInfo + + reader io.Reader +} + +func (f *fsFile) Stat() (fs.FileInfo, error) { + return f.stat, nil +} + +func (f *fsFile) Read(buf []byte) (int, error) { + if f.reader == nil { + reader, err := f.fs.filer.Read(f.fs.ctx, f.name) + if err != nil { + return 0, err + } + f.reader = reader + } + + return f.reader.Read(buf) +} + +func (f *fsFile) Close() error { + if f.reader == nil { + return fs.ErrClosed + } + f.reader = nil + return nil +} + +// Type that implements fs.ReadDirFile for a filer-backed fs.FS. +type fsDir struct { + fs *filerFS + name string + stat fs.FileInfo + + open bool + entries []fs.DirEntry +} + +func (f *fsDir) Stat() (fs.FileInfo, error) { + return f.stat, nil +} + +func (f *fsDir) Read(buf []byte) (int, error) { + return 0, fs.ErrInvalid +} + +func (f *fsDir) ReadDir(n int) ([]fs.DirEntry, error) { + // Load all directory entries if not already loaded. + if !f.open { + entries, err := f.fs.ReadDir(f.name) + if err != nil { + return nil, err + } + f.open = true + f.entries = entries + } + + // Return all entries if n <= 0. + if n <= 0 { + entries := f.entries + f.entries = nil + return entries, nil + } + + // If there are no more entries, return io.EOF. + if len(f.entries) == 0 { + return nil, io.EOF + } + + // If there are less than n entries, return all entries. + if len(f.entries) < n { + entries := f.entries + f.entries = nil + return entries, nil + } + + // Return n entries. + entries := f.entries[:n] + f.entries = f.entries[n:] + return entries, nil +} + +func (f *fsDir) Close() error { + if !f.open { + return fs.ErrClosed + } + f.open = false + f.entries = nil + return nil +} diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go new file mode 100644 index 00000000..b8c8892e --- /dev/null +++ b/libs/filer/fs_test.go @@ -0,0 +1,288 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeDirEntry struct { + fakeFileInfo +} + +func (entry fakeDirEntry) Type() fs.FileMode { + typ := fs.ModePerm + if entry.dir { + typ |= fs.ModeDir + } + return typ +} + +func (entry fakeDirEntry) Info() (fs.FileInfo, error) { + return entry.fakeFileInfo, nil +} + +type fakeFileInfo struct { + name string + size int64 + dir bool + mode fs.FileMode +} + +func (info fakeFileInfo) Name() string { + return info.name +} + +func (info fakeFileInfo) Size() int64 { + return info.size +} + +func (info fakeFileInfo) Mode() fs.FileMode { + return info.mode +} + +func (info fakeFileInfo) ModTime() time.Time { + return time.Now() +} + +func (info fakeFileInfo) IsDir() bool { + return info.dir +} + +func (info fakeFileInfo) Sys() any { + return nil +} + +type fakeFiler struct { + entries map[string]fakeFileInfo +} + +func (f *fakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *fakeFiler) Read(ctx context.Context, p string) (io.Reader, error) { + _, ok := f.entries[p] + if !ok { + return nil, fs.ErrNotExist + } + + return strings.NewReader("foo"), nil +} + +func (f *fakeFiler) Delete(ctx context.Context, p string) error { + return fmt.Errorf("not implemented") +} + +func (f *fakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { + entry, ok := f.entries[p] + if !ok { + return nil, fs.ErrNotExist + } + + if !entry.dir { + return nil, fs.ErrInvalid + } + + // Find all entries contained in the specified directory `p`. + var out []fs.DirEntry + for k, v := range f.entries { + if k == p || path.Dir(k) != p { + continue + } + + out = append(out, fakeDirEntry{v}) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) + return out, nil +} + +func (f *fakeFiler) Mkdir(ctx context.Context, path string) error { + return fmt.Errorf("not implemented") +} + +func (f *fakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { + entry, ok := f.entries[path] + if !ok { + return nil, fs.ErrNotExist + } + + return entry, nil +} + +func TestFsImplementsFS(t *testing.T) { + var _ fs.FS = &filerFS{} +} + +func TestFsImplementsReadDirFS(t *testing.T) { + var _ fs.ReadDirFS = &filerFS{} +} + +func TestFsImplementsReadFileFS(t *testing.T) { + var _ fs.ReadDirFS = &filerFS{} +} + +func TestFsImplementsStatFS(t *testing.T) { + var _ fs.StatFS = &filerFS{} +} + +func TestFsFileImplementsFsFile(t *testing.T) { + var _ fs.File = &fsFile{} +} + +func TestFsDirImplementsFsReadDirFile(t *testing.T) { + var _ fs.ReadDirFile = &fsDir{} +} + +func fakeFS() fs.FS { + fakeFiler := &fakeFiler{ + entries: map[string]fakeFileInfo{ + ".": {name: "root", dir: true}, + "dirA": {dir: true}, + "dirB": {dir: true}, + "fileA": {size: 3}, + }, + } + + for k, v := range fakeFiler.entries { + if v.name != "" { + continue + } + v.name = path.Base(k) + fakeFiler.entries[k] = v + } + + return NewFS(context.Background(), fakeFiler) +} + +func TestFsGlob(t *testing.T) { + fakeFS := fakeFS() + matches, err := fs.Glob(fakeFS, "*") + require.NoError(t, err) + assert.Equal(t, []string{"dirA", "dirB", "fileA"}, matches) +} + +func TestFsOpenFile(t *testing.T) { + fakeFS := fakeFS() + fakeFile, err := fakeFS.Open("fileA") + require.NoError(t, err) + + info, err := fakeFile.Stat() + require.NoError(t, err) + assert.Equal(t, "fileA", info.Name()) + assert.Equal(t, int64(3), info.Size()) + assert.Equal(t, fs.FileMode(0), info.Mode()) + assert.Equal(t, false, info.IsDir()) + + // Read until closed. + b := make([]byte, 3) + n, err := fakeFile.Read(b) + require.NoError(t, err) + assert.Equal(t, 3, n) + assert.Equal(t, []byte{'f', 'o', 'o'}, b) + _, err = fakeFile.Read(b) + assert.ErrorIs(t, err, io.EOF) + + // Close. + err = fakeFile.Close() + assert.NoError(t, err) + + // Close again. + err = fakeFile.Close() + assert.ErrorIs(t, err, fs.ErrClosed) +} + +func TestFsOpenDir(t *testing.T) { + fakeFS := fakeFS() + fakeFile, err := fakeFS.Open(".") + require.NoError(t, err) + + info, err := fakeFile.Stat() + require.NoError(t, err) + assert.Equal(t, "root", info.Name()) + assert.Equal(t, true, info.IsDir()) + + de, ok := fakeFile.(fs.ReadDirFile) + require.True(t, ok) + + // Read all entries in one shot. + reference, err := de.ReadDir(-1) + require.NoError(t, err) + + // Read entries one at a time. + { + var tmp, entries []fs.DirEntry + var err error + + de.Close() + + for i := 0; i < 3; i++ { + tmp, err = de.ReadDir(1) + require.NoError(t, err) + entries = append(entries, tmp...) + } + + _, err = de.ReadDir(1) + require.ErrorIs(t, err, io.EOF, err) + + // Compare to reference. + assert.Equal(t, reference, entries) + } + + // Read entries and overshoot at the end. + { + var tmp, entries []fs.DirEntry + var err error + + de.Close() + + tmp, err = de.ReadDir(1) + require.NoError(t, err) + entries = append(entries, tmp...) + + tmp, err = de.ReadDir(20) + require.NoError(t, err) + entries = append(entries, tmp...) + + _, err = de.ReadDir(1) + require.ErrorIs(t, err, io.EOF, err) + + // Compare to reference. + assert.Equal(t, reference, entries) + } +} + +func TestFsReadDir(t *testing.T) { + fakeFS := fakeFS().(fs.ReadDirFS) + entries, err := fakeFS.ReadDir(".") + require.NoError(t, err) + assert.Len(t, entries, 3) + assert.Equal(t, "dirA", entries[0].Name()) + assert.Equal(t, "dirB", entries[1].Name()) + assert.Equal(t, "fileA", entries[2].Name()) +} + +func TestFsReadFile(t *testing.T) { + fakeFS := fakeFS().(fs.ReadFileFS) + buf, err := fakeFS.ReadFile("fileA") + require.NoError(t, err) + assert.Equal(t, []byte("foo"), buf) +} + +func TestFsStat(t *testing.T) { + fakeFS := fakeFS().(fs.StatFS) + info, err := fakeFS.Stat("fileA") + require.NoError(t, err) + assert.Equal(t, "fileA", info.Name()) + assert.Equal(t, int64(3), info.Size()) +}