Add databricks fs ls command for WSFS

This commit is contained in:
Shreyas Goenka 2023-05-26 11:35:13 +02:00
parent 273271bc59
commit e1780a4a16
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
5 changed files with 124 additions and 10 deletions

View File

@ -2,22 +2,84 @@ package fs
import (
"fmt"
"path"
"time"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/filer"
"github.com/spf13/cobra"
)
func parseFileInfo(info filer.FileInfo, parentDir string, isAbsolute bool) map[string]string {
fullName := info.Name
if isAbsolute {
fullName = path.Join(parentDir, info.Name)
}
return map[string]string{
"Name": fullName,
"ModTime": info.ModTime.UTC().Format(time.UnixDate),
"Size": fmt.Sprint(info.Size),
"Type": info.Type,
}
}
// lsCmd represents the ls command
var lsCmd = &cobra.Command{
Use: "ls <dir-name>",
Short: "Lists files",
Long: `Lists files`,
Hidden: true,
Use: "ls <dir-name>",
Short: "Lists files",
Long: `Lists files in a DBFS or WSFS directory`,
Args: cobra.MaximumNArgs(1),
Annotations: map[string]string{},
PreRunE: root.MustWorkspaceClient,
RunE: func(cmd *cobra.Command, args []string) error {
return fmt.Errorf("TODO")
// Assign template according to whether -l is specified
template := cmdio.Heredoc(`
{{range .}}{{.Name}}
{{end}}
`)
if longMode {
template = cmdio.Heredoc(`
{{range .}}{{.Type|printf "%-10s"}} {{.Size}} {{.ModTime}} {{.Name}}
{{end}}
`)
}
// Path to list files from. Defaults to`/`
path := "/"
if len(args) > 0 {
path = args[0]
}
// Initialize workspace client
ctx := cmd.Context()
w := root.WorkspaceClient(ctx)
f, err := filer.NewWorkspaceFilesClient(w, path)
if err != nil {
return err
}
// Get file info
filesInfo, err := f.ReadDir(ctx, "")
if err != nil {
return err
}
// Parse it so it's ready to be rendered
output := make([]map[string]string, 0)
for _, info := range filesInfo {
output = append(output, parseFileInfo(info, path, absolute))
}
return cmdio.RenderWithTemplate(ctx, output, template)
},
}
var longMode bool
var absolute bool
func init() {
lsCmd.Flags().BoolVarP(&longMode, "long", "l", false, "Displays full information including size, file type and modification time since Epoch in milliseconds.")
lsCmd.Flags().BoolVar(&absolute, "absolute", false, "Displays absolute paths.")
fsCmd.AddCommand(lsCmd)
}

View File

@ -71,6 +71,18 @@ func (c *cmdIO) Render(v any) error {
}
}
func RenderWithTemplate(ctx context.Context, v any, template string) error {
c := fromContext(ctx)
switch c.outputFormat {
case flags.OutputJSON:
return renderJson(c.out, v)
case flags.OutputText:
return renderTemplate(c.out, template, v)
default:
return fmt.Errorf("invalid output format: %s", c.outputFormat)
}
}
func Render(ctx context.Context, v any) error {
c := fromContext(ctx)
return c.Render(v)

View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"time"
)
type WriteMode int
@ -13,6 +14,20 @@ const (
CreateParentDirectories = iota << 1
)
type FileInfo struct {
// The type of the file in workspace
Type string
// Base name of the file
Name string
// Size in bytes of the file
Size int64
// Last Modified time of the file
ModTime time.Time
}
type FileAlreadyExistsError struct {
path string
}
@ -41,4 +56,7 @@ type Filer interface {
// Delete file at `path`.
Delete(ctx context.Context, path string) error
// Return contents of directory at `path`
ReadDir(ctx context.Context, path string) ([]FileInfo, error)
}

View File

@ -30,10 +30,5 @@ func (p *RootPath) Join(name string) (string, error) {
return "", fmt.Errorf("relative path escapes root: %s", name)
}
// Don't allow name to resolve to the root path.
if strings.TrimPrefix(absPath, p.rootPath) == "" {
return "", fmt.Errorf("relative path resolves to root: %s", name)
}
return absPath, nil
}

View File

@ -10,6 +10,7 @@ import (
"net/url"
"path"
"strings"
"time"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
@ -31,6 +32,7 @@ type WorkspaceFilesClient struct {
}
func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, error) {
//TODO
apiClient, err := client.New(w.Config)
if err != nil {
return nil, err
@ -128,3 +130,28 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string) error {
Recursive: false,
})
}
func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err
}
objects, err := w.workspaceClient.Workspace.ListAll(ctx, workspace.ListWorkspaceRequest{
Path: absPath,
})
if err != nil {
return nil, err
}
info := make([]FileInfo, 0)
for _, i := range objects {
info = append(info, FileInfo{
Type: string(i.ObjectType),
Name: path.Base(i.Path),
Size: i.Size,
ModTime: time.UnixMilli(i.ModifiedAt),
})
}
return info, nil
}