steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove dependency on global state for remaining commands (#613) 

## Changes

This removes the remaining dependency on global state and unblocks work
to parallelize integration tests. As is, we can already uncomment an
integration test that had to be skipped because of other tests tainting
global state. This is no longer an issue.

Also see #595 and #606.

## Tests

* Unit and integration tests pass.
* Manually confirmed the help output is the same.
This commit is contained in:
Pieter Noordhuis 2023-07-27 12:03:08 +02:00 committed by GitHub
parent ed972f7ae0
commit bee7a16cb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 476 additions and 502 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/api/api.go
View File

@ -5,7 +5,6 @@ import (
"net/http"
"strings"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/databricks-sdk-go/client"
@ -13,9 +12,22 @@ import (
"github.com/spf13/cobra"
)
var apiCmd = &cobra.Command{
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "api",
Short: "Perform Databricks API call",
}
cmd.AddCommand(
makeCommand(http.MethodGet),
makeCommand(http.MethodHead),
makeCommand(http.MethodPost),
makeCommand(http.MethodPut),
makeCommand(http.MethodPatch),
makeCommand(http.MethodDelete),
)
return cmd
}
func makeCommand(method string) *cobra.Command {
@ -59,15 +71,3 @@ func makeCommand(method string) *cobra.Command {
command.Flags().Var(&payload, "json", `either inline JSON string or @path/to/file.json with request body`)
return command
}
func init() {
apiCmd.AddCommand(
makeCommand(http.MethodGet),
makeCommand(http.MethodHead),
makeCommand(http.MethodPost),
makeCommand(http.MethodPut),
makeCommand(http.MethodPatch),
makeCommand(http.MethodDelete),
)
root.RootCmd.AddCommand(apiCmd)
}

23
cmd/auth/auth.go
View File

@ -3,18 +3,27 @@ package auth
import (
"context"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/auth"
"github.com/databricks/cli/libs/cmdio"
"github.com/spf13/cobra"
)
var authCmd = &cobra.Command{
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "auth",
Short: "Authentication related commands",
}
}
var persistentAuth auth.PersistentAuth
var perisistentAuth auth.PersistentAuth
cmd.PersistentFlags().StringVar(&perisistentAuth.Host, "host", perisistentAuth.Host, "Databricks Host")
cmd.PersistentFlags().StringVar(&perisistentAuth.AccountID, "account-id", perisistentAuth.AccountID, "Databricks Account ID")
cmd.AddCommand(newEnvCommand())
cmd.AddCommand(newLoginCommand(&perisistentAuth))
cmd.AddCommand(newProfilesCommand())
cmd.AddCommand(newTokenCommand(&perisistentAuth))
return cmd
}
func promptForHost(ctx context.Context) (string, error) {
prompt := cmdio.Prompt(ctx)
@ -41,9 +50,3 @@ func promptForAccountID(ctx context.Context) (string, error) {
}
return accountId, nil
}
func init() {
root.RootCmd.AddCommand(authCmd)
authCmd.PersistentFlags().StringVar(&persistentAuth.Host, "host", persistentAuth.Host, "Databricks Host")
authCmd.PersistentFlags().StringVar(&persistentAuth.AccountID, "account-id", persistentAuth.AccountID, "Databricks Account ID")
}

23
cmd/auth/env.go
View File

@ -89,10 +89,18 @@ func loadFromDatabricksCfg(cfg *config.Config) error {
return nil
}
var envCmd = &cobra.Command{
func newEnvCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "env",
Short: "Get env",
RunE: func(cmd *cobra.Command, args []string) error {
}
var host string
var profile string
cmd.Flags().StringVar(&host, "host", host, "Hostname to get auth env for")
cmd.Flags().StringVar(&profile, "profile", profile, "Profile to get auth env for")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
cfg := &config.Config{
Host: host,
Profile: profile,
@ -130,14 +138,7 @@ var envCmd = &cobra.Command{
}
cmd.OutOrStdout().Write(raw)
return nil
},
}
}
var host string
var profile string
func init() {
authCmd.AddCommand(envCmd)
envCmd.Flags().StringVar(&host, "host", host, "Hostname to get auth env for")
envCmd.Flags().StringVar(&profile, "profile", profile, "Profile to get auth env for")
return cmd
}

32
cmd/auth/login.go
View File

@ -14,10 +14,7 @@ import (
"github.com/spf13/cobra"
)
var loginTimeout time.Duration
var configureCluster bool
func configureHost(ctx context.Context, args []string, argIndex int) error {
func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, args []string, argIndex int) error {
if len(args) > argIndex {
persistentAuth.Host = args[argIndex]
return nil
@ -31,13 +28,23 @@ func configureHost(ctx context.Context, args []string, argIndex int) error {
return nil
}
var loginCmd = &cobra.Command{
func newLoginCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
cmd := &cobra.Command{
Use: "login [HOST]",
Short: "Authenticate this machine",
RunE: func(cmd *cobra.Command, args []string) error {
}
var loginTimeout time.Duration
var configureCluster bool
cmd.Flags().DurationVar(&loginTimeout, "timeout", auth.DefaultTimeout,
"Timeout for completing login challenge in the browser")
cmd.Flags().BoolVar(&configureCluster, "configure-cluster", false,
"Prompts to configure cluster")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if persistentAuth.Host == "" {
configureHost(ctx, args, 0)
configureHost(ctx, persistentAuth, args, 0)
}
defer persistentAuth.Close()
@ -108,14 +115,7 @@ var loginCmd = &cobra.Command{
cmdio.LogString(ctx, fmt.Sprintf("Profile %s was successfully saved", profileName))
return nil
},
}
}
func init() {
authCmd.AddCommand(loginCmd)
loginCmd.Flags().DurationVar(&loginTimeout, "timeout", auth.DefaultTimeout,
"Timeout for completing login challenge in the browser")
loginCmd.Flags().BoolVar(&configureCluster, "configure-cluster", false,
"Prompts to configure cluster")
return cmd
}

23
cmd/auth/profiles.go
View File

@ -44,7 +44,7 @@ func (c *profileMetadata) IsEmpty() bool {
return c.Host == "" && c.AccountID == ""
}
func (c *profileMetadata) Load(ctx context.Context) {
func (c *profileMetadata) Load(ctx context.Context, skipValidate bool) {
// TODO: disable config loaders other than configfile
cfg := &config.Config{Profile: c.Name}
_ = cfg.EnsureResolved()
@ -94,7 +94,8 @@ func (c *profileMetadata) Load(ctx context.Context) {
c.Host = cfg.Host
}
var profilesCmd = &cobra.Command{
func newProfilesCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "profiles",
Short: "Lists profiles from ~/.databrickscfg",
Annotations: map[string]string{
@ -103,7 +104,12 @@ var profilesCmd = &cobra.Command{
{{range .Profiles}}{{.Name | green}} {{.Host|cyan}} {{bool .Valid}}
{{end}}`),
},
RunE: func(cmd *cobra.Command, args []string) error {
}
var skipValidate bool
cmd.Flags().BoolVar(&skipValidate, "skip-validate", false, "Whether to skip validating the profiles")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
var profiles []*profileMetadata
iniFile, err := getDatabricksCfg()
if os.IsNotExist(err) {
@ -126,7 +132,7 @@ var profilesCmd = &cobra.Command{
wg.Add(1)
go func() {
// load more information about profile
profile.Load(cmd.Context())
profile.Load(cmd.Context(), skipValidate)
wg.Done()
}()
profiles = append(profiles, profile)
@ -135,12 +141,7 @@ var profilesCmd = &cobra.Command{
return cmdio.Render(cmd.Context(), struct {
Profiles []*profileMetadata `json:"profiles"`
}{profiles})
},
}
}
var skipValidate bool
func init() {
authCmd.AddCommand(profilesCmd)
profilesCmd.Flags().BoolVar(&skipValidate, "skip-validate", false, "Whether to skip validating the profiles")
return cmd
}

23
cmd/auth/token.go
View File

@ -9,15 +9,20 @@ import (
"github.com/spf13/cobra"
)
var tokenTimeout time.Duration
var tokenCmd = &cobra.Command{
func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
cmd := &cobra.Command{
Use: "token [HOST]",
Short: "Get authentication token",
RunE: func(cmd *cobra.Command, args []string) error {
}
var tokenTimeout time.Duration
cmd.Flags().DurationVar(&tokenTimeout, "timeout", auth.DefaultTimeout,
"Timeout for acquiring a token.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if persistentAuth.Host == "" {
configureHost(ctx, args, 0)
configureHost(ctx, persistentAuth, args, 0)
}
defer persistentAuth.Close()
@ -33,11 +38,7 @@ var tokenCmd = &cobra.Command{
}
cmd.OutOrStdout().Write(raw)
return nil
},
}
}
func init() {
authCmd.AddCommand(tokenCmd)
tokenCmd.Flags().DurationVar(&tokenTimeout, "timeout", auth.DefaultTimeout,
"Timeout for acquiring a token.")
return cmd
}

23
cmd/bundle/bundle.go Normal file
View File

@ -0,0 +1,23 @@
package bundle
import (
"github.com/spf13/cobra"
)
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "bundle",
Short: "Databricks Asset Bundles",
}
initVariableFlag(cmd)
cmd.AddCommand(newDeployCommand())
cmd.AddCommand(newDestroyCommand())
cmd.AddCommand(newLaunchCommand())
cmd.AddCommand(newRunCommand())
cmd.AddCommand(newSchemaCommand())
cmd.AddCommand(newSyncCommand())
cmd.AddCommand(newTestCommand())
cmd.AddCommand(newValidateCommand())
return cmd
}

19
cmd/bundle/debug/debug.go
View File

@ -1,19 +0,0 @@
package debug
import (
"github.com/spf13/cobra"
parent "github.com/databricks/cli/cmd/bundle"
)
var debugCmd = &cobra.Command{
Use: "debug",
}
func AddCommand(cmd *cobra.Command) {
debugCmd.AddCommand(cmd)
}
func init() {
parent.AddCommand(debugCmd)
}

30
cmd/bundle/debug/whoami.go
View File

@ -1,30 +0,0 @@
package debug
import (
"fmt"
"github.com/databricks/cli/bundle"
bundleCmd "github.com/databricks/cli/cmd/bundle"
"github.com/spf13/cobra"
)
var whoamiCmd = &cobra.Command{
Use: "whoami",
PreRunE: bundleCmd.ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
w := bundle.Get(ctx).WorkspaceClient()
user, err := w.CurrentUser.Me(ctx)
if err != nil {
return err
}
fmt.Fprintln(cmd.OutOrStdout(), user.UserName)
return nil
},
}
func init() {
debugCmd.AddCommand(whoamiCmd)
}

24
cmd/bundle/deploy.go
View File

@ -6,12 +6,19 @@ import (
"github.com/spf13/cobra"
)
var deployCmd = &cobra.Command{
func newDeployCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "deploy",
Short: "Deploy bundle",
PreRunE: ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
}
var forceDeploy bool
var computeID string
cmd.Flags().BoolVar(&forceDeploy, "force", false, "Force acquisition of deployment lock.")
cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
b := bundle.Get(cmd.Context())
// If `--force` is specified, force acquisition of the deployment lock.
@ -23,14 +30,7 @@ var deployCmd = &cobra.Command{
phases.Build(),
phases.Deploy(),
))
},
}
}
var forceDeploy bool
var computeID string
func init() {
AddCommand(deployCmd)
deployCmd.Flags().BoolVar(&forceDeploy, "force", false, "Force acquisition of deployment lock.")
deployCmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.")
return cmd
}

23
cmd/bundle/destroy.go
View File

@ -12,12 +12,20 @@ import (
"golang.org/x/term"
)
var destroyCmd = &cobra.Command{
func newDestroyCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "destroy",
Short: "Destroy deployed bundle resources",
PreRunE: ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
}
var autoApprove bool
var forceDestroy bool
cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals for deleting resources and files")
cmd.Flags().BoolVar(&forceDestroy, "force", false, "Force acquisition of deployment lock.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
b := bundle.Get(ctx)
@ -47,14 +55,7 @@ var destroyCmd = &cobra.Command{
phases.Build(),
phases.Destroy(),
))
},
}
}
var autoApprove bool
var forceDestroy bool
func init() {
AddCommand(destroyCmd)
destroyCmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals for deleting resources and files")
destroyCmd.Flags().BoolVar(&forceDestroy, "force", false, "Force acquisition of deployment lock.")
return cmd
}

13
cmd/bundle/launch.go
View File

@ -7,7 +7,8 @@ import (
"github.com/spf13/cobra"
)
var launchCmd = &cobra.Command{
func newLaunchCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "launch",
Short: "Launches a notebook on development cluster",
Long: `Reads a file and executes it on dev cluster`,
@ -17,7 +18,9 @@ var launchCmd = &cobra.Command{
Hidden: true,
PreRunE: root.MustConfigureBundle,
RunE: func(cmd *cobra.Command, args []string) error {
}
cmd.RunE = func(cmd *cobra.Command, args []string) error {
return fmt.Errorf("TODO")
// contents, err := os.ReadFile(args[0])
// if err != nil {
@ -29,9 +32,7 @@ var launchCmd = &cobra.Command{
// }
// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
// return nil
},
}
}
func init() {
AddCommand(launchCmd)
return cmd
}

23
cmd/bundle/root.go
View File

@ -1,23 +0,0 @@
package bundle
import (
"github.com/databricks/cli/cmd/root"
"github.com/spf13/cobra"
)
// rootCmd represents the root command for the bundle subcommand.
var rootCmd = &cobra.Command{
Use: "bundle",
Short: "Databricks Asset Bundles",
}
func AddCommand(cmd *cobra.Command) {
rootCmd.AddCommand(cmd)
}
var variables []string
func init() {
root.RootCmd.AddCommand(rootCmd)
AddVariableFlag(rootCmd)
}

28
cmd/bundle/run.go
View File

@ -13,16 +13,22 @@ import (
"github.com/spf13/cobra"
)
var runOptions run.Options
var noWait bool
var runCmd = &cobra.Command{
func newRunCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "run [flags] KEY",
Short: "Run a workload (e.g. a job or a pipeline)",
Args: cobra.ExactArgs(1),
PreRunE: ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
}
var runOptions run.Options
runOptions.Define(cmd.Flags())
var noWait bool
cmd.Flags().BoolVar(&noWait, "no-wait", false, "Don't wait for the run to complete.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
b := bundle.Get(cmd.Context())
err := bundle.Apply(cmd.Context(), b, bundle.Seq(
@ -65,9 +71,9 @@ var runCmd = &cobra.Command{
}
}
return nil
},
}
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if len(args) > 0 {
return nil, cobra.ShellCompDirectiveNoFileComp
}
@ -86,11 +92,7 @@ var runCmd = &cobra.Command{
}
return run.ResourceCompletions(b), cobra.ShellCompDirectiveNoFileComp
},
}
}
func init() {
runOptions.Define(runCmd.Flags())
rootCmd.AddCommand(runCmd)
runCmd.Flags().BoolVar(&noWait, "no-wait", false, "Don't wait for the run to complete.")
return cmd
}

22
cmd/bundle/schema.go
View File

@ -9,11 +9,18 @@ import (
"github.com/spf13/cobra"
)
var schemaCmd = &cobra.Command{
func newSchemaCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "schema",
Short: "Generate JSON Schema for bundle configuration",
}
RunE: func(cmd *cobra.Command, args []string) error {
var openapi string
var onlyDocs bool
cmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
cmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
docs, err := schema.BundleDocs(openapi)
if err != nil {
return err
@ -34,14 +41,7 @@ var schemaCmd = &cobra.Command{
}
cmd.OutOrStdout().Write(result)
return nil
},
}
}
var openapi string
var onlyDocs bool
func init() {
AddCommand(schemaCmd)
schemaCmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
schemaCmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
return cmd
}

41
cmd/bundle/sync.go
View File

@ -11,7 +11,13 @@ import (
"github.com/spf13/cobra"
)
func syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) {
type syncFlags struct {
interval time.Duration
full bool
watch bool
}
func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) {
cacheDir, err := b.CacheDir()
if err != nil {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
@ -20,8 +26,8 @@ func syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOpti
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Full: full,
PollInterval: interval,
Full: f.full,
PollInterval: f.interval,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
@ -29,13 +35,21 @@ func syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOpti
return &opts, nil
}
var syncCmd = &cobra.Command{
func newSyncCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "sync [flags]",
Short: "Synchronize bundle tree to the workspace",
Args: cobra.NoArgs,
PreRunE: ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
}
var f syncFlags
cmd.Flags().DurationVar(&f.interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)")
cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
b := bundle.Get(cmd.Context())
// Run initialize phase to make sure paths are set.
@ -44,7 +58,7 @@ var syncCmd = &cobra.Command{
return err
}
opts, err := syncOptionsFromBundle(cmd, b)
opts, err := f.syncOptionsFromBundle(cmd, b)
if err != nil {
return err
}
@ -57,21 +71,12 @@ var syncCmd = &cobra.Command{
log.Infof(ctx, "Remote file sync location: %v", opts.RemotePath)
if watch {
if f.watch {
return s.RunContinuous(ctx)
}
return s.RunOnce(ctx)
},
}
}
var interval time.Duration
var full bool
var watch bool
func init() {
AddCommand(syncCmd)
syncCmd.Flags().DurationVar(&interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
syncCmd.Flags().BoolVar(&full, "full", false, "perform full synchronization (default is incremental)")
syncCmd.Flags().BoolVar(&watch, "watch", false, "watch local file system for changes")
return cmd
}

13
cmd/bundle/test.go
View File

@ -7,7 +7,8 @@ import (
"github.com/spf13/cobra"
)
var testCmd = &cobra.Command{
func newTestCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "test",
Short: "run tests for the project",
Long: `This is longer description of the command`,
@ -16,7 +17,9 @@ var testCmd = &cobra.Command{
Hidden: true,
PreRunE: root.MustConfigureBundle,
RunE: func(cmd *cobra.Command, args []string) error {
}
cmd.RunE = func(cmd *cobra.Command, args []string) error {
return fmt.Errorf("TODO")
// results := project.RunPythonOnDev(cmd.Context(), `return 1`)
// if results.Failed() {
@ -24,9 +27,7 @@ var testCmd = &cobra.Command{
// }
// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
// return nil
},
}
}
func init() {
AddCommand(testCmd)
return cmd
}

13
cmd/bundle/validate.go
View File

@ -8,12 +8,15 @@ import (
"github.com/spf13/cobra"
)
var validateCmd = &cobra.Command{
func newValidateCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "validate",
Short: "Validate configuration",
PreRunE: ConfigureBundleWithVariables,
RunE: func(cmd *cobra.Command, args []string) error {
}
cmd.RunE = func(cmd *cobra.Command, args []string) error {
b := bundle.Get(cmd.Context())
err := bundle.Apply(cmd.Context(), b, phases.Initialize())
@ -27,9 +30,7 @@ var validateCmd = &cobra.Command{
}
cmd.OutOrStdout().Write(buf)
return nil
},
}
}
func init() {
AddCommand(validateCmd)
return cmd
}

9
cmd/bundle/variables.go
View File

@ -13,11 +13,16 @@ func ConfigureBundleWithVariables(cmd *cobra.Command, args []string) error {
return err
}
variables, err := cmd.Flags().GetStringSlice("var")
if err != nil {
return err
}
// Initialize variables by assigning them values passed as command line flags
b := bundle.Get(cmd.Context())
return b.Config.InitializeVariables(variables)
}
func AddVariableFlag(cmd *cobra.Command) {
cmd.PersistentFlags().StringSliceVar(&variables, "var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`)
func initVariableFlag(cmd *cobra.Command) {
cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`)
}

30
cmd/cmd.go
View File

@ -1,23 +1,21 @@
package cmd
import (
"sync"
"github.com/databricks/cli/cmd/account"
"github.com/databricks/cli/cmd/api"
"github.com/databricks/cli/cmd/auth"
"github.com/databricks/cli/cmd/bundle"
"github.com/databricks/cli/cmd/configure"
"github.com/databricks/cli/cmd/fs"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/cmd/sync"
"github.com/databricks/cli/cmd/version"
"github.com/databricks/cli/cmd/workspace"
"github.com/spf13/cobra"
)
var once sync.Once
var cmd *cobra.Command
func New() *cobra.Command {
// TODO: this command is still a global.
// Once the non-generated commands are all instantiatable,
// we can remove the global and instantiate this as well.
once.Do(func() {
cli := root.RootCmd
cli := root.New()
// Add account subcommand.
cli.AddCommand(account.New())
@ -33,8 +31,14 @@ func New() *cobra.Command {
cli.AddGroup(&groups[i])
}
cmd = cli
})
// Add other subcommands.
cli.AddCommand(api.New())
cli.AddCommand(auth.New())
cli.AddCommand(bundle.New())
cli.AddCommand(configure.New())
cli.AddCommand(fs.New())
cli.AddCommand(sync.New())
cli.AddCommand(version.New())
return cmd
return cli
}

39
cmd/configure/configure.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"net/url"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/databrickscfg"
"github.com/databricks/databricks-sdk-go/config"
@ -112,19 +111,30 @@ func configureNonInteractive(cmd *cobra.Command, ctx context.Context, cfg *confi
return nil
}
var configureCmd = &cobra.Command{
func newConfigureCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "configure",
Short: "Configure authentication",
Long: `Configure authentication.
This command adds a profile to your ~/.databrickscfg file.
You can write to a different file by setting the DATABRICKS_CONFIG_FILE environment variable.
This command adds a profile to your ~/.databrickscfg file.
You can write to a different file by setting the DATABRICKS_CONFIG_FILE environment variable.
If this command is invoked in non-interactive mode, it will read the token from stdin.
The host must be specified with the --host flag.
If this command is invoked in non-interactive mode, it will read the token from stdin.
The host must be specified with the --host flag.
`,
Hidden: true,
RunE: func(cmd *cobra.Command, args []string) error {
}
cmd.Flags().String("host", "", "Databricks workspace host.")
cmd.Flags().String("profile", "DEFAULT", "Name for the connection profile to configure.")
// Include token flag for compatibility with the legacy CLI.
// It doesn't actually do anything because we always use PATs.
cmd.Flags().BoolP("token", "t", true, "Configure using Databricks Personal Access Token")
cmd.Flags().MarkHidden("token")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
var cfg config.Config
// Load environment variables, possibly the DEFAULT profile.
@ -152,16 +162,11 @@ The host must be specified with the --host flag.
// Save profile to config file.
return databrickscfg.SaveToProfile(ctx, &cfg)
},
}
return cmd
}
func init() {
root.RootCmd.AddCommand(configureCmd)
configureCmd.Flags().String("host", "", "Databricks workspace host.")
configureCmd.Flags().String("profile", "DEFAULT", "Name for the connection profile to configure.")
// Include token flag for compatibility with the legacy CLI.
// It doesn't actually do anything because we always use PATs.
configureCmd.Flags().BoolP("token", "t", true, "Configure using Databricks Personal Access Token")
configureCmd.Flags().MarkHidden("token")
func New() *cobra.Command {
return newConfigureCommand()
}

19
cmd/configure/configure_test.go
View File

@ -1,4 +1,4 @@
package configure
package configure_test
import (
"context"
@ -7,7 +7,7 @@ import (
"runtime"
"testing"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/cmd"
"github.com/stretchr/testify/assert"
"gopkg.in/ini.v1"
)
@ -54,9 +54,10 @@ func TestDefaultConfigureNoInteractive(t *testing.T) {
})
os.Stdin = inp
root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "https://host"})
cmd := cmd.New()
cmd.SetArgs([]string{"configure", "--token", "--host", "https://host"})
err := root.RootCmd.ExecuteContext(ctx)
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
cfgPath := filepath.Join(tempHomeDir, ".databrickscfg")
@ -86,9 +87,10 @@ func TestConfigFileFromEnvNoInteractive(t *testing.T) {
t.Cleanup(func() { os.Stdin = oldStdin })
os.Stdin = inp
root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "https://host"})
cmd := cmd.New()
cmd.SetArgs([]string{"configure", "--token", "--host", "https://host"})
err := root.RootCmd.ExecuteContext(ctx)
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
_, err = os.Stat(cfgPath)
@ -114,9 +116,10 @@ func TestCustomProfileConfigureNoInteractive(t *testing.T) {
t.Cleanup(func() { os.Stdin = oldStdin })
os.Stdin = inp
root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "https://host", "--profile", "CUSTOM"})
cmd := cmd.New()
cmd.SetArgs([]string{"configure", "--token", "--host", "https://host", "--profile", "CUSTOM"})
err := root.RootCmd.ExecuteContext(ctx)
err := cmd.ExecuteContext(ctx)
assert.NoError(t, err)
_, err = os.Stat(cfgPath)

12
cmd/fs/cat.go
View File

@ -6,14 +6,16 @@ import (
"github.com/spf13/cobra"
)
var catCmd = &cobra.Command{
func newCatCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "cat FILE_PATH",
Short: "Show file content",
Long: `Show the contents of a file.`,
Args: cobra.ExactArgs(1),
PreRunE: root.MustWorkspaceClient,
}
RunE: func(cmd *cobra.Command, args []string) error {
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
f, path, err := filerForPath(ctx, args[0])
@ -26,9 +28,7 @@ var catCmd = &cobra.Command{
return err
}
return cmdio.RenderReader(ctx, r)
},
}
}
func init() {
fsCmd.AddCommand(catCmd)
return cmd
}

49
cmd/fs/cp.go
View File

@ -15,6 +15,9 @@ import (
)
type copy struct {
overwrite bool
recursive bool
ctx context.Context
sourceFiler filer.Filer
targetFiler filer.Filer
@ -48,7 +51,7 @@ func (c *copy) cpWriteCallback(sourceDir, targetDir string) fs.WalkDirFunc {
}
func (c *copy) cpDirToDir(sourceDir, targetDir string) error {
if !cpRecursive {
if !c.recursive {
return fmt.Errorf("source path %s is a directory. Please specify the --recursive flag", sourceDir)
}
@ -71,7 +74,7 @@ func (c *copy) cpFileToFile(sourcePath, targetPath string) error {
}
defer r.Close()
if cpOverwrite {
if c.overwrite {
err = c.targetFiler.Write(c.ctx, targetPath, r, filer.OverwriteIfExists)
if err != nil {
return err
@ -123,11 +126,8 @@ func (c *copy) emitFileCopiedEvent(sourcePath, targetPath string) error {
return cmdio.RenderWithTemplate(c.ctx, event, template)
}
var cpOverwrite bool
var cpRecursive bool
// cpCmd represents the fs cp command
var cpCmd = &cobra.Command{
func newCpCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "cp SOURCE_PATH TARGET_PATH",
Short: "Copy files and directories to and from DBFS.",
Long: `Copy files to and from DBFS.
@ -140,11 +140,16 @@ var cpCmd = &cobra.Command{
When copying a file, if TARGET_PATH is a directory, the file will be created
inside the directory, otherwise the file is created at TARGET_PATH.
`,
`,
Args: cobra.ExactArgs(2),
PreRunE: root.MustWorkspaceClient,
}
RunE: func(cmd *cobra.Command, args []string) error {
var c copy
cmd.Flags().BoolVar(&c.overwrite, "overwrite", false, "overwrite existing files")
cmd.Flags().BoolVarP(&c.recursive, "recursive", "r", false, "recursively copy files from directory")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// TODO: Error if a user uses '\' as path separator on windows when "file"
@ -164,22 +169,18 @@ var cpCmd = &cobra.Command{
return err
}
sourceScheme := ""
c.sourceScheme = ""
if isDbfsPath(fullSourcePath) {
sourceScheme = "dbfs"
c.sourceScheme = "dbfs"
}
targetScheme := ""
c.targetScheme = ""
if isDbfsPath(fullTargetPath) {
targetScheme = "dbfs"
c.targetScheme = "dbfs"
}
c := copy{
ctx: ctx,
sourceFiler: sourceFiler,
targetFiler: targetFiler,
sourceScheme: sourceScheme,
targetScheme: targetScheme,
}
c.ctx = ctx
c.sourceFiler = sourceFiler
c.targetFiler = targetFiler
// Get information about file at source path
sourceInfo, err := sourceFiler.Stat(ctx, sourcePath)
@ -200,11 +201,7 @@ var cpCmd = &cobra.Command{
// case 3: source path is a file, and target path is a file
return c.cpFileToFile(sourcePath, targetPath)
},
}
}
func init() {
cpCmd.Flags().BoolVar(&cpOverwrite, "overwrite", false, "overwrite existing files")
cpCmd.Flags().BoolVarP(&cpRecursive, "recursive", "r", false, "recursively copy files from directory")
fsCmd.AddCommand(cpCmd)
return cmd
}

18
cmd/fs/fs.go
View File

@ -1,17 +1,23 @@
package fs
import (
"github.com/databricks/cli/cmd/root"
"github.com/spf13/cobra"
)
// fsCmd represents the fs command
var fsCmd = &cobra.Command{
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "fs",
Short: "Filesystem related commands",
Long: `Commands to do DBFS operations.`,
}
}
func init() {
root.RootCmd.AddCommand(fsCmd)
cmd.AddCommand(
newCatCommand(),
newCpCommand(),
newLsCommand(),
newMkdirCommand(),
newRmCommand(),
)
return cmd
}

27
cmd/fs/ls.go
View File

@ -37,15 +37,21 @@ func toJsonDirEntry(f fs.DirEntry, baseDir string, isAbsolute bool) (*jsonDirEnt
}, nil
}
// lsCmd represents the ls command
var lsCmd = &cobra.Command{
func newLsCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "ls DIR_PATH",
Short: "Lists files",
Long: `Lists files`,
Args: cobra.ExactArgs(1),
PreRunE: root.MustWorkspaceClient,
}
RunE: func(cmd *cobra.Command, args []string) error {
var long bool
var absolute bool
cmd.Flags().BoolVarP(&long, "long", "l", false, "Displays full information including size, file type and modification time since Epoch in milliseconds.")
cmd.Flags().BoolVar(&absolute, "absolute", false, "Displays absolute paths.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
f, path, err := filerForPath(ctx, args[0])
@ -60,7 +66,7 @@ var lsCmd = &cobra.Command{
jsonDirEntries := make([]jsonDirEntry, len(entries))
for i, entry := range entries {
jsonDirEntry, err := toJsonDirEntry(entry, args[0], lsAbsolute)
jsonDirEntry, err := toJsonDirEntry(entry, args[0], absolute)
if err != nil {
return err
}
@ -71,7 +77,7 @@ var lsCmd = &cobra.Command{
})
// Use template for long mode if the flag is set
if longMode {
if long {
return cmdio.RenderWithTemplate(ctx, jsonDirEntries, cmdio.Heredoc(`
{{range .}}{{if .IsDir}}DIRECTORY {{else}}FILE {{end}}{{.Size}} {{.ModTime|pretty_date}} {{.Name}}
{{end}}
@ -81,14 +87,7 @@ var lsCmd = &cobra.Command{
{{range .}}{{.Name}}
{{end}}
`))
},
}
}
var longMode bool
var lsAbsolute bool
func init() {
lsCmd.Flags().BoolVarP(&longMode, "long", "l", false, "Displays full information including size, file type and modification time since Epoch in milliseconds.")
lsCmd.Flags().BoolVar(&lsAbsolute, "absolute", false, "Displays absolute paths.")
fsCmd.AddCommand(lsCmd)
return cmd
}

12
cmd/fs/mkdir.go
View File

@ -5,7 +5,8 @@ import (
"github.com/spf13/cobra"
)
var mkdirCmd = &cobra.Command{
func newMkdirCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "mkdir DIR_PATH",
// Alias `mkdirs` for this command exists for legacy purposes. This command
// is called databricks fs mkdirs in our legacy CLI: https://github.com/databricks/databricks-cli
@ -14,8 +15,9 @@ var mkdirCmd = &cobra.Command{
Long: `Mkdir will create directories along the path to the argument directory.`,
Args: cobra.ExactArgs(1),
PreRunE: root.MustWorkspaceClient,
}
RunE: func(cmd *cobra.Command, args []string) error {
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
f, path, err := filerForPath(ctx, args[0])
@ -24,9 +26,7 @@ var mkdirCmd = &cobra.Command{
}
return f.Mkdir(ctx, path)
},
}
}
func init() {
fsCmd.AddCommand(mkdirCmd)
return cmd
}

18
cmd/fs/rm.go
View File

@ -6,14 +6,19 @@ import (
"github.com/spf13/cobra"
)
var rmCmd = &cobra.Command{
func newRmCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "rm PATH",
Short: "Remove files and directories from dbfs.",
Long: `Remove files and directories from dbfs.`,
Args: cobra.ExactArgs(1),
PreRunE: root.MustWorkspaceClient,
}
RunE: func(cmd *cobra.Command, args []string) error {
var recursive bool
cmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively delete a non-empty directory.")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
f, path, err := filerForPath(ctx, args[0])
@ -25,12 +30,7 @@ var rmCmd = &cobra.Command{
return f.Delete(ctx, path, filer.DeleteRecursively)
}
return f.Delete(ctx, path)
},
}
}
var recursive bool
func init() {
rmCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Recursively delete a non-empty directory.")
fsCmd.AddCommand(rmCmd)
return cmd
}

3
cmd/root/root.go
View File

@ -115,6 +115,3 @@ func Execute(cmd *cobra.Command) {
os.Exit(1)
}
}
// Keep a global copy until all commands can be initialized.
var RootCmd = New()

61
cmd/sync/sync.go
View File

@ -17,7 +17,15 @@ import (
"github.com/spf13/cobra"
)
func syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
type syncFlags struct {
// project files polling interval
interval time.Duration
full bool
watch bool
output flags.Output
}
func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
if len(args) > 0 {
return nil, fmt.Errorf("SRC and DST are not configurable in the context of a bundle")
}
@ -30,8 +38,8 @@ func syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle)
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Full: full,
PollInterval: interval,
Full: f.full,
PollInterval: f.interval,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
@ -39,7 +47,7 @@ func syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle)
return &opts, nil
}
func syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions, error) {
func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions, error) {
if len(args) != 2 {
return nil, flag.ErrHelp
}
@ -47,8 +55,8 @@ func syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions,
opts := sync.SyncOptions{
LocalPath: args[0],
RemotePath: args[1],
Full: full,
PollInterval: interval,
Full: f.full,
PollInterval: f.interval,
// We keep existing behavior for VS Code extension where if there is
// no bundle defined, we store the snapshots in `.databricks`.
@ -60,13 +68,22 @@ func syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions,
return &opts, nil
}
var syncCmd = &cobra.Command{
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "sync [flags] SRC DST",
Short: "Synchronize a local directory to a workspace directory",
Args: cobra.MaximumNArgs(2),
}
// PreRunE: root.TryConfigureBundle,
RunE: func(cmd *cobra.Command, args []string) error {
f := syncFlags{
output: flags.OutputText,
}
cmd.Flags().DurationVar(&f.interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)")
cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes")
cmd.Flags().Var(&f.output, "output", "type of output format")
cmd.RunE = func(cmd *cobra.Command, args []string) error {
var opts *sync.SyncOptions
var err error
@ -84,7 +101,7 @@ var syncCmd = &cobra.Command{
// }
// opts, err = syncOptionsFromBundle(cmd, args, b)
// } else {
opts, err = syncOptionsFromArgs(cmd, args)
opts, err = f.syncOptionsFromArgs(cmd, args)
// }
if err != nil {
return err
@ -97,7 +114,7 @@ var syncCmd = &cobra.Command{
}
var outputFunc func(context.Context, <-chan sync.Event, io.Writer)
switch output {
switch f.output {
case flags.OutputText:
outputFunc = textOutput
case flags.OutputJSON:
@ -113,7 +130,7 @@ var syncCmd = &cobra.Command{
}()
}
if watch {
if f.watch {
err = s.RunContinuous(ctx)
} else {
err = s.RunOnce(ctx)
@ -122,9 +139,9 @@ var syncCmd = &cobra.Command{
s.Close()
wg.Wait()
return err
},
}
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
err := root.TryConfigureBundle(cmd, args)
if err != nil {
return nil, cobra.ShellCompDirectiveError
@ -149,19 +166,7 @@ var syncCmd = &cobra.Command{
default:
return nil, cobra.ShellCompDirectiveNoFileComp
}
},
}
}
// project files polling interval
var interval time.Duration
var full bool
var watch bool
var output flags.Output = flags.OutputText
func init() {
root.RootCmd.AddCommand(syncCmd)
syncCmd.Flags().DurationVar(&interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
syncCmd.Flags().BoolVar(&full, "full", false, "perform full synchronization (default is incremental)")
syncCmd.Flags().BoolVar(&watch, "watch", false, "watch local file system for changes")
syncCmd.Flags().Var(&output, "output", "type of output format")
return cmd
}

13
cmd/sync/sync_test.go
View File

@ -27,7 +27,8 @@ func TestSyncOptionsFromBundle(t *testing.T) {
},
}
opts, err := syncOptionsFromBundle(syncCmd, []string{}, b)
f := syncFlags{}
opts, err := f.syncOptionsFromBundle(New(), []string{}, b)
require.NoError(t, err)
assert.Equal(t, tempDir, opts.LocalPath)
assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath)
@ -37,16 +38,18 @@ func TestSyncOptionsFromBundle(t *testing.T) {
func TestSyncOptionsFromArgsRequiredTwoArgs(t *testing.T) {
var err error
_, err = syncOptionsFromArgs(syncCmd, []string{})
f := syncFlags{}
_, err = f.syncOptionsFromArgs(New(), []string{})
require.ErrorIs(t, err, flag.ErrHelp)
_, err = syncOptionsFromArgs(syncCmd, []string{"foo"})
_, err = f.syncOptionsFromArgs(New(), []string{"foo"})
require.ErrorIs(t, err, flag.ErrHelp)
_, err = syncOptionsFromArgs(syncCmd, []string{"foo", "bar", "qux"})
_, err = f.syncOptionsFromArgs(New(), []string{"foo", "bar", "qux"})
require.ErrorIs(t, err, flag.ErrHelp)
}
func TestSyncOptionsFromArgs(t *testing.T) {
opts, err := syncOptionsFromArgs(syncCmd, []string{"/local", "/remote"})
f := syncFlags{}
opts, err := f.syncOptionsFromArgs(New(), []string{"/local", "/remote"})
require.NoError(t, err)
assert.Equal(t, "/local", opts.LocalPath)
assert.Equal(t, "/remote", opts.RemotePath)

13
cmd/version/version.go
View File

@ -1,25 +1,24 @@
package version
import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/internal/build"
"github.com/databricks/cli/libs/cmdio"
"github.com/spf13/cobra"
)
var versionCmd = &cobra.Command{
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Args: cobra.NoArgs,
Annotations: map[string]string{
"template": "Databricks CLI v{{.Version}}\n",
},
}
RunE: func(cmd *cobra.Command, args []string) error {
cmd.RunE = func(cmd *cobra.Command, args []string) error {
return cmdio.Render(cmd.Context(), build.GetInfo())
},
}
}
func init() {
root.RootCmd.AddCommand(versionCmd)
return cmd
}

7
internal/secrets_test.go
View File

@ -77,13 +77,6 @@ func TestSecretsPutSecretStringValue(tt *testing.T) {
func TestSecretsPutSecretBytesValue(tt *testing.T) {
ctx, t := acc.WorkspaceTest(tt)
if true {
// Uncomment below to run this test in isolation.
// To be addressed once none of the commands taint global state.
t.Skip("skipping because the test above clobbers global state")
}
scope := temporarySecretScope(ctx, t)
key := "test-key"
value := []byte{0x00, 0x01, 0x02, 0x03}

10
main.go
View File

@ -2,17 +2,7 @@ package main
import (
"github.com/databricks/cli/cmd"
_ "github.com/databricks/cli/cmd/account"
_ "github.com/databricks/cli/cmd/api"
_ "github.com/databricks/cli/cmd/auth"
_ "github.com/databricks/cli/cmd/bundle"
_ "github.com/databricks/cli/cmd/bundle/debug"
_ "github.com/databricks/cli/cmd/configure"
_ "github.com/databricks/cli/cmd/fs"
"github.com/databricks/cli/cmd/root"
_ "github.com/databricks/cli/cmd/sync"
_ "github.com/databricks/cli/cmd/version"
_ "github.com/databricks/cli/cmd/workspace"
)
func main() {