Improve the output of the `databricks bundle init` command (#795)

Improve the output of help, prompts, and so on for `databricks bundle
init` and the default template.

Among other things, this PR adds support for a new `welcome_message`
property that lets a template print a custom message on success:

```
$ databricks bundle init
Template to use [default-python]:
Unique name for this project [my_project]: lennart_project
Include a stub (sample) notebook in 'lennart_project/src': yes
Include a stub (sample) Delta Live Tables pipeline in 'lennart_project/src': yes
Include a stub (sample) Python package in 'lennart_project/src': yes

 Your new project has been created in the 'lennart_project' directory!

Please refer to the README.md of your project for further instructions on getting started.
Or read the documentation on Databricks Asset Bundles at https://docs.databricks.com/dev-tools/bundles/index.html.
```

---------

Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
This commit is contained in:
Lennart Kats (databricks) 2023-10-19 09:08:36 +02:00 committed by GitHub
parent 5273d0c51a
commit a2ee8bb45b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 121 additions and 41 deletions

View File

@ -45,20 +45,29 @@ func repoName(url string) string {
func newInitCommand() *cobra.Command { func newInitCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "init [TEMPLATE_PATH]", Use: "init [TEMPLATE_PATH]",
Short: "Initialize Template", Short: "Initialize using a bundle template",
Args: cobra.MaximumNArgs(1), Args: cobra.MaximumNArgs(1),
Long: `Initialize using a bundle template.
TEMPLATE_PATH optionally specifies which template to use. It can be one of the following:
- 'default-python' for the default Python template
- a local file system path with a template directory
- a Git repository URL, e.g. https://github.com/my/repository
See https://docs.databricks.com//dev-tools/bundles/templates.html for more information on templates.`,
} }
var configFile string var configFile string
var outputDir string var outputDir string
var templateDir string var templateDir string
cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.")
cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory within repository that holds the template specification.") cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory path within a Git repository containing the template.")
cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.")
cmd.PreRunE = root.MustWorkspaceClient cmd.PreRunE = root.MustWorkspaceClient
cmd.RunE = func(cmd *cobra.Command, args []string) error { cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
var templatePath string var templatePath string
if len(args) > 0 { if len(args) > 0 {
templatePath = args[0] templatePath = args[0]
@ -79,6 +88,9 @@ func newInitCommand() *cobra.Command {
} }
if !isRepoUrl(templatePath) { if !isRepoUrl(templatePath) {
if templateDir != "" {
return errors.New("--template-dir can only be used with a Git repository URL")
}
// skip downloading the repo because input arg is not a URL. We assume // skip downloading the repo because input arg is not a URL. We assume
// it's a path on the local file system in that case // it's a path on the local file system in that case
return template.Materialize(ctx, configFile, templatePath, outputDir) return template.Materialize(ctx, configFile, templatePath, outputDir)

View File

@ -12,6 +12,9 @@ type Extension struct {
// that do have an order defined. // that do have an order defined.
Order *int `json:"order,omitempty"` Order *int `json:"order,omitempty"`
// The message to print after the template is successfully initalized
SuccessMessage string `json:"success_message,omitempty"`
// PatternMatchFailureMessage is a user defined message that is displayed to the // PatternMatchFailureMessage is a user defined message that is displayed to the
// user if a JSON schema pattern match fails. // user if a JSON schema pattern match fails.
PatternMatchFailureMessage string `json:"pattern_match_failure_message,omitempty"` PatternMatchFailureMessage string `json:"pattern_match_failure_message,omitempty"`

View File

@ -39,14 +39,17 @@ func (s *Schema) LoadInstance(path string) (map[string]any, error) {
return instance, s.ValidateInstance(instance) return instance, s.ValidateInstance(instance)
} }
// Validate an instance against the schema
func (s *Schema) ValidateInstance(instance map[string]any) error { func (s *Schema) ValidateInstance(instance map[string]any) error {
for _, fn := range []func(map[string]any) error{ validations := []func(map[string]any) error{
s.validateAdditionalProperties, s.validateAdditionalProperties,
s.validateEnum, s.validateEnum,
s.validateRequired, s.validateRequired,
s.validateTypes, s.validateTypes,
s.validatePattern, s.validatePattern,
} { }
for _, fn := range validations {
err := fn(instance) err := fn(instance)
if err != nil { if err != nil {
return err return err

View File

@ -65,7 +65,7 @@ func (c *config) assignValuesFromFile(path string) error {
} }
// Assigns default values from schema to input config map // Assigns default values from schema to input config map
func (c *config) assignDefaultValues() error { func (c *config) assignDefaultValues(r *renderer) error {
for name, property := range c.schema.Properties { for name, property := range c.schema.Properties {
// Config already has a value assigned // Config already has a value assigned
if _, ok := c.values[name]; ok { if _, ok := c.values[name]; ok {
@ -75,13 +75,25 @@ func (c *config) assignDefaultValues() error {
if property.Default == nil { if property.Default == nil {
continue continue
} }
c.values[name] = property.Default defaultVal, err := jsonschema.ToString(property.Default, property.Type)
if err != nil {
return err
}
defaultVal, err = r.executeTemplate(defaultVal)
if err != nil {
return err
}
defaultValTyped, err := jsonschema.FromString(defaultVal, property.Type)
if err != nil {
return err
}
c.values[name] = defaultValTyped
} }
return nil return nil
} }
// Prompts user for values for properties that do not have a value set yet // Prompts user for values for properties that do not have a value set yet
func (c *config) promptForValues() error { func (c *config) promptForValues(r *renderer) error {
for _, p := range c.schema.OrderedProperties() { for _, p := range c.schema.OrderedProperties() {
name := p.Name name := p.Name
property := p.Schema property := p.Schema
@ -95,10 +107,19 @@ func (c *config) promptForValues() error {
var defaultVal string var defaultVal string
var err error var err error
if property.Default != nil { if property.Default != nil {
defaultVal, err = jsonschema.ToString(property.Default, property.Type) defaultValRaw, err := jsonschema.ToString(property.Default, property.Type)
if err != nil { if err != nil {
return err return err
} }
defaultVal, err = r.executeTemplate(defaultValRaw)
if err != nil {
return err
}
}
description, err := r.executeTemplate(property.Description)
if err != nil {
return err
} }
// Get user input by running the prompt // Get user input by running the prompt
@ -109,21 +130,15 @@ func (c *config) promptForValues() error {
if err != nil { if err != nil {
return err return err
} }
userInput, err = cmdio.AskSelect(c.ctx, property.Description, enums) userInput, err = cmdio.AskSelect(c.ctx, description, enums)
if err != nil { if err != nil {
return err return err
} }
} else { } else {
userInput, err = cmdio.Ask(c.ctx, property.Description, defaultVal) userInput, err = cmdio.Ask(c.ctx, description, defaultVal)
if err != nil { if err != nil {
return err return err
} }
}
// Validate the property matches any specified regex pattern.
if err := jsonschema.ValidatePatternMatch(name, userInput, property); err != nil {
return err
} }
// Convert user input string back to a value // Convert user input string back to a value
@ -131,23 +146,28 @@ func (c *config) promptForValues() error {
if err != nil { if err != nil {
return err return err
} }
// Validate the partial config based on this update
if err := c.schema.ValidateInstance(c.values); err != nil {
return err
}
} }
return nil return nil
} }
// Prompt user for any missing config values. Assign default values if // Prompt user for any missing config values. Assign default values if
// terminal is not TTY // terminal is not TTY
func (c *config) promptOrAssignDefaultValues() error { func (c *config) promptOrAssignDefaultValues(r *renderer) error {
if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) { if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) {
return c.promptForValues() return c.promptForValues(r)
} }
return c.assignDefaultValues() return c.assignDefaultValues(r)
} }
// Validates the configuration. If passes, the configuration is ready to be used // Validates the configuration. If passes, the configuration is ready to be used
// to initialize the template. // to initialize the template.
func (c *config) validate() error { func (c *config) validate() error {
// All properties in the JSON schema should have a value defined. // For final validation, all properties in the JSON schema should have a value defined.
c.schema.Required = maps.Keys(c.schema.Properties) c.schema.Required = maps.Keys(c.schema.Properties)
if err := c.schema.ValidateInstance(c.values); err != nil { if err := c.schema.ValidateInstance(c.values); err != nil {
return fmt.Errorf("validation for template input parameters failed. %w", err) return fmt.Errorf("validation for template input parameters failed. %w", err)

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"testing" "testing"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/jsonschema" "github.com/databricks/cli/libs/jsonschema"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -52,11 +53,17 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te
func TestTemplateConfigAssignDefaultValues(t *testing.T) { func TestTemplateConfigAssignDefaultValues(t *testing.T) {
c := testConfig(t) c := testConfig(t)
err := c.assignDefaultValues() ctx := context.Background()
ctx = root.SetWorkspaceClient(ctx, nil)
helpers := loadHelpers(ctx)
r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", t.TempDir())
require.NoError(t, err)
err = c.assignDefaultValues(r)
assert.NoError(t, err) assert.NoError(t, err)
assert.Len(t, c.values, 2) assert.Len(t, c.values, 2)
assert.Equal(t, "abc", c.values["string_val"]) assert.Equal(t, "my_file", c.values["string_val"])
assert.Equal(t, int64(123), c.values["int_val"]) assert.Equal(t, int64(123), c.values["int_val"])
} }
@ -169,3 +176,16 @@ func TestTemplateEnumValidation(t *testing.T) {
} }
assert.NoError(t, c.validate()) assert.NoError(t, c.validate())
} }
func TestAssignDefaultValuesWithTemplatedDefaults(t *testing.T) {
c := testConfig(t)
ctx := context.Background()
ctx = root.SetWorkspaceClient(ctx, nil)
helpers := loadHelpers(ctx)
r, err := newRenderer(ctx, nil, helpers, "./testdata/templated-defaults/template", "./testdata/templated-defaults/library", t.TempDir())
require.NoError(t, err)
err = c.assignDefaultValues(r)
assert.NoError(t, err)
assert.Equal(t, "my_file", c.values["string_val"])
}

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/url" "net/url"
"os"
"regexp" "regexp"
"text/template" "text/template"
@ -65,7 +66,7 @@ func loadHelpers(ctx context.Context) template.FuncMap {
// Get smallest node type (follows Terraform's GetSmallestNodeType) // Get smallest node type (follows Terraform's GetSmallestNodeType)
"smallest_node_type": func() (string, error) { "smallest_node_type": func() (string, error) {
if w.Config.Host == "" { if w.Config.Host == "" {
return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks auth login'") return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'")
} }
if w.Config.IsAzure() { if w.Config.IsAzure() {
return "Standard_D3_v2", nil return "Standard_D3_v2", nil
@ -74,9 +75,12 @@ func loadHelpers(ctx context.Context) template.FuncMap {
} }
return "i3.xlarge", nil return "i3.xlarge", nil
}, },
"path_separator": func() string {
return string(os.PathSeparator)
},
"workspace_host": func() (string, error) { "workspace_host": func() (string, error) {
if w.Config.Host == "" { if w.Config.Host == "" {
return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks auth login'") return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'")
} }
return w.Config.Host, nil return w.Config.Host, nil
}, },

View File

@ -56,23 +56,23 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
} }
} }
// Prompt user for any missing config values. Assign default values if r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
// terminal is not TTY
err = config.promptOrAssignDefaultValues()
if err != nil { if err != nil {
return err return err
} }
// Prompt user for any missing config values. Assign default values if
// terminal is not TTY
err = config.promptOrAssignDefaultValues(r)
if err != nil {
return err
}
err = config.validate() err = config.validate()
if err != nil { if err != nil {
return err return err
} }
// Walk and render the template, since input configuration is complete // Walk and render the template, since input configuration is complete
r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir)
if err != nil {
return err
}
err = r.walk() err = r.walk()
if err != nil { if err != nil {
return err return err
@ -82,7 +82,17 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st
if err != nil { if err != nil {
return err return err
} }
cmdio.LogString(ctx, "✨ Successfully initialized template")
success := config.schema.SuccessMessage
if success == "" {
cmdio.LogString(ctx, "✨ Successfully initialized template")
} else {
success, err = r.executeTemplate(success)
if err != nil {
return err
}
cmdio.LogString(ctx, success)
}
return nil return nil
} }

View File

@ -311,7 +311,7 @@ func (r *renderer) persistToDisk() error {
path := file.DstPath().absPath() path := file.DstPath().absPath()
_, err := os.Stat(path) _, err := os.Stat(path)
if err == nil { if err == nil {
return fmt.Errorf("failed to persist to disk, conflict with existing file: %s", path) return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path)
} }
if err != nil && !os.IsNotExist(err) { if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err) return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err)

View File

@ -516,7 +516,7 @@ func TestRendererErrorOnConflictingFile(t *testing.T) {
}, },
} }
err = r.persistToDisk() err = r.persistToDisk()
assert.EqualError(t, err, fmt.Sprintf("failed to persist to disk, conflict with existing file: %s", filepath.Join(tmpDir, "a"))) assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", filepath.Join(tmpDir, "a")))
} }
func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) { func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) {

View File

@ -5,29 +5,30 @@
"default": "my_project", "default": "my_project",
"description": "Unique name for this project", "description": "Unique name for this project",
"order": 1, "order": 1,
"pattern": "^[A-Za-z0-9_]*$", "pattern": "^[A-Za-z0-9_]+$",
"pattern_match_failure_message": "Must consist of letter and underscores only." "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores."
}, },
"include_notebook": { "include_notebook": {
"type": "string", "type": "string",
"default": "yes", "default": "yes",
"enum": ["yes", "no"], "enum": ["yes", "no"],
"description": "Include a stub (sample) notebook in 'my_project/src'", "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'",
"order": 2 "order": 2
}, },
"include_dlt": { "include_dlt": {
"type": "string", "type": "string",
"default": "yes", "default": "yes",
"enum": ["yes", "no"], "enum": ["yes", "no"],
"description": "Include a stub (sample) DLT pipeline in 'my_project/src'", "description": "Include a stub (sample) Delta Live Tables pipeline in '{{.project_name}}{{path_separator}}src'",
"order": 3 "order": 3
}, },
"include_python": { "include_python": {
"type": "string", "type": "string",
"default": "yes", "default": "yes",
"enum": ["yes", "no"], "enum": ["yes", "no"],
"description": "Include a stub (sample) Python package in '{{.project_name}}/src'", "description": "Include a stub (sample) Python package in '{{.project_name}}{{path_separator}}src'",
"order": 4 "order": 4
} }
} },
"success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md of your project for further instructions on getting started.\nOr read the documentation on Databricks Asset Bundles at https://docs.databricks.com/dev-tools/bundles/index.html."
} }

View File

@ -12,7 +12,7 @@
}, },
"string_val": { "string_val": {
"type": "string", "type": "string",
"default": "abc" "default": "{{template \"file_name\"}}"
} }
} }
} }

View File

@ -0,0 +1,7 @@
{{define "dir_name" -}}
my_directory
{{- end}}
{{define "file_name" -}}
my_file
{{- end}}