From a2ee8bb45bf672b2b5f52005fc7f093a0ac929a6 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 19 Oct 2023 09:08:36 +0200 Subject: [PATCH] Improve the output of the `databricks bundle init` command (#795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve the output of help, prompts, and so on for `databricks bundle init` and the default template. Among other things, this PR adds support for a new `welcome_message` property that lets a template print a custom message on success: ``` $ databricks bundle init Template to use [default-python]: Unique name for this project [my_project]: lennart_project Include a stub (sample) notebook in 'lennart_project/src': yes Include a stub (sample) Delta Live Tables pipeline in 'lennart_project/src': yes Include a stub (sample) Python package in 'lennart_project/src': yes ✨ Your new project has been created in the 'lennart_project' directory! Please refer to the README.md of your project for further instructions on getting started. Or read the documentation on Databricks Asset Bundles at https://docs.databricks.com/dev-tools/bundles/index.html. ``` --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- cmd/bundle/init.go | 16 +++++- libs/jsonschema/extension.go | 3 ++ libs/jsonschema/instance.go | 7 ++- libs/template/config.go | 52 +++++++++++++------ libs/template/config_test.go | 24 ++++++++- libs/template/helpers.go | 8 ++- libs/template/materialize.go | 26 +++++++--- libs/template/renderer.go | 2 +- libs/template/renderer_test.go | 2 +- .../databricks_template_schema.json | 13 ++--- .../config-test-schema/test-schema.json | 2 +- .../templated-defaults/library/my_funcs.tmpl | 7 +++ .../{{template `file_name`}} | 0 13 files changed, 121 insertions(+), 41 deletions(-) create mode 100644 libs/template/testdata/templated-defaults/library/my_funcs.tmpl create mode 100644 libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index eec05687..603878be 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -45,20 +45,29 @@ func repoName(url string) string { func newInitCommand() *cobra.Command { cmd := &cobra.Command{ Use: "init [TEMPLATE_PATH]", - Short: "Initialize Template", + Short: "Initialize using a bundle template", Args: cobra.MaximumNArgs(1), + Long: `Initialize using a bundle template. + +TEMPLATE_PATH optionally specifies which template to use. It can be one of the following: +- 'default-python' for the default Python template +- a local file system path with a template directory +- a Git repository URL, e.g. https://github.com/my/repository + +See https://docs.databricks.com//dev-tools/bundles/templates.html for more information on templates.`, } var configFile string var outputDir string var templateDir string cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") - cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory within repository that holds the template specification.") + cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory path within a Git repository containing the template.") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.") cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + var templatePath string if len(args) > 0 { templatePath = args[0] @@ -79,6 +88,9 @@ func newInitCommand() *cobra.Command { } if !isRepoUrl(templatePath) { + if templateDir != "" { + return errors.New("--template-dir can only be used with a Git repository URL") + } // skip downloading the repo because input arg is not a URL. We assume // it's a path on the local file system in that case return template.Materialize(ctx, configFile, templatePath, outputDir) diff --git a/libs/jsonschema/extension.go b/libs/jsonschema/extension.go index 57f3e873..572c248a 100644 --- a/libs/jsonschema/extension.go +++ b/libs/jsonschema/extension.go @@ -12,6 +12,9 @@ type Extension struct { // that do have an order defined. Order *int `json:"order,omitempty"` + // The message to print after the template is successfully initalized + SuccessMessage string `json:"success_message,omitempty"` + // PatternMatchFailureMessage is a user defined message that is displayed to the // user if a JSON schema pattern match fails. PatternMatchFailureMessage string `json:"pattern_match_failure_message,omitempty"` diff --git a/libs/jsonschema/instance.go b/libs/jsonschema/instance.go index 6b3e3af4..d08ed519 100644 --- a/libs/jsonschema/instance.go +++ b/libs/jsonschema/instance.go @@ -39,14 +39,17 @@ func (s *Schema) LoadInstance(path string) (map[string]any, error) { return instance, s.ValidateInstance(instance) } +// Validate an instance against the schema func (s *Schema) ValidateInstance(instance map[string]any) error { - for _, fn := range []func(map[string]any) error{ + validations := []func(map[string]any) error{ s.validateAdditionalProperties, s.validateEnum, s.validateRequired, s.validateTypes, s.validatePattern, - } { + } + + for _, fn := range validations { err := fn(instance) if err != nil { return err diff --git a/libs/template/config.go b/libs/template/config.go index 2062f320..8ace307b 100644 --- a/libs/template/config.go +++ b/libs/template/config.go @@ -65,7 +65,7 @@ func (c *config) assignValuesFromFile(path string) error { } // Assigns default values from schema to input config map -func (c *config) assignDefaultValues() error { +func (c *config) assignDefaultValues(r *renderer) error { for name, property := range c.schema.Properties { // Config already has a value assigned if _, ok := c.values[name]; ok { @@ -75,13 +75,25 @@ func (c *config) assignDefaultValues() error { if property.Default == nil { continue } - c.values[name] = property.Default + defaultVal, err := jsonschema.ToString(property.Default, property.Type) + if err != nil { + return err + } + defaultVal, err = r.executeTemplate(defaultVal) + if err != nil { + return err + } + defaultValTyped, err := jsonschema.FromString(defaultVal, property.Type) + if err != nil { + return err + } + c.values[name] = defaultValTyped } return nil } // Prompts user for values for properties that do not have a value set yet -func (c *config) promptForValues() error { +func (c *config) promptForValues(r *renderer) error { for _, p := range c.schema.OrderedProperties() { name := p.Name property := p.Schema @@ -95,10 +107,19 @@ func (c *config) promptForValues() error { var defaultVal string var err error if property.Default != nil { - defaultVal, err = jsonschema.ToString(property.Default, property.Type) + defaultValRaw, err := jsonschema.ToString(property.Default, property.Type) if err != nil { return err } + defaultVal, err = r.executeTemplate(defaultValRaw) + if err != nil { + return err + } + } + + description, err := r.executeTemplate(property.Description) + if err != nil { + return err } // Get user input by running the prompt @@ -109,21 +130,15 @@ func (c *config) promptForValues() error { if err != nil { return err } - userInput, err = cmdio.AskSelect(c.ctx, property.Description, enums) + userInput, err = cmdio.AskSelect(c.ctx, description, enums) if err != nil { return err } } else { - userInput, err = cmdio.Ask(c.ctx, property.Description, defaultVal) + userInput, err = cmdio.Ask(c.ctx, description, defaultVal) if err != nil { return err } - - } - - // Validate the property matches any specified regex pattern. - if err := jsonschema.ValidatePatternMatch(name, userInput, property); err != nil { - return err } // Convert user input string back to a value @@ -131,23 +146,28 @@ func (c *config) promptForValues() error { if err != nil { return err } + + // Validate the partial config based on this update + if err := c.schema.ValidateInstance(c.values); err != nil { + return err + } } return nil } // Prompt user for any missing config values. Assign default values if // terminal is not TTY -func (c *config) promptOrAssignDefaultValues() error { +func (c *config) promptOrAssignDefaultValues(r *renderer) error { if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) { - return c.promptForValues() + return c.promptForValues(r) } - return c.assignDefaultValues() + return c.assignDefaultValues(r) } // Validates the configuration. If passes, the configuration is ready to be used // to initialize the template. func (c *config) validate() error { - // All properties in the JSON schema should have a value defined. + // For final validation, all properties in the JSON schema should have a value defined. c.schema.Required = maps.Keys(c.schema.Properties) if err := c.schema.ValidateInstance(c.values); err != nil { return fmt.Errorf("validation for template input parameters failed. %w", err) diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 1b1fc338..9a0a9931 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,11 +53,17 @@ func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *te func TestTemplateConfigAssignDefaultValues(t *testing.T) { c := testConfig(t) - err := c.assignDefaultValues() + ctx := context.Background() + ctx = root.SetWorkspaceClient(ctx, nil) + helpers := loadHelpers(ctx) + r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", t.TempDir()) + require.NoError(t, err) + + err = c.assignDefaultValues(r) assert.NoError(t, err) assert.Len(t, c.values, 2) - assert.Equal(t, "abc", c.values["string_val"]) + assert.Equal(t, "my_file", c.values["string_val"]) assert.Equal(t, int64(123), c.values["int_val"]) } @@ -169,3 +176,16 @@ func TestTemplateEnumValidation(t *testing.T) { } assert.NoError(t, c.validate()) } + +func TestAssignDefaultValuesWithTemplatedDefaults(t *testing.T) { + c := testConfig(t) + ctx := context.Background() + ctx = root.SetWorkspaceClient(ctx, nil) + helpers := loadHelpers(ctx) + r, err := newRenderer(ctx, nil, helpers, "./testdata/templated-defaults/template", "./testdata/templated-defaults/library", t.TempDir()) + require.NoError(t, err) + + err = c.assignDefaultValues(r) + assert.NoError(t, err) + assert.Equal(t, "my_file", c.values["string_val"]) +} diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 31752270..7f306a3a 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "os" "regexp" "text/template" @@ -65,7 +66,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { // Get smallest node type (follows Terraform's GetSmallestNodeType) "smallest_node_type": func() (string, error) { if w.Config.Host == "" { - return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks auth login'") + return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'") } if w.Config.IsAzure() { return "Standard_D3_v2", nil @@ -74,9 +75,12 @@ func loadHelpers(ctx context.Context) template.FuncMap { } return "i3.xlarge", nil }, + "path_separator": func() string { + return string(os.PathSeparator) + }, "workspace_host": func() (string, error) { if w.Config.Host == "" { - return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks auth login'") + return "", errors.New("cannot determine target workspace, please first setup a configuration profile using 'databricks configure'") } return w.Config.Host, nil }, diff --git a/libs/template/materialize.go b/libs/template/materialize.go index 8517858f..ec62e41f 100644 --- a/libs/template/materialize.go +++ b/libs/template/materialize.go @@ -56,23 +56,23 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st } } - // Prompt user for any missing config values. Assign default values if - // terminal is not TTY - err = config.promptOrAssignDefaultValues() + r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir) if err != nil { return err } + // Prompt user for any missing config values. Assign default values if + // terminal is not TTY + err = config.promptOrAssignDefaultValues(r) + if err != nil { + return err + } err = config.validate() if err != nil { return err } // Walk and render the template, since input configuration is complete - r, err := newRenderer(ctx, config.values, helpers, templatePath, libraryPath, outputDir) - if err != nil { - return err - } err = r.walk() if err != nil { return err @@ -82,7 +82,17 @@ func Materialize(ctx context.Context, configFilePath, templateRoot, outputDir st if err != nil { return err } - cmdio.LogString(ctx, "✨ Successfully initialized template") + + success := config.schema.SuccessMessage + if success == "" { + cmdio.LogString(ctx, "✨ Successfully initialized template") + } else { + success, err = r.executeTemplate(success) + if err != nil { + return err + } + cmdio.LogString(ctx, success) + } return nil } diff --git a/libs/template/renderer.go b/libs/template/renderer.go index 09ccc3f5..6415cd84 100644 --- a/libs/template/renderer.go +++ b/libs/template/renderer.go @@ -311,7 +311,7 @@ func (r *renderer) persistToDisk() error { path := file.DstPath().absPath() _, err := os.Stat(path) if err == nil { - return fmt.Errorf("failed to persist to disk, conflict with existing file: %s", path) + return fmt.Errorf("failed to initialize template, one or more files already exist: %s", path) } if err != nil && !os.IsNotExist(err) { return fmt.Errorf("error while verifying file %s does not already exist: %w", path, err) diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 254b06cf..d513eac8 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -516,7 +516,7 @@ func TestRendererErrorOnConflictingFile(t *testing.T) { }, } err = r.persistToDisk() - assert.EqualError(t, err, fmt.Sprintf("failed to persist to disk, conflict with existing file: %s", filepath.Join(tmpDir, "a"))) + assert.EqualError(t, err, fmt.Sprintf("failed to initialize template, one or more files already exist: %s", filepath.Join(tmpDir, "a"))) } func TestRendererNoErrorOnConflictingFileIfSkipped(t *testing.T) { diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json index 59697a61..8b26ee70 100644 --- a/libs/template/templates/default-python/databricks_template_schema.json +++ b/libs/template/templates/default-python/databricks_template_schema.json @@ -5,29 +5,30 @@ "default": "my_project", "description": "Unique name for this project", "order": 1, - "pattern": "^[A-Za-z0-9_]*$", - "pattern_match_failure_message": "Must consist of letter and underscores only." + "pattern": "^[A-Za-z0-9_]+$", + "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." }, "include_notebook": { "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) notebook in 'my_project/src'", + "description": "Include a stub (sample) notebook in '{{.project_name}}{{path_separator}}src'", "order": 2 }, "include_dlt": { "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) DLT pipeline in 'my_project/src'", + "description": "Include a stub (sample) Delta Live Tables pipeline in '{{.project_name}}{{path_separator}}src'", "order": 3 }, "include_python": { "type": "string", "default": "yes", "enum": ["yes", "no"], - "description": "Include a stub (sample) Python package in '{{.project_name}}/src'", + "description": "Include a stub (sample) Python package in '{{.project_name}}{{path_separator}}src'", "order": 4 } - } + }, + "success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md of your project for further instructions on getting started.\nOr read the documentation on Databricks Asset Bundles at https://docs.databricks.com/dev-tools/bundles/index.html." } diff --git a/libs/template/testdata/config-test-schema/test-schema.json b/libs/template/testdata/config-test-schema/test-schema.json index 41eb8251..6daf4959 100644 --- a/libs/template/testdata/config-test-schema/test-schema.json +++ b/libs/template/testdata/config-test-schema/test-schema.json @@ -12,7 +12,7 @@ }, "string_val": { "type": "string", - "default": "abc" + "default": "{{template \"file_name\"}}" } } } diff --git a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl b/libs/template/testdata/templated-defaults/library/my_funcs.tmpl new file mode 100644 index 00000000..3415ad77 --- /dev/null +++ b/libs/template/testdata/templated-defaults/library/my_funcs.tmpl @@ -0,0 +1,7 @@ +{{define "dir_name" -}} +my_directory +{{- end}} + +{{define "file_name" -}} +my_file +{{- end}} diff --git a/libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} b/libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} new file mode 100644 index 00000000..e69de29b