diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 8d1216f8..c933ec9c 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -19,5 +19,6 @@ func New() *cobra.Command { cmd.AddCommand(newSyncCommand()) cmd.AddCommand(newTestCommand()) cmd.AddCommand(newValidateCommand()) + cmd.AddCommand(newInitCommand()) return cmd } diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go new file mode 100644 index 00000000..e3d76ecf --- /dev/null +++ b/cmd/bundle/init.go @@ -0,0 +1,79 @@ +package bundle + +import ( + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/git" + "github.com/databricks/cli/libs/template" + "github.com/spf13/cobra" +) + +var gitUrlPrefixes = []string{ + "https://", + "git@", +} + +func isRepoUrl(url string) bool { + result := false + for _, prefix := range gitUrlPrefixes { + if strings.HasPrefix(url, prefix) { + result = true + break + } + } + return result +} + +// Computes the repo name from the repo URL. Treats the last non empty word +// when splitting at '/' as the repo name. For example: for url git@github.com:databricks/cli.git +// the name would be "cli.git" +func repoName(url string) string { + parts := strings.Split(strings.TrimRight(url, "/"), "/") + return parts[len(parts)-1] +} + +func newInitCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "init TEMPLATE_PATH", + Short: "Initialize Template", + Args: cobra.ExactArgs(1), + } + + var configFile string + var projectDir string + cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") + cmd.Flags().StringVar(&projectDir, "project-dir", "", "The project will be initialized in this directory.") + cmd.MarkFlagRequired("project-dir") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + templatePath := args[0] + ctx := cmd.Context() + + if !isRepoUrl(templatePath) { + // skip downloading the repo because input arg is not a URL. We assume + // it's a path on the local file system in that case + return template.Materialize(ctx, configFile, templatePath, projectDir) + } + + // Download the template in a temporary directory + tmpDir := os.TempDir() + templateURL := templatePath + templateDir := filepath.Join(tmpDir, repoName(templateURL)) + err := os.MkdirAll(templateDir, 0755) + if err != nil { + return err + } + // TODO: Add automated test that the downloaded git repo is cleaned up. + err = git.Clone(ctx, templateURL, "", templateDir) + if err != nil { + return err + } + defer os.RemoveAll(templateDir) + + return template.Materialize(ctx, configFile, templateDir, projectDir) + } + + return cmd +} diff --git a/cmd/bundle/init_test.go b/cmd/bundle/init_test.go new file mode 100644 index 00000000..4a795160 --- /dev/null +++ b/cmd/bundle/init_test.go @@ -0,0 +1,27 @@ +package bundle + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBundleInitIsRepoUrl(t *testing.T) { + assert.True(t, isRepoUrl("git@github.com:databricks/cli.git")) + assert.True(t, isRepoUrl("https://github.com/databricks/cli.git")) + + assert.False(t, isRepoUrl("./local")) + assert.False(t, isRepoUrl("foo")) +} + +func TestBundleInitRepoName(t *testing.T) { + // Test valid URLs + assert.Equal(t, "cli.git", repoName("git@github.com:databricks/cli.git")) + assert.Equal(t, "cli", repoName("https://github.com/databricks/cli/")) + + // test invalid URLs. In these cases the error would be floated when the + // git clone operation fails. + assert.Equal(t, "git@github.com:databricks", repoName("git@github.com:databricks")) + assert.Equal(t, "invalid-url", repoName("invalid-url")) + assert.Equal(t, "www.github.com", repoName("https://www.github.com")) +} diff --git a/libs/template/config.go b/libs/template/config.go new file mode 100644 index 00000000..ee5fcbef --- /dev/null +++ b/libs/template/config.go @@ -0,0 +1,198 @@ +package template + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/jsonschema" +) + +type config struct { + ctx context.Context + values map[string]any + schema *jsonschema.Schema +} + +func newConfig(ctx context.Context, schemaPath string) (*config, error) { + // Read config schema + schemaBytes, err := os.ReadFile(schemaPath) + if err != nil { + return nil, err + } + schema := &jsonschema.Schema{} + err = json.Unmarshal(schemaBytes, schema) + if err != nil { + return nil, err + } + + // Return config + return &config{ + ctx: ctx, + schema: schema, + values: make(map[string]any, 0), + }, nil +} + +// Reads json file at path and assigns values from the file +func (c *config) assignValuesFromFile(path string) error { + // Read the config file + configFromFile := make(map[string]any, 0) + b, err := os.ReadFile(path) + if err != nil { + return err + } + err = json.Unmarshal(b, &configFromFile) + if err != nil { + return err + } + + // Cast any integer properties, from float to integer. Required because + // the json unmarshaller treats all json numbers as floating point + for name, floatVal := range configFromFile { + property, ok := c.schema.Properties[name] + if !ok { + return fmt.Errorf("%s is not defined as an input parameter for the template", name) + } + if property.Type != jsonschema.IntegerType { + continue + } + v, err := toInteger(floatVal) + if err != nil { + return fmt.Errorf("failed to cast value %v of property %s from file %s to an integer: %w", floatVal, name, path, err) + } + configFromFile[name] = v + } + + // Write configs from the file to the input map, not overwriting any existing + // configurations. + for name, val := range configFromFile { + if _, ok := c.values[name]; ok { + continue + } + c.values[name] = val + } + return nil +} + +// Assigns default values from schema to input config map +func (c *config) assignDefaultValues() error { + for name, property := range c.schema.Properties { + // Config already has a value assigned + if _, ok := c.values[name]; ok { + continue + } + + // No default value defined for the property + if property.Default == nil { + continue + } + + // Assign default value if property is not an integer + if property.Type != jsonschema.IntegerType { + c.values[name] = property.Default + continue + } + + // Cast default value to int before assigning to an integer configuration. + // Required because untyped field Default will read all numbers as floats + // during unmarshalling + v, err := toInteger(property.Default) + if err != nil { + return fmt.Errorf("failed to cast default value %v of property %s to an integer: %w", property.Default, name, err) + } + c.values[name] = v + } + return nil +} + +// Prompts user for values for properties that do not have a value set yet +func (c *config) promptForValues() error { + for name, property := range c.schema.Properties { + // Config already has a value assigned + if _, ok := c.values[name]; ok { + continue + } + + // Initialize Prompt dialog + var err error + prompt := cmdio.Prompt(c.ctx) + prompt.Label = property.Description + prompt.AllowEdit = true + + // Compute default value to display by converting it to a string + if property.Default != nil { + prompt.Default, err = toString(property.Default, property.Type) + if err != nil { + return err + } + } + + // Get user input by running the prompt + userInput, err := prompt.Run() + if err != nil { + return err + } + + // Convert user input string back to a value + c.values[name], err = fromString(userInput, property.Type) + if err != nil { + return err + } + } + return nil +} + +// Prompt user for any missing config values. Assign default values if +// terminal is not TTY +func (c *config) promptOrAssignDefaultValues() error { + if cmdio.IsOutTTY(c.ctx) && cmdio.IsInTTY(c.ctx) { + return c.promptForValues() + } + return c.assignDefaultValues() +} + +// Validates the configuration. If passes, the configuration is ready to be used +// to initialize the template. +func (c *config) validate() error { + validateFns := []func() error{ + c.validateValuesDefined, + c.validateValuesType, + } + + for _, fn := range validateFns { + err := fn() + if err != nil { + return err + } + } + return nil +} + +// Validates all input properties have a user defined value assigned to them +func (c *config) validateValuesDefined() error { + for k := range c.schema.Properties { + if _, ok := c.values[k]; ok { + continue + } + return fmt.Errorf("no value has been assigned to input parameter %s", k) + } + return nil +} + +// Validates the types of all input properties values match their types defined in the schema +func (c *config) validateValuesType() error { + for k, v := range c.values { + fieldInfo, ok := c.schema.Properties[k] + if !ok { + return fmt.Errorf("%s is not defined as an input parameter for the template", k) + } + err := validateType(v, fieldInfo.Type) + if err != nil { + return fmt.Errorf("incorrect type for %s. %w", k, err) + } + } + return nil +} diff --git a/libs/template/config_test.go b/libs/template/config_test.go new file mode 100644 index 00000000..7b8341ec --- /dev/null +++ b/libs/template/config_test.go @@ -0,0 +1,163 @@ +package template + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/libs/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testSchema(t *testing.T) *jsonschema.Schema { + schemaJson := `{ + "properties": { + "int_val": { + "type": "integer", + "default": 123 + }, + "float_val": { + "type": "number" + }, + "bool_val": { + "type": "boolean" + }, + "string_val": { + "type": "string", + "default": "abc" + } + } + }` + var jsonSchema jsonschema.Schema + err := json.Unmarshal([]byte(schemaJson), &jsonSchema) + require.NoError(t, err) + return &jsonSchema +} + +func TestTemplateConfigAssignValuesFromFile(t *testing.T) { + c := config{ + schema: testSchema(t), + values: make(map[string]any), + } + + err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") + assert.NoError(t, err) + + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) +} + +func TestTemplateConfigAssignValuesFromFileForUnknownField(t *testing.T) { + c := config{ + schema: testSchema(t), + values: make(map[string]any), + } + + err := c.assignValuesFromFile("./testdata/config-assign-from-file-unknown-property/config.json") + assert.EqualError(t, err, "unknown_prop is not defined as an input parameter for the template") +} + +func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { + c := config{ + schema: testSchema(t), + values: make(map[string]any), + } + + err := c.assignValuesFromFile("./testdata/config-assign-from-file-invalid-int/config.json") + assert.EqualError(t, err, "failed to cast value abc of property int_val from file ./testdata/config-assign-from-file-invalid-int/config.json to an integer: cannot convert \"abc\" to an integer") +} + +func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { + c := config{ + schema: testSchema(t), + values: map[string]any{ + "string_val": "this-is-not-overwritten", + }, + } + + err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") + assert.NoError(t, err) + + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) +} + +func TestTemplateConfigAssignDefaultValues(t *testing.T) { + c := config{ + schema: testSchema(t), + values: make(map[string]any), + } + + err := c.assignDefaultValues() + assert.NoError(t, err) + + assert.Len(t, c.values, 2) + assert.Equal(t, "abc", c.values["string_val"]) + assert.Equal(t, int64(123), c.values["int_val"]) +} + +func TestTemplateConfigValidateValuesDefined(t *testing.T) { + c := config{ + schema: testSchema(t), + values: map[string]any{ + "int_val": 1, + "float_val": 1.0, + "bool_val": false, + }, + } + + err := c.validateValuesDefined() + assert.EqualError(t, err, "no value has been assigned to input parameter string_val") +} + +func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { + c := &config{ + schema: testSchema(t), + values: map[string]any{ + "int_val": 1, + "float_val": 1.1, + "bool_val": true, + "string_val": "abcd", + }, + } + + err := c.validateValuesType() + assert.NoError(t, err) + + err = c.validate() + assert.NoError(t, err) +} + +func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { + c := &config{ + schema: testSchema(t), + values: map[string]any{ + "unknown_prop": 1, + }, + } + + err := c.validateValuesType() + assert.EqualError(t, err, "unknown_prop is not defined as an input parameter for the template") +} + +func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { + c := &config{ + schema: testSchema(t), + values: map[string]any{ + "int_val": "this-should-be-an-int", + "float_val": 1.1, + "bool_val": true, + "string_val": "abcd", + }, + } + + err := c.validateValuesType() + assert.EqualError(t, err, `incorrect type for int_val. expected type integer, but value is "this-should-be-an-int"`) + + err = c.validate() + assert.EqualError(t, err, `incorrect type for int_val. expected type integer, but value is "this-should-be-an-int"`) +} diff --git a/libs/template/materialize.go b/libs/template/materialize.go new file mode 100644 index 00000000..bbc9e8da --- /dev/null +++ b/libs/template/materialize.go @@ -0,0 +1,60 @@ +package template + +import ( + "context" + "path/filepath" +) + +const libraryDirName = "library" +const templateDirName = "template" +const schemaFileName = "databricks_template_schema.json" + +// This function materializes the input templates as a project, using user defined +// configurations. +// Parameters: +// +// ctx: context containing a cmdio object. This is used to prompt the user +// configFilePath: file path containing user defined config values +// templateRoot: root of the template definition +// projectDir: root of directory where to initialize the project +func Materialize(ctx context.Context, configFilePath, templateRoot, projectDir string) error { + templatePath := filepath.Join(templateRoot, templateDirName) + libraryPath := filepath.Join(templateRoot, libraryDirName) + schemaPath := filepath.Join(templateRoot, schemaFileName) + + config, err := newConfig(ctx, schemaPath) + if err != nil { + return err + } + + // Read and assign config values from file + if configFilePath != "" { + err = config.assignValuesFromFile(configFilePath) + if err != nil { + return err + } + } + + // Prompt user for any missing config values. Assign default values if + // terminal is not TTY + err = config.promptOrAssignDefaultValues() + if err != nil { + return err + } + + err = config.validate() + if err != nil { + return err + } + + // Walk and render the template, since input configuration is complete + r, err := newRenderer(ctx, config.values, templatePath, libraryPath, projectDir) + if err != nil { + return err + } + err = r.walk() + if err != nil { + return err + } + return r.persistToDisk() +} diff --git a/libs/template/schema.go b/libs/template/schema.go deleted file mode 100644 index 957cd66c..00000000 --- a/libs/template/schema.go +++ /dev/null @@ -1,121 +0,0 @@ -package template - -import ( - "encoding/json" - "fmt" - "os" - - "github.com/databricks/cli/libs/jsonschema" -) - -// function to check whether a float value represents an integer -func isIntegerValue(v float64) bool { - return v == float64(int(v)) -} - -// cast value to integer for config values that are floats but are supposed to be -// integers according to the schema -// -// Needed because the default json unmarshaler for maps converts all numbers to floats -func castFloatConfigValuesToInt(config map[string]any, jsonSchema *jsonschema.Schema) error { - for k, v := range config { - // error because all config keys should be defined in schema too - fieldInfo, ok := jsonSchema.Properties[k] - if !ok { - return fmt.Errorf("%s is not defined as an input parameter for the template", k) - } - // skip non integer fields - if fieldInfo.Type != jsonschema.IntegerType { - continue - } - - // convert floating point type values to integer - switch floatVal := v.(type) { - case float32: - if !isIntegerValue(float64(floatVal)) { - return fmt.Errorf("expected %s to have integer value but it is %v", k, v) - } - config[k] = int(floatVal) - case float64: - if !isIntegerValue(floatVal) { - return fmt.Errorf("expected %s to have integer value but it is %v", k, v) - } - config[k] = int(floatVal) - } - } - return nil -} - -func assignDefaultConfigValues(config map[string]any, schema *jsonschema.Schema) error { - for k, v := range schema.Properties { - if _, ok := config[k]; ok { - continue - } - if v.Default == nil { - return fmt.Errorf("input parameter %s is not defined in config", k) - } - config[k] = v.Default - } - return nil -} - -func validateConfigValueTypes(config map[string]any, schema *jsonschema.Schema) error { - // validate types defined in config - for k, v := range config { - fieldInfo, ok := schema.Properties[k] - if !ok { - return fmt.Errorf("%s is not defined as an input parameter for the template", k) - } - err := validateType(v, fieldInfo.Type) - if err != nil { - return fmt.Errorf("incorrect type for %s. %w", k, err) - } - } - return nil -} - -func ReadSchema(path string) (*jsonschema.Schema, error) { - schemaBytes, err := os.ReadFile(path) - if err != nil { - return nil, err - } - schema := &jsonschema.Schema{} - err = json.Unmarshal(schemaBytes, schema) - if err != nil { - return nil, err - } - return schema, nil -} - -func ReadConfig(path string, jsonSchema *jsonschema.Schema) (map[string]any, error) { - // Read config file - var config map[string]any - b, err := os.ReadFile(path) - if err != nil { - return nil, err - } - err = json.Unmarshal(b, &config) - if err != nil { - return nil, err - } - - // Assign default value to any fields that do not have a value yet - err = assignDefaultConfigValues(config, jsonSchema) - if err != nil { - return nil, err - } - - // cast any fields that are supposed to be integers. The json unmarshalling - // for a generic map converts all numbers to floating point - err = castFloatConfigValuesToInt(config, jsonSchema) - if err != nil { - return nil, err - } - - // validate config according to schema - err = validateConfigValueTypes(config, jsonSchema) - if err != nil { - return nil, err - } - return config, nil -} diff --git a/libs/template/schema_test.go b/libs/template/schema_test.go deleted file mode 100644 index ba30f81a..00000000 --- a/libs/template/schema_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package template - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/libs/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func testSchema(t *testing.T) *jsonschema.Schema { - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer" - }, - "float_val": { - "type": "number" - }, - "bool_val": { - "type": "boolean" - }, - "string_val": { - "type": "string" - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) - require.NoError(t, err) - return &jsonSchema -} - -func TestTemplateSchemaIsInteger(t *testing.T) { - assert.False(t, isIntegerValue(1.1)) - assert.False(t, isIntegerValue(0.1)) - assert.False(t, isIntegerValue(-0.1)) - - assert.True(t, isIntegerValue(-1.0)) - assert.True(t, isIntegerValue(0.0)) - assert.True(t, isIntegerValue(2.0)) -} - -func TestTemplateSchemaCastFloatToInt(t *testing.T) { - // define schema for config - jsonSchema := testSchema(t) - - // define the config - configJson := `{ - "int_val": 1, - "float_val": 2, - "bool_val": true, - "string_val": "main hoon na" - }` - var config map[string]any - err := json.Unmarshal([]byte(configJson), &config) - require.NoError(t, err) - - // assert types before casting, checking that the integer was indeed loaded - // as a floating point - assert.IsType(t, float64(0), config["int_val"]) - assert.IsType(t, float64(0), config["float_val"]) - assert.IsType(t, true, config["bool_val"]) - assert.IsType(t, "abc", config["string_val"]) - - err = castFloatConfigValuesToInt(config, jsonSchema) - require.NoError(t, err) - - // assert type after casting, that the float value was converted to an integer - // for int_val. - assert.IsType(t, int(0), config["int_val"]) - assert.IsType(t, float64(0), config["float_val"]) - assert.IsType(t, true, config["bool_val"]) - assert.IsType(t, "abc", config["string_val"]) -} - -func TestTemplateSchemaCastFloatToIntFailsForUnknownTypes(t *testing.T) { - // define schema for config - schemaJson := `{ - "properties": { - "foo": { - "type": "integer" - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) - require.NoError(t, err) - - // define the config - configJson := `{ - "bar": true - }` - var config map[string]any - err = json.Unmarshal([]byte(configJson), &config) - require.NoError(t, err) - - err = castFloatConfigValuesToInt(config, &jsonSchema) - assert.ErrorContains(t, err, "bar is not defined as an input parameter for the template") -} - -func TestTemplateSchemaCastFloatToIntFailsWhenWithNonIntValues(t *testing.T) { - // define schema for config - schemaJson := `{ - "properties": { - "foo": { - "type": "integer" - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) - require.NoError(t, err) - - // define the config - configJson := `{ - "foo": 1.1 - }` - var config map[string]any - err = json.Unmarshal([]byte(configJson), &config) - require.NoError(t, err) - - err = castFloatConfigValuesToInt(config, &jsonSchema) - assert.ErrorContains(t, err, "expected foo to have integer value but it is 1.1") -} - -func TestTemplateSchemaValidateType(t *testing.T) { - // assert validation passing - err := validateType(int(0), jsonschema.IntegerType) - assert.NoError(t, err) - err = validateType(int32(1), jsonschema.IntegerType) - assert.NoError(t, err) - err = validateType(int64(1), jsonschema.IntegerType) - assert.NoError(t, err) - - err = validateType(float32(1.1), jsonschema.NumberType) - assert.NoError(t, err) - err = validateType(float64(1.2), jsonschema.NumberType) - assert.NoError(t, err) - err = validateType(int(1), jsonschema.NumberType) - assert.NoError(t, err) - - err = validateType(false, jsonschema.BooleanType) - assert.NoError(t, err) - - err = validateType("abc", jsonschema.StringType) - assert.NoError(t, err) - - // assert validation failing for integers - err = validateType(float64(1.2), jsonschema.IntegerType) - assert.ErrorContains(t, err, "expected type integer, but value is 1.2") - err = validateType(true, jsonschema.IntegerType) - assert.ErrorContains(t, err, "expected type integer, but value is true") - err = validateType("abc", jsonschema.IntegerType) - assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") - - // assert validation failing for floats - err = validateType(true, jsonschema.NumberType) - assert.ErrorContains(t, err, "expected type float, but value is true") - err = validateType("abc", jsonschema.NumberType) - assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") - - // assert validation failing for boolean - err = validateType(int(1), jsonschema.BooleanType) - assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType(float64(1), jsonschema.BooleanType) - assert.ErrorContains(t, err, "expected type boolean, but value is 1") - err = validateType("abc", jsonschema.BooleanType) - assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") - - // assert validation failing for string - err = validateType(int(1), jsonschema.StringType) - assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(float64(1), jsonschema.StringType) - assert.ErrorContains(t, err, "expected type string, but value is 1") - err = validateType(false, jsonschema.StringType) - assert.ErrorContains(t, err, "expected type string, but value is false") -} - -func TestTemplateSchemaValidateConfig(t *testing.T) { - // define schema for config - jsonSchema := testSchema(t) - - // define the config - config := map[string]any{ - "int_val": 1, - "float_val": 1.1, - "bool_val": true, - "string_val": "abc", - } - - err := validateConfigValueTypes(config, jsonSchema) - assert.NoError(t, err) -} - -func TestTemplateSchemaValidateConfigFailsForUnknownField(t *testing.T) { - // define schema for config - jsonSchema := testSchema(t) - - // define the config - config := map[string]any{ - "foo": 1, - "float_val": 1.1, - "bool_val": true, - "string_val": "abc", - } - - err := validateConfigValueTypes(config, jsonSchema) - assert.ErrorContains(t, err, "foo is not defined as an input parameter for the template") -} - -func TestTemplateSchemaValidateConfigFailsForWhenIncorrectTypes(t *testing.T) { - // define schema for config - jsonSchema := testSchema(t) - - // define the config - config := map[string]any{ - "int_val": 1, - "float_val": 1.1, - "bool_val": "true", - "string_val": "abc", - } - - err := validateConfigValueTypes(config, jsonSchema) - assert.ErrorContains(t, err, "incorrect type for bool_val. expected type boolean, but value is \"true\"") -} - -func TestTemplateSchemaValidateConfigFailsForWhenMissingInputParams(t *testing.T) { - // define schema for config - schemaJson := `{ - "properties": { - "int_val": { - "type": "integer" - }, - "string_val": { - "type": "string" - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) - require.NoError(t, err) - - // define the config - config := map[string]any{ - "int_val": 1, - } - - err = assignDefaultConfigValues(config, &jsonSchema) - assert.ErrorContains(t, err, "input parameter string_val is not defined in config") -} - -func TestTemplateDefaultAssignment(t *testing.T) { - // define schema for config - schemaJson := `{ - "properties": { - "foo": { - "type": "integer", - "default": 1 - } - } - }` - var jsonSchema jsonschema.Schema - err := json.Unmarshal([]byte(schemaJson), &jsonSchema) - require.NoError(t, err) - - // define the config - config := map[string]any{} - - err = assignDefaultConfigValues(config, &jsonSchema) - assert.NoError(t, err) - assert.Equal(t, 1.0, config["foo"]) -} diff --git a/libs/template/testdata/config-assign-from-file-invalid-int/config.json b/libs/template/testdata/config-assign-from-file-invalid-int/config.json new file mode 100644 index 00000000..a97bf0c2 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-invalid-int/config.json @@ -0,0 +1,6 @@ +{ + "int_val": "abc", + "float_val": 2, + "bool_val": true, + "string_val": "hello" +} diff --git a/libs/template/testdata/config-assign-from-file-unknown-property/config.json b/libs/template/testdata/config-assign-from-file-unknown-property/config.json new file mode 100644 index 00000000..518eaa6a --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-unknown-property/config.json @@ -0,0 +1,3 @@ +{ + "unknown_prop": 123 +} diff --git a/libs/template/testdata/config-assign-from-file/config.json b/libs/template/testdata/config-assign-from-file/config.json new file mode 100644 index 00000000..564001e5 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file/config.json @@ -0,0 +1,6 @@ +{ + "int_val": 1, + "float_val": 2, + "bool_val": true, + "string_val": "hello" +} diff --git a/libs/template/utils.go b/libs/template/utils.go new file mode 100644 index 00000000..bf11ed86 --- /dev/null +++ b/libs/template/utils.go @@ -0,0 +1,99 @@ +package template + +import ( + "errors" + "fmt" + "strconv" + + "github.com/databricks/cli/libs/jsonschema" +) + +// function to check whether a float value represents an integer +func isIntegerValue(v float64) bool { + return v == float64(int64(v)) +} + +func toInteger(v any) (int64, error) { + switch typedVal := v.(type) { + // cast float to int + case float32: + if !isIntegerValue(float64(typedVal)) { + return 0, fmt.Errorf("expected integer value, got: %v", v) + } + return int64(typedVal), nil + case float64: + if !isIntegerValue(typedVal) { + return 0, fmt.Errorf("expected integer value, got: %v", v) + } + return int64(typedVal), nil + + // pass through common integer cases + case int: + return int64(typedVal), nil + case int32: + return int64(typedVal), nil + case int64: + return typedVal, nil + + default: + return 0, fmt.Errorf("cannot convert %#v to an integer", v) + } +} + +func toString(v any, T jsonschema.Type) (string, error) { + switch T { + case jsonschema.BooleanType: + boolVal, ok := v.(bool) + if !ok { + return "", fmt.Errorf("expected bool, got: %#v", v) + } + return strconv.FormatBool(boolVal), nil + case jsonschema.StringType: + strVal, ok := v.(string) + if !ok { + return "", fmt.Errorf("expected string, got: %#v", v) + } + return strVal, nil + case jsonschema.NumberType: + floatVal, ok := v.(float64) + if !ok { + return "", fmt.Errorf("expected float, got: %#v", v) + } + return strconv.FormatFloat(floatVal, 'f', -1, 64), nil + case jsonschema.IntegerType: + intVal, err := toInteger(v) + if err != nil { + return "", err + } + return strconv.FormatInt(intVal, 10), nil + default: + return "", fmt.Errorf("cannot format object of type %s as a string. Value of object: %#v", T, v) + } +} + +func fromString(s string, T jsonschema.Type) (any, error) { + if T == jsonschema.StringType { + return s, nil + } + + // Variables to store value and error from parsing + var v any + var err error + + switch T { + case jsonschema.BooleanType: + v, err = strconv.ParseBool(s) + case jsonschema.NumberType: + v, err = strconv.ParseFloat(s, 32) + case jsonschema.IntegerType: + v, err = strconv.ParseInt(s, 10, 64) + default: + return "", fmt.Errorf("cannot parse string as object of type %s. Value of string: %q", T, s) + } + + // Return more readable error incase of a syntax error + if errors.Is(err, strconv.ErrSyntax) { + return nil, fmt.Errorf("could not parse %q as a %s: %w", s, T, err) + } + return v, err +} diff --git a/libs/template/utils_test.go b/libs/template/utils_test.go new file mode 100644 index 00000000..5fe70243 --- /dev/null +++ b/libs/template/utils_test.go @@ -0,0 +1,115 @@ +package template + +import ( + "math" + "testing" + + "github.com/databricks/cli/libs/jsonschema" + "github.com/stretchr/testify/assert" +) + +func TestTemplateIsInteger(t *testing.T) { + assert.False(t, isIntegerValue(1.1)) + assert.False(t, isIntegerValue(0.1)) + assert.False(t, isIntegerValue(-0.1)) + + assert.True(t, isIntegerValue(-1.0)) + assert.True(t, isIntegerValue(0.0)) + assert.True(t, isIntegerValue(2.0)) +} + +func TestTemplateToInteger(t *testing.T) { + v, err := toInteger(float32(2)) + assert.NoError(t, err) + assert.Equal(t, int64(2), v) + + v, err = toInteger(float64(4)) + assert.NoError(t, err) + assert.Equal(t, int64(4), v) + + v, err = toInteger(float64(4)) + assert.NoError(t, err) + assert.Equal(t, int64(4), v) + + v, err = toInteger(float64(math.MaxInt32 + 10)) + assert.NoError(t, err) + assert.Equal(t, int64(2147483657), v) + + v, err = toInteger(2) + assert.NoError(t, err) + assert.Equal(t, int64(2), v) + + _, err = toInteger(float32(2.2)) + assert.EqualError(t, err, "expected integer value, got: 2.2") + + _, err = toInteger(float64(math.MaxInt32 + 100.1)) + assert.ErrorContains(t, err, "expected integer value, got: 2.1474837471e+09") + + _, err = toInteger("abcd") + assert.EqualError(t, err, "cannot convert \"abcd\" to an integer") +} + +func TestTemplateToString(t *testing.T) { + s, err := toString(true, jsonschema.BooleanType) + assert.NoError(t, err) + assert.Equal(t, "true", s) + + s, err = toString("abc", jsonschema.StringType) + assert.NoError(t, err) + assert.Equal(t, "abc", s) + + s, err = toString(1.1, jsonschema.NumberType) + assert.NoError(t, err) + assert.Equal(t, "1.1", s) + + s, err = toString(2, jsonschema.IntegerType) + assert.NoError(t, err) + assert.Equal(t, "2", s) + + _, err = toString([]string{}, jsonschema.ArrayType) + assert.EqualError(t, err, "cannot format object of type array as a string. Value of object: []string{}") + + _, err = toString("true", jsonschema.BooleanType) + assert.EqualError(t, err, "expected bool, got: \"true\"") + + _, err = toString(123, jsonschema.StringType) + assert.EqualError(t, err, "expected string, got: 123") + + _, err = toString(false, jsonschema.NumberType) + assert.EqualError(t, err, "expected float, got: false") + + _, err = toString("abc", jsonschema.IntegerType) + assert.EqualError(t, err, "cannot convert \"abc\" to an integer") +} + +func TestTemplateFromString(t *testing.T) { + v, err := fromString("true", jsonschema.BooleanType) + assert.NoError(t, err) + assert.Equal(t, true, v) + + v, err = fromString("abc", jsonschema.StringType) + assert.NoError(t, err) + assert.Equal(t, "abc", v) + + v, err = fromString("1.1", jsonschema.NumberType) + assert.NoError(t, err) + // Floating point conversions are not perfect + assert.True(t, (v.(float64)-1.1) < 0.000001) + + v, err = fromString("12345", jsonschema.IntegerType) + assert.NoError(t, err) + assert.Equal(t, int64(12345), v) + + v, err = fromString("123", jsonschema.NumberType) + assert.NoError(t, err) + assert.Equal(t, float64(123), v) + + _, err = fromString("qrt", jsonschema.ArrayType) + assert.EqualError(t, err, "cannot parse string as object of type array. Value of string: \"qrt\"") + + _, err = fromString("abc", jsonschema.IntegerType) + assert.EqualError(t, err, "could not parse \"abc\" as a integer: strconv.ParseInt: parsing \"abc\": invalid syntax") + + _, err = fromString("1.0", jsonschema.IntegerType) + assert.EqualError(t, err, "could not parse \"1.0\" as a integer: strconv.ParseInt: parsing \"1.0\": invalid syntax") +} diff --git a/libs/template/validators.go b/libs/template/validators.go index 0ae41e46..57eda093 100644 --- a/libs/template/validators.go +++ b/libs/template/validators.go @@ -33,9 +33,7 @@ func validateBoolean(v any) error { } func validateNumber(v any) error { - if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64, reflect.Int, - reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, - reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64}, + if !slices.Contains([]reflect.Kind{reflect.Float32, reflect.Float64}, reflect.TypeOf(v).Kind()) { return fmt.Errorf("expected type float, but value is %#v", v) } diff --git a/libs/template/validators_test.go b/libs/template/validators_test.go index f0cbf8a1..f34f037a 100644 --- a/libs/template/validators_test.go +++ b/libs/template/validators_test.go @@ -3,8 +3,8 @@ package template import ( "testing" + "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestValidatorString(t *testing.T) { @@ -40,10 +40,10 @@ func TestValidatorNumber(t *testing.T) { assert.ErrorContains(t, err, "expected type float, but value is true") err = validateNumber(int32(1)) - require.NoError(t, err) + assert.ErrorContains(t, err, "expected type float, but value is 1") - err = validateNumber(int64(1)) - require.NoError(t, err) + err = validateNumber(int64(2)) + assert.ErrorContains(t, err, "expected type float, but value is 2") err = validateNumber(float32(1)) assert.NoError(t, err) @@ -74,3 +74,56 @@ func TestValidatorInt(t *testing.T) { err = validateInteger("abc") assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") } + +func TestTemplateValidateType(t *testing.T) { + // assert validation passing + err := validateType(int(0), jsonschema.IntegerType) + assert.NoError(t, err) + err = validateType(int32(1), jsonschema.IntegerType) + assert.NoError(t, err) + err = validateType(int64(1), jsonschema.IntegerType) + assert.NoError(t, err) + + err = validateType(float32(1.1), jsonschema.NumberType) + assert.NoError(t, err) + err = validateType(float64(1.2), jsonschema.NumberType) + assert.NoError(t, err) + + err = validateType(false, jsonschema.BooleanType) + assert.NoError(t, err) + + err = validateType("abc", jsonschema.StringType) + assert.NoError(t, err) + + // assert validation failing for integers + err = validateType(float64(1.2), jsonschema.IntegerType) + assert.ErrorContains(t, err, "expected type integer, but value is 1.2") + err = validateType(true, jsonschema.IntegerType) + assert.ErrorContains(t, err, "expected type integer, but value is true") + err = validateType("abc", jsonschema.IntegerType) + assert.ErrorContains(t, err, "expected type integer, but value is \"abc\"") + + // assert validation failing for floats + err = validateType(true, jsonschema.NumberType) + assert.ErrorContains(t, err, "expected type float, but value is true") + err = validateType("abc", jsonschema.NumberType) + assert.ErrorContains(t, err, "expected type float, but value is \"abc\"") + err = validateType(int(1), jsonschema.NumberType) + assert.ErrorContains(t, err, "expected type float, but value is 1") + + // assert validation failing for boolean + err = validateType(int(1), jsonschema.BooleanType) + assert.ErrorContains(t, err, "expected type boolean, but value is 1") + err = validateType(float64(1), jsonschema.BooleanType) + assert.ErrorContains(t, err, "expected type boolean, but value is 1") + err = validateType("abc", jsonschema.BooleanType) + assert.ErrorContains(t, err, "expected type boolean, but value is \"abc\"") + + // assert validation failing for string + err = validateType(int(1), jsonschema.StringType) + assert.ErrorContains(t, err, "expected type string, but value is 1") + err = validateType(float64(1), jsonschema.StringType) + assert.ErrorContains(t, err, "expected type string, but value is 1") + err = validateType(false, jsonschema.StringType) + assert.ErrorContains(t, err, "expected type string, but value is false") +}