Add configuration normalization code (#915)

## Changes

This is similar to #904 but instead of converting the dynamic
configuration to Go structs, this normalizes a `config.Value` according
to the type of a Go struct and returns the new, normalized
`config.Value`.

This will be used to ensure that two `config.Value` trees are
type-compatible before we can merge them (i.e. instances from different
files).

Warnings and errors during normalization are accumulated and returned as
a `diag.Diagnostics` structure. We can use this to surface warnings
about unknown fields, or errors about invalid types, in aggregate
instead of one-by-one. This approach is inspired by the pattern to
accumulate diagnostics in Terraform provider code.

## Tests

New unit tests.
This commit is contained in:
Pieter Noordhuis 2023-10-25 13:56:42 +02:00 committed by GitHub
parent 486bf59627
commit a60c40e71e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 755 additions and 0 deletions

View File

@ -0,0 +1,235 @@
package convert
import (
"fmt"
"reflect"
"strconv"
"github.com/databricks/cli/libs/config"
"github.com/databricks/cli/libs/diag"
)
func Normalize(dst any, src config.Value) (config.Value, diag.Diagnostics) {
return normalizeType(reflect.TypeOf(dst), src)
}
func normalizeType(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
for typ.Kind() == reflect.Pointer {
typ = typ.Elem()
}
switch typ.Kind() {
case reflect.Struct:
return normalizeStruct(typ, src)
case reflect.Map:
return normalizeMap(typ, src)
case reflect.Slice:
return normalizeSlice(typ, src)
case reflect.String:
return normalizeString(typ, src)
case reflect.Bool:
return normalizeBool(typ, src)
case reflect.Int, reflect.Int32, reflect.Int64:
return normalizeInt(typ, src)
case reflect.Float32, reflect.Float64:
return normalizeFloat(typ, src)
}
return config.NilValue, diag.Errorf("unsupported type: %s", typ.Kind())
}
func typeMismatch(expected config.Kind, src config.Value) diag.Diagnostic {
return diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()),
Location: src.Location(),
}
}
func normalizeStruct(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
switch src.Kind() {
case config.KindMap:
out := make(map[string]config.Value)
info := getStructInfo(typ)
for k, v := range src.MustMap() {
index, ok := info.Fields[k]
if !ok {
diags = diags.Append(diag.Diagnostic{
Severity: diag.Warning,
Summary: fmt.Sprintf("unknown field: %s", k),
Location: src.Location(),
})
continue
}
// Normalize the value according to the field type.
v, err := normalizeType(typ.FieldByIndex(index).Type, v)
if err != nil {
diags = diags.Extend(err)
// Skip the element if it cannot be normalized.
if err.HasError() {
continue
}
}
out[k] = v
}
return config.NewValue(out, src.Location()), diags
case config.KindNil:
return src, diags
}
return config.NilValue, diags.Append(typeMismatch(config.KindMap, src))
}
func normalizeMap(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
switch src.Kind() {
case config.KindMap:
out := make(map[string]config.Value)
for k, v := range src.MustMap() {
// Normalize the value according to the map element type.
v, err := normalizeType(typ.Elem(), v)
if err != nil {
diags = diags.Extend(err)
// Skip the element if it cannot be normalized.
if err.HasError() {
continue
}
}
out[k] = v
}
return config.NewValue(out, src.Location()), diags
case config.KindNil:
return src, diags
}
return config.NilValue, diags.Append(typeMismatch(config.KindMap, src))
}
func normalizeSlice(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
switch src.Kind() {
case config.KindSequence:
out := make([]config.Value, 0, len(src.MustSequence()))
for _, v := range src.MustSequence() {
// Normalize the value according to the slice element type.
v, err := normalizeType(typ.Elem(), v)
if err != nil {
diags = diags.Extend(err)
// Skip the element if it cannot be normalized.
if err.HasError() {
continue
}
}
out = append(out, v)
}
return config.NewValue(out, src.Location()), diags
case config.KindNil:
return src, diags
}
return config.NilValue, diags.Append(typeMismatch(config.KindSequence, src))
}
func normalizeString(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
var out string
switch src.Kind() {
case config.KindString:
out = src.MustString()
case config.KindBool:
out = strconv.FormatBool(src.MustBool())
case config.KindInt:
out = strconv.FormatInt(src.MustInt(), 10)
case config.KindFloat:
out = strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)
default:
return config.NilValue, diags.Append(typeMismatch(config.KindString, src))
}
return config.NewValue(out, src.Location()), diags
}
func normalizeBool(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
var out bool
switch src.Kind() {
case config.KindBool:
out = src.MustBool()
case config.KindString:
// See https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/decode.go#L684-L693.
switch src.MustString() {
case "true", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON":
out = true
case "false", "n", "N", "no", "No", "NO", "off", "Off", "OFF":
out = false
default:
// Cannot interpret as a boolean.
return config.NilValue, diags.Append(typeMismatch(config.KindBool, src))
}
default:
return config.NilValue, diags.Append(typeMismatch(config.KindBool, src))
}
return config.NewValue(out, src.Location()), diags
}
func normalizeInt(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
var out int64
switch src.Kind() {
case config.KindInt:
out = src.MustInt()
case config.KindString:
var err error
out, err = strconv.ParseInt(src.MustString(), 10, 64)
if err != nil {
return config.NilValue, diags.Append(diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()),
Location: src.Location(),
})
}
default:
return config.NilValue, diags.Append(typeMismatch(config.KindInt, src))
}
return config.NewValue(out, src.Location()), diags
}
func normalizeFloat(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
var diags diag.Diagnostics
var out float64
switch src.Kind() {
case config.KindFloat:
out = src.MustFloat()
case config.KindString:
var err error
out, err = strconv.ParseFloat(src.MustString(), 64)
if err != nil {
return config.NilValue, diags.Append(diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()),
Location: src.Location(),
})
}
default:
return config.NilValue, diags.Append(typeMismatch(config.KindFloat, src))
}
return config.NewValue(out, src.Location()), diags
}

View File

@ -0,0 +1,435 @@
package convert
import (
"testing"
"github.com/databricks/cli/libs/config"
"github.com/databricks/cli/libs/diag"
"github.com/stretchr/testify/assert"
)
func TestNormalizeStruct(t *testing.T) {
type Tmp struct {
Foo string `json:"foo"`
Bar string `json:"bar"`
}
var typ Tmp
vin := config.V(map[string]config.Value{
"foo": config.V("bar"),
"bar": config.V("baz"),
})
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeStructElementDiagnostic(t *testing.T) {
type Tmp struct {
Foo string `json:"foo"`
Bar string `json:"bar"`
}
var typ Tmp
vin := config.V(map[string]config.Value{
"foo": config.V("bar"),
"bar": config.V(map[string]config.Value{"an": config.V("error")}),
})
vout, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected string, found map`,
Location: config.Location{},
}, err[0])
// Elements that encounter an error during normalization are dropped.
assert.Equal(t, map[string]any{
"foo": "bar",
}, vout.AsAny())
}
func TestNormalizeStructUnknownField(t *testing.T) {
type Tmp struct {
Foo string `json:"foo"`
}
var typ Tmp
vin := config.V(map[string]config.Value{
"foo": config.V("bar"),
"bar": config.V("baz"),
})
vout, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Warning,
Summary: `unknown field: bar`,
Location: vin.Get("foo").Location(),
}, err[0])
// The field that can be mapped to the struct field is retained.
assert.Equal(t, map[string]any{
"foo": "bar",
}, vout.AsAny())
}
func TestNormalizeStructNil(t *testing.T) {
type Tmp struct {
Foo string `json:"foo"`
}
var typ Tmp
vin := config.NilValue
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeStructError(t *testing.T) {
type Tmp struct {
Foo string `json:"foo"`
}
var typ Tmp
vin := config.V("string")
_, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected map, found string`,
Location: vin.Get("foo").Location(),
}, err[0])
}
func TestNormalizeMap(t *testing.T) {
var typ map[string]string
vin := config.V(map[string]config.Value{
"foo": config.V("bar"),
"bar": config.V("baz"),
})
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeMapElementDiagnostic(t *testing.T) {
var typ map[string]string
vin := config.V(map[string]config.Value{
"foo": config.V("bar"),
"bar": config.V(map[string]config.Value{"an": config.V("error")}),
})
vout, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected string, found map`,
Location: config.Location{},
}, err[0])
// Elements that encounter an error during normalization are dropped.
assert.Equal(t, map[string]any{
"foo": "bar",
}, vout.AsAny())
}
func TestNormalizeMapNil(t *testing.T) {
var typ map[string]string
vin := config.NilValue
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeMapError(t *testing.T) {
var typ map[string]string
vin := config.V("string")
_, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected map, found string`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeSlice(t *testing.T) {
var typ []string
vin := config.V([]config.Value{
config.V("foo"),
config.V("bar"),
})
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeSliceElementDiagnostic(t *testing.T) {
var typ []string
vin := config.V([]config.Value{
config.V("foo"),
config.V("bar"),
config.V(map[string]config.Value{"an": config.V("error")}),
})
vout, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected string, found map`,
Location: config.Location{},
}, err[0])
// Elements that encounter an error during normalization are dropped.
assert.Equal(t, []any{"foo", "bar"}, vout.AsAny())
}
func TestNormalizeSliceNil(t *testing.T) {
var typ []string
vin := config.NilValue
vout, err := Normalize(typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeSliceError(t *testing.T) {
var typ []string
vin := config.V("string")
_, err := Normalize(typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected sequence, found string`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeString(t *testing.T) {
var typ string
vin := config.V("string")
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, vin, vout)
}
func TestNormalizeStringNil(t *testing.T) {
var typ string
vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected string, found nil`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeStringFromBool(t *testing.T) {
var typ string
vin := config.NewValue(true, config.Location{File: "file", Line: 1, Column: 1})
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.NewValue("true", vin.Location()), vout)
}
func TestNormalizeStringFromInt(t *testing.T) {
var typ string
vin := config.NewValue(123, config.Location{File: "file", Line: 1, Column: 1})
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.NewValue("123", vin.Location()), vout)
}
func TestNormalizeStringFromFloat(t *testing.T) {
var typ string
vin := config.NewValue(1.20, config.Location{File: "file", Line: 1, Column: 1})
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.NewValue("1.2", vin.Location()), vout)
}
func TestNormalizeStringError(t *testing.T) {
var typ string
vin := config.V(map[string]config.Value{"an": config.V("error")})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected string, found map`,
Location: config.Location{},
}, err[0])
}
func TestNormalizeBool(t *testing.T) {
var typ bool
vin := config.V(true)
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(true), vout)
}
func TestNormalizeBoolNil(t *testing.T) {
var typ bool
vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected bool, found nil`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeBoolFromString(t *testing.T) {
var typ bool
for _, c := range []struct {
Input string
Output bool
}{
{"true", true},
{"false", false},
{"Y", true},
{"N", false},
{"on", true},
{"off", false},
} {
vin := config.V(c.Input)
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(c.Output), vout)
}
}
func TestNormalizeBoolFromStringError(t *testing.T) {
var typ bool
vin := config.V("abc")
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected bool, found string`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeBoolError(t *testing.T) {
var typ bool
vin := config.V(map[string]config.Value{"an": config.V("error")})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected bool, found map`,
Location: config.Location{},
}, err[0])
}
func TestNormalizeInt(t *testing.T) {
var typ int
vin := config.V(123)
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(int64(123)), vout)
}
func TestNormalizeIntNil(t *testing.T) {
var typ int
vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected int, found nil`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeIntFromString(t *testing.T) {
var typ int
vin := config.V("123")
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(int64(123)), vout)
}
func TestNormalizeIntFromStringError(t *testing.T) {
var typ int
vin := config.V("abc")
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `cannot parse "abc" as an integer`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeIntError(t *testing.T) {
var typ int
vin := config.V(map[string]config.Value{"an": config.V("error")})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected int, found map`,
Location: config.Location{},
}, err[0])
}
func TestNormalizeFloat(t *testing.T) {
var typ float64
vin := config.V(1.2)
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(1.2), vout)
}
func TestNormalizeFloatNil(t *testing.T) {
var typ float64
vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected float, found nil`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeFloatFromString(t *testing.T) {
var typ float64
vin := config.V("1.2")
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, config.V(1.2), vout)
}
func TestNormalizeFloatFromStringError(t *testing.T) {
var typ float64
vin := config.V("abc")
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `cannot parse "abc" as a floating point number`,
Location: vin.Location(),
}, err[0])
}
func TestNormalizeFloatError(t *testing.T) {
var typ float64
vin := config.V(map[string]config.Value{"an": config.V("error")})
_, err := Normalize(&typ, vin)
assert.Len(t, err, 1)
assert.Equal(t, diag.Diagnostic{
Severity: diag.Error,
Summary: `expected float, found map`,
Location: config.Location{},
}, err[0])
}

76
libs/diag/diagnostic.go Normal file
View File

@ -0,0 +1,76 @@
package diag
import (
"fmt"
"github.com/databricks/cli/libs/config"
)
type Diagnostic struct {
Severity Severity
// Summary is a short description of the diagnostic.
// This is expected to be a single line and always present.
Summary string
// Detail is a longer description of the diagnostic.
// This may be multiple lines and may be nil.
Detail string
// Location is a source code location associated with the diagnostic message.
// It may be zero if there is no associated location.
Location config.Location
}
// Errorf creates a new error diagnostic.
func Errorf(format string, args ...any) Diagnostics {
return []Diagnostic{
{
Severity: Error,
Summary: fmt.Sprintf(format, args...),
},
}
}
// Warningf creates a new warning diagnostic.
func Warningf(format string, args ...any) Diagnostics {
return []Diagnostic{
{
Severity: Warning,
Summary: fmt.Sprintf(format, args...),
},
}
}
// Infof creates a new info diagnostic.
func Infof(format string, args ...any) Diagnostics {
return []Diagnostic{
{
Severity: Info,
Summary: fmt.Sprintf(format, args...),
},
}
}
// Diagsnostics holds zero or more instances of [Diagnostic].
type Diagnostics []Diagnostic
// Append adds a new diagnostic to the end of the list.
func (ds Diagnostics) Append(d Diagnostic) Diagnostics {
return append(ds, d)
}
// Extend adds all diagnostics from another list to the end of the list.
func (ds Diagnostics) Extend(other Diagnostics) Diagnostics {
return append(ds, other...)
}
// HasError returns true if any of the diagnostics are errors.
func (ds Diagnostics) HasError() bool {
for _, d := range ds {
if d.Severity == Error {
return true
}
}
return false
}

9
libs/diag/severity.go Normal file
View File

@ -0,0 +1,9 @@
package diag
type Severity int
const (
Error Severity = iota
Warning
Info
)