Add configuration normalization code (#915)

## Changes

This is similar to #904 but instead of converting the dynamic
configuration to Go structs, this normalizes a `config.Value` according
to the type of a Go struct and returns the new, normalized
`config.Value`.

This will be used to ensure that two `config.Value` trees are
type-compatible before we can merge them (i.e. instances from different
files).

Warnings and errors during normalization are accumulated and returned as
a `diag.Diagnostics` structure. We can use this to surface warnings
about unknown fields, or errors about invalid types, in aggregate
instead of one-by-one. This approach is inspired by the pattern to
accumulate diagnostics in Terraform provider code.

## Tests

New unit tests.

libs/config/convert/normalize.go

@@ -0,0 +1,235 @@
+package convert
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+
+	"github.com/databricks/cli/libs/config"
+	"github.com/databricks/cli/libs/diag"
+)
+
+func Normalize(dst any, src config.Value) (config.Value, diag.Diagnostics) {
+	return normalizeType(reflect.TypeOf(dst), src)
+}
+
+func normalizeType(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	for typ.Kind() == reflect.Pointer {
+		typ = typ.Elem()
+	}
+
+	switch typ.Kind() {
+	case reflect.Struct:
+		return normalizeStruct(typ, src)
+	case reflect.Map:
+		return normalizeMap(typ, src)
+	case reflect.Slice:
+		return normalizeSlice(typ, src)
+	case reflect.String:
+		return normalizeString(typ, src)
+	case reflect.Bool:
+		return normalizeBool(typ, src)
+	case reflect.Int, reflect.Int32, reflect.Int64:
+		return normalizeInt(typ, src)
+	case reflect.Float32, reflect.Float64:
+		return normalizeFloat(typ, src)
+	}
+
+	return config.NilValue, diag.Errorf("unsupported type: %s", typ.Kind())
+}
+
+func typeMismatch(expected config.Kind, src config.Value) diag.Diagnostic {
+	return diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  fmt.Sprintf("expected %s, found %s", expected, src.Kind()),
+		Location: src.Location(),
+	}
+}
+
+func normalizeStruct(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	switch src.Kind() {
+	case config.KindMap:
+		out := make(map[string]config.Value)
+		info := getStructInfo(typ)
+		for k, v := range src.MustMap() {
+			index, ok := info.Fields[k]
+			if !ok {
+				diags = diags.Append(diag.Diagnostic{
+					Severity: diag.Warning,
+					Summary:  fmt.Sprintf("unknown field: %s", k),
+					Location: src.Location(),
+				})
+				continue
+			}
+
+			// Normalize the value according to the field type.
+			v, err := normalizeType(typ.FieldByIndex(index).Type, v)
+			if err != nil {
+				diags = diags.Extend(err)
+				// Skip the element if it cannot be normalized.
+				if err.HasError() {
+					continue
+				}
+			}
+
+			out[k] = v
+		}
+
+		return config.NewValue(out, src.Location()), diags
+	case config.KindNil:
+		return src, diags
+	}
+
+	return config.NilValue, diags.Append(typeMismatch(config.KindMap, src))
+}
+
+func normalizeMap(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	switch src.Kind() {
+	case config.KindMap:
+		out := make(map[string]config.Value)
+		for k, v := range src.MustMap() {
+			// Normalize the value according to the map element type.
+			v, err := normalizeType(typ.Elem(), v)
+			if err != nil {
+				diags = diags.Extend(err)
+				// Skip the element if it cannot be normalized.
+				if err.HasError() {
+					continue
+				}
+			}
+
+			out[k] = v
+		}
+
+		return config.NewValue(out, src.Location()), diags
+	case config.KindNil:
+		return src, diags
+	}
+
+	return config.NilValue, diags.Append(typeMismatch(config.KindMap, src))
+}
+
+func normalizeSlice(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+
+	switch src.Kind() {
+	case config.KindSequence:
+		out := make([]config.Value, 0, len(src.MustSequence()))
+		for _, v := range src.MustSequence() {
+			// Normalize the value according to the slice element type.
+			v, err := normalizeType(typ.Elem(), v)
+			if err != nil {
+				diags = diags.Extend(err)
+				// Skip the element if it cannot be normalized.
+				if err.HasError() {
+					continue
+				}
+			}
+
+			out = append(out, v)
+		}
+
+		return config.NewValue(out, src.Location()), diags
+	case config.KindNil:
+		return src, diags
+	}
+
+	return config.NilValue, diags.Append(typeMismatch(config.KindSequence, src))
+}
+
+func normalizeString(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	var out string
+
+	switch src.Kind() {
+	case config.KindString:
+		out = src.MustString()
+	case config.KindBool:
+		out = strconv.FormatBool(src.MustBool())
+	case config.KindInt:
+		out = strconv.FormatInt(src.MustInt(), 10)
+	case config.KindFloat:
+		out = strconv.FormatFloat(src.MustFloat(), 'f', -1, 64)
+	default:
+		return config.NilValue, diags.Append(typeMismatch(config.KindString, src))
+	}
+
+	return config.NewValue(out, src.Location()), diags
+}
+
+func normalizeBool(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	var out bool
+
+	switch src.Kind() {
+	case config.KindBool:
+		out = src.MustBool()
+	case config.KindString:
+		// See https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/decode.go#L684-L693.
+		switch src.MustString() {
+		case "true", "y", "Y", "yes", "Yes", "YES", "on", "On", "ON":
+			out = true
+		case "false", "n", "N", "no", "No", "NO", "off", "Off", "OFF":
+			out = false
+		default:
+			// Cannot interpret as a boolean.
+			return config.NilValue, diags.Append(typeMismatch(config.KindBool, src))
+		}
+	default:
+		return config.NilValue, diags.Append(typeMismatch(config.KindBool, src))
+	}
+
+	return config.NewValue(out, src.Location()), diags
+}
+
+func normalizeInt(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	var out int64
+
+	switch src.Kind() {
+	case config.KindInt:
+		out = src.MustInt()
+	case config.KindString:
+		var err error
+		out, err = strconv.ParseInt(src.MustString(), 10, 64)
+		if err != nil {
+			return config.NilValue, diags.Append(diag.Diagnostic{
+				Severity: diag.Error,
+				Summary:  fmt.Sprintf("cannot parse %q as an integer", src.MustString()),
+				Location: src.Location(),
+			})
+		}
+	default:
+		return config.NilValue, diags.Append(typeMismatch(config.KindInt, src))
+	}
+
+	return config.NewValue(out, src.Location()), diags
+}
+
+func normalizeFloat(typ reflect.Type, src config.Value) (config.Value, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	var out float64
+
+	switch src.Kind() {
+	case config.KindFloat:
+		out = src.MustFloat()
+	case config.KindString:
+		var err error
+		out, err = strconv.ParseFloat(src.MustString(), 64)
+		if err != nil {
+			return config.NilValue, diags.Append(diag.Diagnostic{
+				Severity: diag.Error,
+				Summary:  fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()),
+				Location: src.Location(),
+			})
+		}
+	default:
+		return config.NilValue, diags.Append(typeMismatch(config.KindFloat, src))
+	}
+
+	return config.NewValue(out, src.Location()), diags
+}

libs/config/convert/normalize_test.go

@@ -0,0 +1,435 @@
+package convert
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/libs/config"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeStruct(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+		Bar string `json:"bar"`
+	}
+
+	var typ Tmp
+	vin := config.V(map[string]config.Value{
+		"foo": config.V("bar"),
+		"bar": config.V("baz"),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeStructElementDiagnostic(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+		Bar string `json:"bar"`
+	}
+
+	var typ Tmp
+	vin := config.V(map[string]config.Value{
+		"foo": config.V("bar"),
+		"bar": config.V(map[string]config.Value{"an": config.V("error")}),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected string, found map`,
+		Location: config.Location{},
+	}, err[0])
+
+	// Elements that encounter an error during normalization are dropped.
+	assert.Equal(t, map[string]any{
+		"foo": "bar",
+	}, vout.AsAny())
+}
+
+func TestNormalizeStructUnknownField(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := config.V(map[string]config.Value{
+		"foo": config.V("bar"),
+		"bar": config.V("baz"),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `unknown field: bar`,
+		Location: vin.Get("foo").Location(),
+	}, err[0])
+
+	// The field that can be mapped to the struct field is retained.
+	assert.Equal(t, map[string]any{
+		"foo": "bar",
+	}, vout.AsAny())
+}
+
+func TestNormalizeStructNil(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := config.NilValue
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeStructError(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := config.V("string")
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected map, found string`,
+		Location: vin.Get("foo").Location(),
+	}, err[0])
+}
+
+func TestNormalizeMap(t *testing.T) {
+	var typ map[string]string
+	vin := config.V(map[string]config.Value{
+		"foo": config.V("bar"),
+		"bar": config.V("baz"),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeMapElementDiagnostic(t *testing.T) {
+	var typ map[string]string
+	vin := config.V(map[string]config.Value{
+		"foo": config.V("bar"),
+		"bar": config.V(map[string]config.Value{"an": config.V("error")}),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected string, found map`,
+		Location: config.Location{},
+	}, err[0])
+
+	// Elements that encounter an error during normalization are dropped.
+	assert.Equal(t, map[string]any{
+		"foo": "bar",
+	}, vout.AsAny())
+}
+
+func TestNormalizeMapNil(t *testing.T) {
+	var typ map[string]string
+	vin := config.NilValue
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeMapError(t *testing.T) {
+	var typ map[string]string
+	vin := config.V("string")
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected map, found string`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeSlice(t *testing.T) {
+	var typ []string
+	vin := config.V([]config.Value{
+		config.V("foo"),
+		config.V("bar"),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeSliceElementDiagnostic(t *testing.T) {
+	var typ []string
+	vin := config.V([]config.Value{
+		config.V("foo"),
+		config.V("bar"),
+		config.V(map[string]config.Value{"an": config.V("error")}),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected string, found map`,
+		Location: config.Location{},
+	}, err[0])
+
+	// Elements that encounter an error during normalization are dropped.
+	assert.Equal(t, []any{"foo", "bar"}, vout.AsAny())
+}
+
+func TestNormalizeSliceNil(t *testing.T) {
+	var typ []string
+	vin := config.NilValue
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeSliceError(t *testing.T) {
+	var typ []string
+	vin := config.V("string")
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected sequence, found string`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeString(t *testing.T) {
+	var typ string
+	vin := config.V("string")
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeStringNil(t *testing.T) {
+	var typ string
+	vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected string, found nil`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeStringFromBool(t *testing.T) {
+	var typ string
+	vin := config.NewValue(true, config.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.NewValue("true", vin.Location()), vout)
+}
+
+func TestNormalizeStringFromInt(t *testing.T) {
+	var typ string
+	vin := config.NewValue(123, config.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.NewValue("123", vin.Location()), vout)
+}
+
+func TestNormalizeStringFromFloat(t *testing.T) {
+	var typ string
+	vin := config.NewValue(1.20, config.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.NewValue("1.2", vin.Location()), vout)
+}
+
+func TestNormalizeStringError(t *testing.T) {
+	var typ string
+	vin := config.V(map[string]config.Value{"an": config.V("error")})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected string, found map`,
+		Location: config.Location{},
+	}, err[0])
+}
+
+func TestNormalizeBool(t *testing.T) {
+	var typ bool
+	vin := config.V(true)
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.V(true), vout)
+}
+
+func TestNormalizeBoolNil(t *testing.T) {
+	var typ bool
+	vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected bool, found nil`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeBoolFromString(t *testing.T) {
+	var typ bool
+
+	for _, c := range []struct {
+		Input  string
+		Output bool
+	}{
+		{"true", true},
+		{"false", false},
+		{"Y", true},
+		{"N", false},
+		{"on", true},
+		{"off", false},
+	} {
+		vin := config.V(c.Input)
+		vout, err := Normalize(&typ, vin)
+		assert.Empty(t, err)
+		assert.Equal(t, config.V(c.Output), vout)
+	}
+}
+
+func TestNormalizeBoolFromStringError(t *testing.T) {
+	var typ bool
+	vin := config.V("abc")
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected bool, found string`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeBoolError(t *testing.T) {
+	var typ bool
+	vin := config.V(map[string]config.Value{"an": config.V("error")})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected bool, found map`,
+		Location: config.Location{},
+	}, err[0])
+}
+
+func TestNormalizeInt(t *testing.T) {
+	var typ int
+	vin := config.V(123)
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.V(int64(123)), vout)
+}
+
+func TestNormalizeIntNil(t *testing.T) {
+	var typ int
+	vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected int, found nil`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeIntFromString(t *testing.T) {
+	var typ int
+	vin := config.V("123")
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.V(int64(123)), vout)
+}
+
+func TestNormalizeIntFromStringError(t *testing.T) {
+	var typ int
+	vin := config.V("abc")
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `cannot parse "abc" as an integer`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeIntError(t *testing.T) {
+	var typ int
+	vin := config.V(map[string]config.Value{"an": config.V("error")})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected int, found map`,
+		Location: config.Location{},
+	}, err[0])
+}
+
+func TestNormalizeFloat(t *testing.T) {
+	var typ float64
+	vin := config.V(1.2)
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.V(1.2), vout)
+}
+
+func TestNormalizeFloatNil(t *testing.T) {
+	var typ float64
+	vin := config.NewValue(nil, config.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected float, found nil`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeFloatFromString(t *testing.T) {
+	var typ float64
+	vin := config.V("1.2")
+	vout, err := Normalize(&typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, config.V(1.2), vout)
+}
+
+func TestNormalizeFloatFromStringError(t *testing.T) {
+	var typ float64
+	vin := config.V("abc")
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `cannot parse "abc" as a floating point number`,
+		Location: vin.Location(),
+	}, err[0])
+}
+
+func TestNormalizeFloatError(t *testing.T) {
+	var typ float64
+	vin := config.V(map[string]config.Value{"an": config.V("error")})
+	_, err := Normalize(&typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Error,
+		Summary:  `expected float, found map`,
+		Location: config.Location{},
+	}, err[0])
+}

libs/diag/diagnostic.go

@@ -0,0 +1,76 @@
+package diag
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/libs/config"
+)
+
+type Diagnostic struct {
+	Severity Severity
+
+	// Summary is a short description of the diagnostic.
+	// This is expected to be a single line and always present.
+	Summary string
+
+	// Detail is a longer description of the diagnostic.
+	// This may be multiple lines and may be nil.
+	Detail string
+
+	// Location is a source code location associated with the diagnostic message.
+	// It may be zero if there is no associated location.
+	Location config.Location
+}
+
+// Errorf creates a new error diagnostic.
+func Errorf(format string, args ...any) Diagnostics {
+	return []Diagnostic{
+		{
+			Severity: Error,
+			Summary:  fmt.Sprintf(format, args...),
+		},
+	}
+}
+
+// Warningf creates a new warning diagnostic.
+func Warningf(format string, args ...any) Diagnostics {
+	return []Diagnostic{
+		{
+			Severity: Warning,
+			Summary:  fmt.Sprintf(format, args...),
+		},
+	}
+}
+
+// Infof creates a new info diagnostic.
+func Infof(format string, args ...any) Diagnostics {
+	return []Diagnostic{
+		{
+			Severity: Info,
+			Summary:  fmt.Sprintf(format, args...),
+		},
+	}
+}
+
+// Diagsnostics holds zero or more instances of [Diagnostic].
+type Diagnostics []Diagnostic
+
+// Append adds a new diagnostic to the end of the list.
+func (ds Diagnostics) Append(d Diagnostic) Diagnostics {
+	return append(ds, d)
+}
+
+// Extend adds all diagnostics from another list to the end of the list.
+func (ds Diagnostics) Extend(other Diagnostics) Diagnostics {
+	return append(ds, other...)
+}
+
+// HasError returns true if any of the diagnostics are errors.
+func (ds Diagnostics) HasError() bool {
+	for _, d := range ds {
+		if d.Severity == Error {
+			return true
+		}
+	}
+	return false
+}

libs/diag/severity.go

@@ -0,0 +1,9 @@
+package diag
+
+type Severity int
+
+const (
+	Error Severity = iota
+	Warning
+	Info
+)