diff --git a/bundle/config/root.go b/bundle/config/root.go index 978b0cee..f02b4c10 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -37,7 +37,7 @@ type Root struct { // Resources contains a description of all Databricks resources // to deploy in this bundle (e.g. jobs, pipelines, etc.). - Resources Resources `json:"resources"` + Resources Resources `json:"resources,omitempty"` // Environments can be used to differentiate settings and resources between // bundle deployment environments (e.g. development, staging, production). diff --git a/bundle/schema/bundle_config_docs.yml b/bundle/schema/bundle_config_docs.yml new file mode 100644 index 00000000..eeb102fc --- /dev/null +++ b/bundle/schema/bundle_config_docs.yml @@ -0,0 +1,12 @@ +documentation: Root of the bundle config +children: + bundle: + documentation: | + Bundle contains details about this bundle, such as its name, + version of the spec (TODO), default cluster, default warehouse, etc. + children: + environment: + documentation: Environment is set by the mutator that selects the environment. + + artifacts: + documentation: Artifacts contains a description of all code artifacts in this bundle. \ No newline at end of file diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go new file mode 100644 index 00000000..2c35dd16 --- /dev/null +++ b/bundle/schema/docs.go @@ -0,0 +1,25 @@ +package schema + +import ( + "os" + + "gopkg.in/yaml.v2" +) + +type Docs struct { + Documentation string `json:"documentation"` + Children map[string]Docs `json:"children"` +} + +func LoadDocs(path string) (*Docs, error) { + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + docs := Docs{} + err = yaml.Unmarshal(bytes, &docs) + if err != nil { + return nil, err + } + return &docs, nil +} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go new file mode 100644 index 00000000..ef939972 --- /dev/null +++ b/bundle/schema/schema.go @@ -0,0 +1,274 @@ +package schema + +import ( + "container/list" + "fmt" + "reflect" + "strings" +) + +// defines schema for a json object +type Schema struct { + // Type of the object + Type JavascriptType `json:"type,omitempty"` + + // Description of the object. This is rendered as inline documentation in the + // IDE. This is manually injected here using schema.Docs + Description string `json:"description,omitempty"` + + // Schemas for the fields of an struct. The keys are the first json tag. + // The values are the schema for the type of the field + Properties map[string]*Schema `json:"properties,omitempty"` + + // The schema for all values of an array + Items *Schema `json:"items,omitempty"` + + // The schema for any properties not mentioned in the Schema.Properties field. + // this validates maps[string]any in bundle configuration + // OR + // A boolean type with value false. Setting false here validates that all + // properties in the config have been defined in the json schema as properties + // + // Its type during runtime will either be *Schema or bool + AdditionalProperties any `json:"additionalProperties,omitempty"` + + // Required properties for the object. Any fields missing the "omitempty" + // json tag will be included + Required []string `json:"required,omitempty"` +} + +// This function translates golang types into json schema. Here is the mapping +// between json schema types and golang types +// +// - GolangType -> Javascript type / Json Schema2 +// +// - bool -> boolean +// +// - string -> string +// +// - int (all variants) -> number +// +// - float (all variants) -> number +// +// - map[string]MyStruct -> { type: object, additionalProperties: {}} +// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#additional-properties +// +// - []MyStruct -> {type: array, items: {}} +// for details visit: https://json-schema.org/understanding-json-schema/reference/array.html#items +// +// - []MyStruct -> {type: object, properties: {}, additionalProperties: false} +// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties +func NewSchema(golangType reflect.Type, docs *Docs) (*Schema, error) { + tracker := newTracker() + schema, err := safeToSchema(golangType, docs, "", tracker) + if err != nil { + return nil, tracker.errWithTrace(err.Error()) + } + return schema, nil +} + +type JavascriptType string + +const ( + Invalid JavascriptType = "invalid" + Boolean JavascriptType = "boolean" + String JavascriptType = "string" + Number JavascriptType = "number" + Object JavascriptType = "object" + Array JavascriptType = "array" +) + +func javascriptType(golangType reflect.Type) (JavascriptType, error) { + switch golangType.Kind() { + case reflect.Bool: + return Boolean, nil + case reflect.String: + return String, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + + return Number, nil + case reflect.Struct: + return Object, nil + case reflect.Map: + if golangType.Key().Kind() != reflect.String { + return Invalid, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) + } + return Object, nil + case reflect.Array, reflect.Slice: + return Array, nil + default: + return Invalid, fmt.Errorf("unhandled golang type: %s", golangType) + } +} + +// A wrapper over toSchema function to: +// 1. Detect cycles in the bundle config struct. +// 2. Update tracker +// +// params: +// +// - golangType: Golang type to generate json schema for +// +// - docs: Contains documentation to be injected into the generated json schema +// +// - traceId: An identifier for the current type, to trace recursive traversal. +// Its value is the first json tag in case of struct fields and "" in other cases +// like array, map or no json tags +// +// - tracker: Keeps track of types / traceIds seen during recursive traversal +func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*Schema, error) { + // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA + // There are mechanisms to deal with cycles though recursive identifiers in json + // schema. However if we use them, we would need to make sure we are able to detect + // cycles where two properties (directly or indirectly) pointing to each other + // + // see: https://json-schema.org/understanding-json-schema/structuring.html#recursion + // for details + if tracker.hasCycle(golangType) { + return nil, fmt.Errorf("cycle detected") + } + + tracker.push(golangType, traceId) + props, err := toSchema(golangType, docs, tracker) + if err != nil { + return nil, err + } + tracker.pop(golangType) + return props, nil +} + +// This function returns all member fields of the provided type. +// If the type has embedded (aka anonymous) fields, this function traverses +// those in a breadth first manner +func getStructFields(golangType reflect.Type) []reflect.StructField { + fields := []reflect.StructField{} + bfsQueue := list.New() + + for i := 0; i < golangType.NumField(); i++ { + bfsQueue.PushBack(golangType.Field(i)) + } + for bfsQueue.Len() > 0 { + front := bfsQueue.Front() + field := front.Value.(reflect.StructField) + bfsQueue.Remove(front) + + if !field.Anonymous { + fields = append(fields, field) + continue + } + + fieldType := field.Type + if fieldType.Kind() == reflect.Pointer { + fieldType = fieldType.Elem() + } + + for i := 0; i < fieldType.NumField(); i++ { + bfsQueue.PushBack(fieldType.Field(i)) + } + } + return fields +} + +func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, error) { + // *Struct and Struct generate identical json schemas + if golangType.Kind() == reflect.Pointer { + return safeToSchema(golangType.Elem(), docs, "", tracker) + } + if golangType.Kind() == reflect.Interface { + return &Schema{}, nil + } + + rootJavascriptType, err := javascriptType(golangType) + if err != nil { + return nil, err + } + schema := &Schema{Type: rootJavascriptType} + + if docs != nil { + schema.Description = docs.Documentation + } + + // case array/slice + if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { + elemGolangType := golangType.Elem() + elemJavascriptType, err := javascriptType(elemGolangType) + if err != nil { + return nil, err + } + elemProps, err := safeToSchema(elemGolangType, docs, "", tracker) + if err != nil { + return nil, err + } + schema.Items = &Schema{ + Type: elemJavascriptType, + Properties: elemProps.Properties, + AdditionalProperties: elemProps.AdditionalProperties, + Items: elemProps.Items, + Required: elemProps.Required, + } + } + + // case map + if golangType.Kind() == reflect.Map { + if golangType.Key().Kind() != reflect.String { + return nil, fmt.Errorf("only string keyed maps allowed") + } + schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), docs, "", tracker) + if err != nil { + return nil, err + } + } + + // case struct + if golangType.Kind() == reflect.Struct { + children := getStructFields(golangType) + properties := map[string]*Schema{} + required := []string{} + for _, child := range children { + // get child json tags + childJsonTag := strings.Split(child.Tag.Get("json"), ",") + childName := childJsonTag[0] + + // skip children that have no json tags, the first json tag is "" + // or the first json tag is "-" + if childName == "" || childName == "-" { + continue + } + + // get docs for the child if they exist + var childDocs *Docs + if docs != nil { + if val, ok := docs.Children[childName]; ok { + childDocs = &val + } + } + + // compute if the child is a required field. Determined by the + // presence of "omitempty" in the json tags + hasOmitEmptyTag := false + for i := 1; i < len(childJsonTag); i++ { + if childJsonTag[i] == "omitempty" { + hasOmitEmptyTag = true + } + } + if !hasOmitEmptyTag { + required = append(required, childName) + } + + // compute Schema.Properties for the child recursively + fieldProps, err := safeToSchema(child.Type, childDocs, childName, tracker) + if err != nil { + return nil, err + } + properties[childName] = fieldProps + } + + schema.AdditionalProperties = false + schema.Properties = properties + schema.Required = required + } + + return schema, nil +} diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go new file mode 100644 index 00000000..8edb7568 --- /dev/null +++ b/bundle/schema/schema_test.go @@ -0,0 +1,1299 @@ +package schema + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIntSchema(t *testing.T) { + var elemInt int + + expected := + `{ + "type": "number" + }` + + schema, err := NewSchema(reflect.TypeOf(elemInt), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestBooleanSchema(t *testing.T) { + var elem bool + + expected := + `{ + "type": "boolean" + }` + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestStringSchema(t *testing.T) { + var elem string + + expected := + `{ + "type": "string" + }` + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestStructOfPrimitivesSchema(t *testing.T) { + type Foo struct { + IntVal int `json:"int_val"` + Int8Val int8 `json:"int8_val"` + Int16Val int16 `json:"int16_val"` + Int32Val int32 `json:"int32_val"` + Int64Val int64 `json:"int64_val"` + + UIntVal uint `json:"uint_val"` + Uint8Val uint8 `json:"uint8_val"` + Uint16Val uint16 `json:"uint16_val"` + Uint32Val uint32 `json:"uint32_val"` + Uint64Val uint64 `json:"uint64_val"` + + Float32Val float32 `json:"float32_val"` + Float64Val float64 `json:"float64_val"` + + StringVal string `json:"string_val"` + + BoolVal bool `json:"bool_val"` + } + + elem := Foo{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "bool_val": { + "type": "boolean" + }, + "float32_val": { + "type": "number" + }, + "float64_val": { + "type": "number" + }, + "int16_val": { + "type": "number" + }, + "int32_val": { + "type": "number" + }, + "int64_val": { + "type": "number" + }, + "int8_val": { + "type": "number" + }, + "int_val": { + "type": "number" + }, + "string_val": { + "type": "string" + }, + "uint16_val": { + "type": "number" + }, + "uint32_val": { + "type": "number" + }, + "uint64_val": { + "type": "number" + }, + "uint8_val": { + "type": "number" + }, + "uint_val": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "int_val", + "int8_val", + "int16_val", + "int32_val", + "int64_val", + "uint_val", + "uint8_val", + "uint16_val", + "uint32_val", + "uint64_val", + "float32_val", + "float64_val", + "string_val", + "bool_val" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestStructOfStructsSchema(t *testing.T) { + type Bar struct { + A int `json:"a"` + B string `json:"b,string"` + } + + type Foo struct { + Bar Bar `json:"bar"` + } + + type MyStruct struct { + Foo Foo `json:"foo"` + } + + elem := MyStruct{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "foo": { + "type": "object", + "properties": { + "bar": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "a", + "b" + ] + } + }, + "additionalProperties": false, + "required": [ + "bar" + ] + } + }, + "additionalProperties": false, + "required": [ + "foo" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestStructOfMapsSchema(t *testing.T) { + type Bar struct { + MyMap map[string]int `json:"my_map"` + } + + type Foo struct { + Bar Bar `json:"bar"` + } + + elem := Foo{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "bar": { + "type": "object", + "properties": { + "my_map": { + "type": "object", + "additionalProperties": { + "type": "number" + } + } + }, + "additionalProperties": false, + "required": [ + "my_map" + ] + } + }, + "additionalProperties": false, + "required": [ + "bar" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestStructOfSliceSchema(t *testing.T) { + type Bar struct { + MySlice []string `json:"my_slice"` + } + + type Foo struct { + Bar Bar `json:"bar"` + } + + elem := Foo{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "bar": { + "type": "object", + "properties": { + "my_slice": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "my_slice" + ] + } + }, + "additionalProperties": false, + "required": [ + "bar" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestMapOfPrimitivesSchema(t *testing.T) { + var elem map[string]int + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "additionalProperties": { + "type": "number" + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestMapOfStructSchema(t *testing.T) { + type Foo struct { + MyInt int `json:"my_int"` + } + + var elem map[string]Foo + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "my_int": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "my_int" + ] + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestMapOfMapSchema(t *testing.T) { + var elem map[string]map[string]int + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "number" + } + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestMapOfSliceSchema(t *testing.T) { + var elem map[string][]string + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestSliceOfPrimitivesSchema(t *testing.T) { + var elem []float32 + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "array", + "items": { + "type": "number" + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestSliceOfSliceSchema(t *testing.T) { + var elem [][]string + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestSliceOfMapSchema(t *testing.T) { + var elem []map[string]int + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "type": "number" + } + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestSliceOfStructSchema(t *testing.T) { + type Foo struct { + MyInt int `json:"my_int"` + } + + var elem []Foo + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "array", + "items": { + "type": "object", + "properties": { + "my_int": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "my_int" + ] + } + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestEmbeddedStructSchema(t *testing.T) { + type Location struct { + Country string `json:"country"` + State string `json:"state,omitempty"` + } + + type Person struct { + Name string `json:"name"` + Age int `json:"age,omitempty"` + Home Location `json:"home"` + } + + type Plot struct { + Events map[string]Person `json:"events"` + } + + type Story struct { + Plot Plot `json:"plot"` + *Person + Location + } + + elem := Story{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "country": { + "type": "string" + }, + "home": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "state": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "country" + ] + }, + "name": { + "type": "string" + }, + "plot": { + "type": "object", + "properties": { + "events": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "home": { + "type": "object", + "properties": { + "country": { + "type": "string" + }, + "state": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "country" + ] + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "home" + ] + } + } + }, + "additionalProperties": false, + "required": [ + "events" + ] + }, + "state": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "plot", + "name", + "home", + "country" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestErrorWithTrace(t *testing.T) { + tracker := newTracker() + dummyType := reflect.TypeOf(struct{}{}) + err := tracker.errWithTrace("with empty trace") + assert.ErrorContains(t, err, "[ERROR] with empty trace. traversal trace: root") + + tracker.push(dummyType, "resources") + err = tracker.errWithTrace("with depth = 1") + assert.ErrorContains(t, err, "[ERROR] with depth = 1. traversal trace: root -> resources") + + tracker.push(dummyType, "pipelines") + tracker.push(dummyType, "datasets") + err = tracker.errWithTrace("with depth = 4") + assert.ErrorContains(t, err, "[ERROR] with depth = 4. traversal trace: root -> resources -> pipelines -> datasets") +} + +func TestNonAnnotatedFieldsAreSkipped(t *testing.T) { + type MyStruct struct { + Foo string + Bar int `json:"bar"` + } + + elem := MyStruct{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expectedSchema := + `{ + "type": "object", + "properties": { + "bar": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "bar" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expectedSchema) + + assert.Equal(t, expectedSchema, string(jsonSchema)) +} + +func TestDashFieldsAreSkipped(t *testing.T) { + type MyStruct struct { + Foo string `json:"-"` + Bar int `json:"bar"` + } + + elem := MyStruct{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expectedSchema := + `{ + "type": "object", + "properties": { + "bar": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "bar" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expectedSchema) + + assert.Equal(t, expectedSchema, string(jsonSchema)) +} + +func TestPointerInStructSchema(t *testing.T) { + + type Bar struct { + PtrVal2 *int `json:"ptr_val2"` + } + + type Foo struct { + PtrInt *int `json:"ptr_int"` + PtrString *string `json:"ptr_string"` + FloatVal float32 `json:"float_val"` + PtrBar *Bar `json:"ptr_bar"` + Bar *Bar `json:"bar"` + } + + elem := Foo{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expectedSchema := + `{ + "type": "object", + "properties": { + "bar": { + "type": "object", + "properties": { + "ptr_val2": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "ptr_val2" + ] + }, + "float_val": { + "type": "number" + }, + "ptr_bar": { + "type": "object", + "properties": { + "ptr_val2": { + "type": "number" + } + }, + "additionalProperties": false, + "required": [ + "ptr_val2" + ] + }, + "ptr_int": { + "type": "number" + }, + "ptr_string": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "ptr_int", + "ptr_string", + "float_val", + "ptr_bar", + "bar" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expectedSchema) + + assert.Equal(t, expectedSchema, string(jsonSchema)) +} + +func TestGenericSchema(t *testing.T) { + type Person struct { + Name string `json:"name"` + Age int `json:"age,omitempty"` + } + + type Plot struct { + Stakes []string `json:"stakes"` + Deaths []Person `json:"deaths"` + Murders map[string]Person `json:"murders"` + } + + type Wedding struct { + Hidden string `json:","` + Groom Person `json:"groom"` + Bride Person `json:"bride"` + Plots []Plot `json:"plots"` + } + + type Story struct { + Hero *Person `json:"hero"` + Villian Person `json:"villian,omitempty"` + Weddings []Wedding `json:"weddings"` + } + + elem := Story{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "hero": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "villian": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "weddings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "bride": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "groom": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "plots": { + "type": "array", + "items": { + "type": "object", + "properties": { + "deaths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + } + }, + "murders": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "age": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + } + }, + "stakes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "stakes", + "deaths", + "murders" + ] + } + } + }, + "additionalProperties": false, + "required": [ + "groom", + "bride", + "plots" + ] + } + } + }, + "additionalProperties": false, + "required": [ + "hero", + "weddings" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) { + + type Papaya struct { + A int `json:"a,string,omitempty"` + B string `json:"b"` + } + + type MyStruct struct { + Foo string `json:"-,omitempty"` + Bar int `json:"bar"` + Apple int `json:"apple,omitempty"` + Mango int `json:",omitempty"` + Guava int `json:","` + Papaya *Papaya `json:"papaya,"` + } + + elem := MyStruct{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expectedSchema := + `{ + "type": "object", + "properties": { + "apple": { + "type": "number" + }, + "bar": { + "type": "number" + }, + "papaya": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "b" + ] + } + }, + "additionalProperties": false, + "required": [ + "bar", + "papaya" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expectedSchema) + + assert.Equal(t, expectedSchema, string(jsonSchema)) +} + +func TestDocIngestionInSchema(t *testing.T) { + docs := &Docs{ + Documentation: "docs for root", + Children: map[string]Docs{ + "my_struct": { + Documentation: "docs for my struct", + }, + "my_val": { + Documentation: "docs for my val", + }, + "my_slice": { + Documentation: "docs for my slice", + Children: map[string]Docs{ + "guava": { + Documentation: "docs for guava", + }, + "pineapple": { + Documentation: "docs for pineapple", + }, + }, + }, + "my_map": { + Documentation: "docs for my map", + Children: map[string]Docs{ + "apple": { + Documentation: "docs for apple", + }, + "mango": { + Documentation: "docs for mango", + }, + }, + }, + }, + } + + type Foo struct { + Apple int `json:"apple"` + Mango int `json:"mango"` + } + + type Bar struct { + Guava int `json:"guava"` + Pineapple int `json:"pineapple"` + } + + type MyStruct struct { + A string `json:"a"` + } + + type Root struct { + MyStruct *MyStruct `json:"my_struct"` + MyVal int `json:"my_val"` + MySlice []Bar `json:"my_slice"` + MyMap map[string]*Foo `json:"my_map"` + } + + elem := Root{} + + schema, err := NewSchema(reflect.TypeOf(elem), docs) + require.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expectedSchema := + `{ + "type": "object", + "description": "docs for root", + "properties": { + "my_map": { + "type": "object", + "description": "docs for my map", + "additionalProperties": { + "type": "object", + "description": "docs for my map", + "properties": { + "apple": { + "type": "number", + "description": "docs for apple" + }, + "mango": { + "type": "number", + "description": "docs for mango" + } + }, + "additionalProperties": false, + "required": [ + "apple", + "mango" + ] + } + }, + "my_slice": { + "type": "array", + "description": "docs for my slice", + "items": { + "type": "object", + "properties": { + "guava": { + "type": "number", + "description": "docs for guava" + }, + "pineapple": { + "type": "number", + "description": "docs for pineapple" + } + }, + "additionalProperties": false, + "required": [ + "guava", + "pineapple" + ] + } + }, + "my_struct": { + "type": "object", + "description": "docs for my struct", + "properties": { + "a": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "a" + ] + }, + "my_val": { + "type": "number", + "description": "docs for my val" + } + }, + "additionalProperties": false, + "required": [ + "my_struct", + "my_val", + "my_slice", + "my_map" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expectedSchema) + + assert.Equal(t, expectedSchema, string(jsonSchema)) +} + +func TestErrorOnMapWithoutStringKey(t *testing.T) { + type Foo struct { + Bar map[int]string `json:"bar"` + } + elem := Foo{} + _, err := NewSchema(reflect.TypeOf(elem), nil) + assert.ErrorContains(t, err, "only strings map keys are valid. key type: int") +} + +func TestErrorIfStructRefersToItself(t *testing.T) { + type Foo struct { + MyFoo *Foo `json:"my_foo"` + } + + elem := Foo{} + _, err := NewSchema(reflect.TypeOf(elem), nil) + assert.ErrorContains(t, err, "ERROR] cycle detected. traversal trace: root -> my_foo") +} + +func TestErrorIfStructHasLoop(t *testing.T) { + type Apple struct { + MyVal int `json:"my_val"` + MyMango struct { + MyGuava struct { + MyPapaya struct { + MyApple *Apple `json:"my_apple"` + } `json:"my_papaya"` + } `json:"my_guava"` + } `json:"my_mango"` + } + + elem := Apple{} + _, err := NewSchema(reflect.TypeOf(elem), nil) + assert.ErrorContains(t, err, "[ERROR] cycle detected. traversal trace: root -> my_mango -> my_guava -> my_papaya -> my_apple") +} + +func TestInterfaceGeneratesEmptySchema(t *testing.T) { + type Foo struct { + Apple int `json:"apple"` + Mango interface{} `json:"mango"` + } + + elem := Foo{} + + schema, err := NewSchema(reflect.TypeOf(elem), nil) + assert.NoError(t, err) + + jsonSchema, err := json.MarshalIndent(schema, " ", " ") + assert.NoError(t, err) + + expected := + `{ + "type": "object", + "properties": { + "apple": { + "type": "number" + }, + "mango": {} + }, + "additionalProperties": false, + "required": [ + "apple", + "mango" + ] + }` + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(jsonSchema)) +} + +// A toy test to generate the schema for bundle. Will be removed once we have a +// command to generate the json schema +// func TestBundleSchema(t *testing.T) { +// elem := config.Root{} + +// docs, err := LoadDocs("./bundle_config_docs.yml") +// require.NoError(t, err) + +// schema, err := NewSchema(reflect.TypeOf(elem), docs) +// assert.NoError(t, err) + +// jsonSchema, err := json.MarshalIndent(schema, " ", " ") +// assert.NoError(t, err) + +// t.Log("[DEBUG] actual: ", string(jsonSchema)) +// assert.True(t, false) +// } diff --git a/bundle/schema/tracker.go b/bundle/schema/tracker.go new file mode 100644 index 00000000..8dc4f3f4 --- /dev/null +++ b/bundle/schema/tracker.go @@ -0,0 +1,54 @@ +package schema + +import ( + "container/list" + "fmt" + "reflect" +) + +type tracker struct { + // Types encountered in current path during the recursive traversal. Used to + // check for cycles + seenTypes map[reflect.Type]struct{} + + // List of field names encountered in current path during the recursive traversal. + // Used to hydrate errors with path to the exact node where error occured. + // + // The field names here are the first tag in the json tags of struct field. + debugTrace *list.List +} + +func newTracker() *tracker { + return &tracker{ + seenTypes: map[reflect.Type]struct{}{}, + debugTrace: list.New(), + } +} + +func (t *tracker) errWithTrace(prefix string) error { + traceString := "root" + curr := t.debugTrace.Front() + for curr != nil { + if curr.Value.(string) != "" { + traceString += " -> " + curr.Value.(string) + } + curr = curr.Next() + } + return fmt.Errorf("[ERROR] " + prefix + ". traversal trace: " + traceString) +} + +func (t *tracker) hasCycle(golangType reflect.Type) bool { + _, ok := t.seenTypes[golangType] + return ok +} + +func (t *tracker) push(nodeType reflect.Type, jsonName string) { + t.seenTypes[nodeType] = struct{}{} + t.debugTrace.PushBack(jsonName) +} + +func (t *tracker) pop(nodeType reflect.Type) { + back := t.debugTrace.Back() + t.debugTrace.Remove(back) + delete(t.seenTypes, nodeType) +} diff --git a/go.mod b/go.mod index 16b155c8..20a457fb 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,6 @@ require ( google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 // indirect )