steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added doc ingension with a test

This commit is contained in:
Shreyas Goenka 2023-01-18 16:02:38 +01:00
parent 2789b61e40
commit 1ee80080d2
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
4 changed files with 227 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bundle/config/root.go
View File

@ -48,6 +48,7 @@ type Root struct {
func Load(path string) (*Root, error) {
var r Root
stat, err := os.Stat(path)
if err != nil {
return nil, err

30
bundle/schema/docs.go Normal file
View File

@ -0,0 +1,30 @@
package schema
import (
"io/ioutil"
"os"
"gopkg.in/yaml.v2"
)
type Docs struct {
Documentation string `json:"documentation"`
Children map[string]Docs `json:"children"`
}
func LoadDocs(path string) (*Docs, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
bytes, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
docs := Docs{}
err = yaml.Unmarshal(bytes, &docs)
if err != nil {
return nil, err
}
return &docs, nil
}

45
bundle/schema/schema.go
View File

@ -8,7 +8,6 @@ import (
)
// TODO: Add tests for the error cases, forcefully triggering them
// TODO: Add required validation for omitempty
// TODO: Add example documentation
// TODO: Do final checks for more validation that can be added to json schema
// TODO: Run all tests to see code coverage and add tests for missing assertions
@ -18,6 +17,10 @@ type Schema struct {
// Type of the object
Type JavascriptType `json:"type"`
// Description of the object. This is rendered as inline documentation in the
// IDE
Description string `json:"description,omitempty"`
// keys are named properties of the object
// values are json schema for the values of the named properties
Properties map[string]*Schema `json:"properties,omitempty"`
@ -72,20 +75,14 @@ type Schema struct {
}
for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties
*/
func NewSchema(golangType reflect.Type) (*Schema, error) {
func NewSchema(golangType reflect.Type, docs *Docs) (*Schema, error) {
seenTypes := map[reflect.Type]struct{}{}
debugTrace := list.New()
rootProp, err := toSchema(golangType, seenTypes, debugTrace)
schema, err := toSchema(golangType, docs, seenTypes, debugTrace)
if err != nil {
return nil, errWithTrace(err.Error(), debugTrace)
}
return &Schema{
Type: rootProp.Type,
Properties: rootProp.Properties,
AdditionalProperties: rootProp.AdditionalProperties,
Items: rootProp.Items,
Required: rootProp.Required,
}, nil
return schema, nil
}
type JavascriptType string
@ -136,7 +133,7 @@ func errWithTrace(prefix string, trace *list.List) error {
}
// A wrapper over toSchema function to detect cycles in the bundle config struct
func safeToSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debugTrace *list.List) (*Schema, error) {
func safeToSchema(golangType reflect.Type, docs *Docs, seenTypes map[reflect.Type]struct{}, debugTrace *list.List) (*Schema, error) {
// WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA
// There are mechanisms to deal with cycles though recursive identifiers in json
// schema. However if we use them, we would need to make sure we are able to detect
@ -151,7 +148,7 @@ func safeToSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{},
}
// Update set of types in current path
seenTypes[golangType] = struct{}{}
props, err := toSchema(golangType, seenTypes, debugTrace)
props, err := toSchema(golangType, docs, seenTypes, debugTrace)
if err != nil {
return nil, err
}
@ -202,10 +199,10 @@ func addStructFields(fields []reflect.StructField, golangType reflect.Type) []re
// Used to identify cycles.
// debugTrace: linked list of golang types encounted. In case of errors this
// helps log where the error originated from
func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debugTrace *list.List) (*Schema, error) {
func toSchema(golangType reflect.Type, docs *Docs, seenTypes map[reflect.Type]struct{}, debugTrace *list.List) (*Schema, error) {
// *Struct and Struct generate identical json schemas
if golangType.Kind() == reflect.Pointer {
return toSchema(golangType.Elem(), seenTypes, debugTrace)
return toSchema(golangType.Elem(), docs, seenTypes, debugTrace)
}
// TODO: add test case for interfaces
@ -218,6 +215,11 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
return nil, err
}
var description string
if docs != nil {
description = docs.Documentation
}
// case array/slice
var items *Schema
if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice {
@ -226,7 +228,7 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
if err != nil {
return nil, err
}
elemProps, err := safeToSchema(elemGolangType, seenTypes, debugTrace)
elemProps, err := safeToSchema(elemGolangType, docs, seenTypes, debugTrace)
if err != nil {
return nil, err
}
@ -250,7 +252,7 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
}
// TODO: Add a test for map of maps, and map of slices. Check that there
// is already a test for map of objects and map of primites
additionalProperties, err = safeToSchema(golangType.Elem(), seenTypes, debugTrace)
additionalProperties, err = safeToSchema(golangType.Elem(), docs, seenTypes, debugTrace)
if err != nil {
return nil, err
}
@ -272,6 +274,14 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
continue
}
// get docs for the child if they exist
var childDocs *Docs
if docs != nil {
if val, ok := docs.Children[childName]; ok {
childDocs = &val
}
}
// TODO: Add test for omitempty
hasOmitEmptyTag := false
for i := 1; i < len(childJsonTag); i++ {
@ -289,7 +299,7 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
debugTrace.PushBack(childName)
// recursively compute properties for this child field
fieldProps, err := safeToSchema(child.Type, seenTypes, debugTrace)
fieldProps, err := safeToSchema(child.Type, childDocs, seenTypes, debugTrace)
if err != nil {
return nil, err
}
@ -306,6 +316,7 @@ func toSchema(golangType reflect.Type, seenTypes map[reflect.Type]struct{}, debu
return &Schema{
Type: rootJavascriptType,
Description: description,
Items: items,
Properties: properties,
AdditionalProperties: additionalProperties,

189
bundle/schema/schema_test.go
View File

@ -29,7 +29,7 @@ func TestIntSchema(t *testing.T) {
"type": "number"
}`
Int, err := NewSchema(reflect.TypeOf(elemInt))
Int, err := NewSchema(reflect.TypeOf(elemInt), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(Int, " ", " ")
@ -48,7 +48,7 @@ func TestBooleanSchema(t *testing.T) {
"type": "boolean"
}`
Int, err := NewSchema(reflect.TypeOf(elem))
Int, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(Int, " ", " ")
@ -67,7 +67,7 @@ func TestStringSchema(t *testing.T) {
"type": "string"
}`
Int, err := NewSchema(reflect.TypeOf(elem))
Int, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(Int, " ", " ")
@ -102,7 +102,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
elem := Foo{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -195,7 +195,7 @@ func TestStructOfStructsSchema(t *testing.T) {
elem := MyStruct{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -253,7 +253,7 @@ func TestStructOfMapsSchema(t *testing.T) {
elem := Foo{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -301,7 +301,7 @@ func TestStructOfSliceSchema(t *testing.T) {
elem := Foo{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -341,7 +341,7 @@ func TestStructOfSliceSchema(t *testing.T) {
func TestMapOfPrimitivesSchema(t *testing.T) {
var elem map[string]int
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -367,7 +367,7 @@ func TestMapOfStructSchema(t *testing.T) {
var elem map[string]Foo
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -398,7 +398,7 @@ func TestMapOfStructSchema(t *testing.T) {
func TestMapOfMapSchema(t *testing.T) {
var elem map[string]map[string]int
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -423,7 +423,7 @@ func TestMapOfMapSchema(t *testing.T) {
func TestMapOfSliceSchema(t *testing.T) {
var elem map[string][]string
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -448,7 +448,7 @@ func TestMapOfSliceSchema(t *testing.T) {
func TestSliceOfPrimitivesSchema(t *testing.T) {
var elem []float32
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -470,7 +470,7 @@ func TestSliceOfPrimitivesSchema(t *testing.T) {
func TestSliceOfSliceSchema(t *testing.T) {
var elem [][]string
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -495,7 +495,7 @@ func TestSliceOfSliceSchema(t *testing.T) {
func TestSliceOfMapSchema(t *testing.T) {
var elem []map[string]int
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -524,7 +524,7 @@ func TestSliceOfStructSchema(t *testing.T) {
var elem []Foo
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -576,7 +576,7 @@ func TestEmbeddedStructSchema(t *testing.T) {
elem := Story{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -694,7 +694,7 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) {
elem := MyStruct{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -728,7 +728,7 @@ func TestDashFieldsAreSkipped(t *testing.T) {
elem := MyStruct{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -770,7 +770,7 @@ func TestPointerInStructSchema(t *testing.T) {
elem := Foo{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -849,7 +849,7 @@ func TestObjectSchema(t *testing.T) {
elem := Story{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
assert.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -936,7 +936,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
elem := MyStruct{}
schema, err := NewSchema(reflect.TypeOf(elem))
schema, err := NewSchema(reflect.TypeOf(elem), nil)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
@ -981,6 +981,153 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
assert.Equal(t, expectedSchema, string(jsonSchema))
}
func TestDocIngestionInSchema(t *testing.T) {
docs := &Docs{
Documentation: "docs for root",
Children: map[string]Docs{
"my_struct": {
Documentation: "docs for my struct",
},
"my_val": {
Documentation: "docs for my val",
},
"my_slice": {
Documentation: "docs for my slice",
Children: map[string]Docs{
"guava": {
Documentation: "docs for guava",
},
"pineapple": {
Documentation: "docs for pineapple",
},
},
},
"my_map": {
Documentation: "docs for my map",
Children: map[string]Docs{
"apple": {
Documentation: "docs for apple",
},
"mango": {
Documentation: "docs for mango",
},
},
},
},
}
type Foo struct {
Apple int `json:"apple"`
Mango int `json:"mango"`
}
type Bar struct {
Guava int `json:"guava"`
Pineapple int `json:"pineapple"`
}
type MyStruct struct {
A string `json:"a"`
}
type Root struct {
MyStruct *MyStruct `json:"my_struct"`
MyVal int `json:"my_val"`
MySlice []Bar `json:"my_slice"`
MyMap map[string]*Foo `json:"my_map"`
}
elem := Root{}
schema, err := NewSchema(reflect.TypeOf(elem), docs)
require.NoError(t, err)
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
assert.NoError(t, err)
expectedSchema :=
`{
"type": "object",
"description": "docs for root",
"properties": {
"my_map": {
"type": "object",
"description": "docs for my map",
"additionalProperties": {
"type": "object",
"description": "docs for my map",
"properties": {
"apple": {
"type": "number",
"description": "docs for apple"
},
"mango": {
"type": "number",
"description": "docs for mango"
}
},
"additionalProperties": false,
"required": [
"apple",
"mango"
]
}
},
"my_slice": {
"type": "array",
"description": "docs for my slice",
"items": {
"type": "object",
"properties": {
"guava": {
"type": "number",
"description": "docs for guava"
},
"pineapple": {
"type": "number",
"description": "docs for pineapple"
}
},
"additionalProperties": false,
"required": [
"guava",
"pineapple"
]
}
},
"my_struct": {
"type": "object",
"description": "docs for my struct",
"properties": {
"a": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"a"
]
},
"my_val": {
"type": "number",
"description": "docs for my val"
}
},
"additionalProperties": false,
"required": [
"my_struct",
"my_val",
"my_slice",
"my_map"
]
}`
t.Log("[DEBUG] actual: ", string(jsonSchema))
t.Log("[DEBUG] expected: ", expectedSchema)
assert.Equal(t, expectedSchema, string(jsonSchema))
}
// // Only for testing bundle, will be removed
// func TestBundleSchema(t *testing.T) {
// elem := config.Root{}