From 83fb89ad3bc6f178053451d2c78b45cea3bb6d31 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 23 Jan 2023 15:00:11 +0100 Subject: [PATCH] Add command for generating JSON schema for DABs bundle config (#171) In the future can add a path flag to generate subschemas. Might be useful depending on how config splits are supported --- bundle/schema/docs.go | 15 +++++++- bundle/schema/schema.go | 2 +- bundle/schema/schema_test.go | 70 ++++++++++++++---------------------- cmd/bundle/schema.go | 36 +++++++++++++++++++ go.mod | 4 +-- 5 files changed, 79 insertions(+), 48 deletions(-) create mode 100644 cmd/bundle/schema.go diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index 2c35dd16..8522b17b 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -1,9 +1,10 @@ package schema import ( + _ "embed" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type Docs struct { @@ -23,3 +24,15 @@ func LoadDocs(path string) (*Docs, error) { } return &docs, nil } + +//go:embed bundle_config_docs.yml +var bundleDocs []byte + +func GetBundleDocs() (*Docs, error) { + docs := Docs{} + err := yaml.Unmarshal(bundleDocs, &docs) + if err != nil { + return nil, err + } + return &docs, nil +} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index ef939972..d6d0295d 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -58,7 +58,7 @@ type Schema struct { // // - []MyStruct -> {type: object, properties: {}, additionalProperties: false} // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties -func NewSchema(golangType reflect.Type, docs *Docs) (*Schema, error) { +func New(golangType reflect.Type, docs *Docs) (*Schema, error) { tracker := newTracker() schema, err := safeToSchema(golangType, docs, "", tracker) if err != nil { diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 8edb7568..1539e35f 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -17,7 +17,7 @@ func TestIntSchema(t *testing.T) { "type": "number" }` - schema, err := NewSchema(reflect.TypeOf(elemInt), nil) + schema, err := New(reflect.TypeOf(elemInt), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -36,7 +36,7 @@ func TestBooleanSchema(t *testing.T) { "type": "boolean" }` - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -55,7 +55,7 @@ func TestStringSchema(t *testing.T) { "type": "string" }` - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -90,7 +90,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) { elem := Foo{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -183,7 +183,7 @@ func TestStructOfStructsSchema(t *testing.T) { elem := MyStruct{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -241,7 +241,7 @@ func TestStructOfMapsSchema(t *testing.T) { elem := Foo{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -289,7 +289,7 @@ func TestStructOfSliceSchema(t *testing.T) { elem := Foo{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -329,7 +329,7 @@ func TestStructOfSliceSchema(t *testing.T) { func TestMapOfPrimitivesSchema(t *testing.T) { var elem map[string]int - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -355,7 +355,7 @@ func TestMapOfStructSchema(t *testing.T) { var elem map[string]Foo - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -386,7 +386,7 @@ func TestMapOfStructSchema(t *testing.T) { func TestMapOfMapSchema(t *testing.T) { var elem map[string]map[string]int - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -411,7 +411,7 @@ func TestMapOfMapSchema(t *testing.T) { func TestMapOfSliceSchema(t *testing.T) { var elem map[string][]string - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -436,7 +436,7 @@ func TestMapOfSliceSchema(t *testing.T) { func TestSliceOfPrimitivesSchema(t *testing.T) { var elem []float32 - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -458,7 +458,7 @@ func TestSliceOfPrimitivesSchema(t *testing.T) { func TestSliceOfSliceSchema(t *testing.T) { var elem [][]string - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -483,7 +483,7 @@ func TestSliceOfSliceSchema(t *testing.T) { func TestSliceOfMapSchema(t *testing.T) { var elem []map[string]int - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -512,7 +512,7 @@ func TestSliceOfStructSchema(t *testing.T) { var elem []Foo - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -564,7 +564,7 @@ func TestEmbeddedStructSchema(t *testing.T) { elem := Story{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -683,7 +683,7 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) { elem := MyStruct{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -717,7 +717,7 @@ func TestDashFieldsAreSkipped(t *testing.T) { elem := MyStruct{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -759,7 +759,7 @@ func TestPointerInStructSchema(t *testing.T) { elem := Foo{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -846,7 +846,7 @@ func TestGenericSchema(t *testing.T) { elem := Story{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -1017,7 +1017,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) { elem := MyStruct{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -1120,7 +1120,7 @@ func TestDocIngestionInSchema(t *testing.T) { elem := Root{} - schema, err := NewSchema(reflect.TypeOf(elem), docs) + schema, err := New(reflect.TypeOf(elem), docs) require.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -1214,7 +1214,7 @@ func TestErrorOnMapWithoutStringKey(t *testing.T) { Bar map[int]string `json:"bar"` } elem := Foo{} - _, err := NewSchema(reflect.TypeOf(elem), nil) + _, err := New(reflect.TypeOf(elem), nil) assert.ErrorContains(t, err, "only strings map keys are valid. key type: int") } @@ -1224,7 +1224,7 @@ func TestErrorIfStructRefersToItself(t *testing.T) { } elem := Foo{} - _, err := NewSchema(reflect.TypeOf(elem), nil) + _, err := New(reflect.TypeOf(elem), nil) assert.ErrorContains(t, err, "ERROR] cycle detected. traversal trace: root -> my_foo") } @@ -1241,7 +1241,7 @@ func TestErrorIfStructHasLoop(t *testing.T) { } elem := Apple{} - _, err := NewSchema(reflect.TypeOf(elem), nil) + _, err := New(reflect.TypeOf(elem), nil) assert.ErrorContains(t, err, "[ERROR] cycle detected. traversal trace: root -> my_mango -> my_guava -> my_papaya -> my_apple") } @@ -1253,7 +1253,7 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) { elem := Foo{} - schema, err := NewSchema(reflect.TypeOf(elem), nil) + schema, err := New(reflect.TypeOf(elem), nil) assert.NoError(t, err) jsonSchema, err := json.MarshalIndent(schema, " ", " ") @@ -1279,21 +1279,3 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) { t.Log("[DEBUG] expected: ", expected) assert.Equal(t, expected, string(jsonSchema)) } - -// A toy test to generate the schema for bundle. Will be removed once we have a -// command to generate the json schema -// func TestBundleSchema(t *testing.T) { -// elem := config.Root{} - -// docs, err := LoadDocs("./bundle_config_docs.yml") -// require.NoError(t, err) - -// schema, err := NewSchema(reflect.TypeOf(elem), docs) -// assert.NoError(t, err) - -// jsonSchema, err := json.MarshalIndent(schema, " ", " ") -// assert.NoError(t, err) - -// t.Log("[DEBUG] actual: ", string(jsonSchema)) -// assert.True(t, false) -// } diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go new file mode 100644 index 00000000..e3c19c46 --- /dev/null +++ b/cmd/bundle/schema.go @@ -0,0 +1,36 @@ +package bundle + +import ( + "encoding/json" + "reflect" + + "github.com/databricks/bricks/bundle/config" + "github.com/databricks/bricks/bundle/schema" + "github.com/spf13/cobra" +) + +var schemaCmd = &cobra.Command{ + Use: "schema", + Short: "Generate JSON Schema for bundle configuration", + + RunE: func(cmd *cobra.Command, args []string) error { + docs, err := schema.GetBundleDocs() + if err != nil { + return err + } + schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) + if err != nil { + return err + } + jsonSchema, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return err + } + cmd.OutOrStdout().Write(jsonSchema) + return nil + }, +} + +func init() { + AddCommand(schemaCmd) +} diff --git a/go.mod b/go.mod index 20a457fb..cf8f4fb8 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,6 @@ require ( google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 )