mirror of https://github.com/databricks/cli.git
more iteration
This commit is contained in:
parent
535f670868
commit
11dfdc036d
|
@ -1,42 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/databricks/cli/bundle/schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
if len(os.Args) != 2 {
|
|
||||||
fmt.Println("Usage: go run main.go <output-file>")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output file, to write the generated schema descriptions to.
|
|
||||||
outputFile := os.Args[1]
|
|
||||||
|
|
||||||
// Input file, the databricks openapi spec.
|
|
||||||
inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
|
|
||||||
if inputFile == "" {
|
|
||||||
log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate the schema descriptions.
|
|
||||||
docs, err := schema.UpdateBundleDescriptions(inputFile)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
result, err := json.MarshalIndent(docs, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the schema descriptions to the output file.
|
|
||||||
err = os.WriteFile(outputFile, result, 0644)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,96 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/libs/jsonschema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func interpolationPattern(s string) string {
|
||||||
|
return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
|
||||||
|
switch s.Type {
|
||||||
|
case jsonschema.ArrayType, jsonschema.ObjectType:
|
||||||
|
// arrays and objects can have complex variable values specified.
|
||||||
|
return jsonschema.Schema{
|
||||||
|
AnyOf: []jsonschema.Schema{s, {
|
||||||
|
Type: jsonschema.StringType,
|
||||||
|
// TODO: Are multi-level complex variable references supported?
|
||||||
|
Pattern: interpolationPattern("var"),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
case jsonschema.StringType, jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType:
|
||||||
|
// primitives can have variable values, or references like ${bundle.xyz}
|
||||||
|
// or ${workspace.xyz}
|
||||||
|
// TODO: Followup, do not allow references like ${} in the schema unless
|
||||||
|
// they are of the permitted patterns?
|
||||||
|
return jsonschema.Schema{
|
||||||
|
AnyOf: []jsonschema.Schema{s,
|
||||||
|
// TODO: Add "resources" here
|
||||||
|
{Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")},
|
||||||
|
{Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")},
|
||||||
|
{Type: jsonschema.StringType, Pattern: interpolationPattern("var")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Add a couple of end to end tests that the bundle schema generated is
|
||||||
|
// correct.
|
||||||
|
// TODO: Call out in the PR description that recursive types like "for_each_task"
|
||||||
|
// are now supported. Manually test for_each_task.
|
||||||
|
// TODO: The bundle_descriptions.json file contains a bunch of custom descriptions
|
||||||
|
// as well. Make sure to pull those in.
|
||||||
|
// TODO: Add unit tests for all permutations of structs, maps and slices for the FromType
|
||||||
|
// method.
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) != 2 {
|
||||||
|
fmt.Println("Usage: go run main.go <output-file>")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output file, where the generated JSON schema will be written to.
|
||||||
|
outputFile := os.Args[1]
|
||||||
|
|
||||||
|
// Input file, the databricks openapi spec.
|
||||||
|
inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
|
||||||
|
if inputFile == "" {
|
||||||
|
log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := newParser(inputFile)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate the JSON schema from the bundle Go struct.
|
||||||
|
s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
|
||||||
|
p.addDescriptions,
|
||||||
|
p.addEnums,
|
||||||
|
addInterpolationPatterns,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.MarshalIndent(s, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the schema descriptions to the output file.
|
||||||
|
err = os.WriteFile(outputFile, b, 0644)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,131 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/libs/jsonschema"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Components struct {
|
||||||
|
Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Specification struct {
|
||||||
|
Components Components `json:"components"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type openapiParser struct {
|
||||||
|
ref map[string]jsonschema.Schema
|
||||||
|
}
|
||||||
|
|
||||||
|
func newParser(path string) (*openapiParser, error) {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
spec := Specification{}
|
||||||
|
err = json.Unmarshal(b, &spec)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
p := &openapiParser{}
|
||||||
|
p.ref = spec.Components.Schemas
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// This function finds any JSON schemas that were defined in the OpenAPI spec
|
||||||
|
// that correspond to the given Go SDK type. It looks both at the type itself
|
||||||
|
// and any embedded types within it.
|
||||||
|
func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) {
|
||||||
|
typs := []reflect.Type{typ}
|
||||||
|
|
||||||
|
// If the type is a struct, the corresponding Go SDK struct might be embedded
|
||||||
|
// in it. We need to check for those as well.
|
||||||
|
if typ.Kind() == reflect.Struct {
|
||||||
|
for i := 0; i < typ.NumField(); i++ {
|
||||||
|
if !typ.Field(i).Anonymous {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deference current type if it's a pointer.
|
||||||
|
ctyp := typ.Field(i).Type
|
||||||
|
for ctyp.Kind() == reflect.Ptr {
|
||||||
|
ctyp = ctyp.Elem()
|
||||||
|
}
|
||||||
|
|
||||||
|
typs = append(typs, ctyp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ctyp := range typs {
|
||||||
|
// Skip if it's not a Go SDK type.
|
||||||
|
if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
pkgName := path.Base(ctyp.PkgPath())
|
||||||
|
k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name())
|
||||||
|
|
||||||
|
// Skip if the type is not in the openapi spec.
|
||||||
|
_, ok := p.ref[k]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the first Go SDK type found in the openapi spec.
|
||||||
|
return p.ref[k], true
|
||||||
|
}
|
||||||
|
|
||||||
|
return jsonschema.Schema{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use the OpenAPI spec to load descriptions for the given type.
|
||||||
|
func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
|
||||||
|
ref, ok := p.findRef(typ)
|
||||||
|
if !ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
s.Description = ref.Description
|
||||||
|
|
||||||
|
// Iterate over properties to load descriptions. This is not needed for any
|
||||||
|
// OpenAPI spec generated from protobufs, which are guaranteed to be one level
|
||||||
|
// deep.
|
||||||
|
// Needed for any hand-written OpenAPI specs.
|
||||||
|
for k, v := range s.Properties {
|
||||||
|
if refProp, ok := ref.Properties[k]; ok {
|
||||||
|
v.Description = refProp.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use the OpenAPI spec add enum values for the given type.
|
||||||
|
func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
|
||||||
|
ref, ok := p.findRef(typ)
|
||||||
|
if !ok {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
s.Enum = append(s.Enum, ref.Enum...)
|
||||||
|
|
||||||
|
// Iterate over properties to load enums. This is not needed for any
|
||||||
|
// OpenAPI spec generated from protobufs, which are guaranteed to be one level
|
||||||
|
// deep.
|
||||||
|
// Needed for any hand-written OpenAPI specs.
|
||||||
|
for k, v := range s.Properties {
|
||||||
|
if refProp, ok := ref.Properties[k]; ok {
|
||||||
|
v.Enum = append(v.Enum, refProp.Enum...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
|
@ -3,7 +3,6 @@ package schema
|
||||||
import (
|
import (
|
||||||
"reflect"
|
"reflect"
|
||||||
|
|
||||||
"github.com/databricks/cli/libs/dyn/dynvar"
|
|
||||||
"github.com/databricks/cli/libs/jsonschema"
|
"github.com/databricks/cli/libs/jsonschema"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -42,26 +41,28 @@ import (
|
||||||
// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties
|
// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties
|
||||||
func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) {
|
func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) {
|
||||||
|
|
||||||
s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema {
|
return nil, nil
|
||||||
if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType {
|
|
||||||
s = jsonschema.Schema{
|
// s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema {
|
||||||
AnyOf: []jsonschema.Schema{
|
// if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType {
|
||||||
s,
|
// s = jsonschema.Schema{
|
||||||
{
|
// AnyOf: []jsonschema.Schema{
|
||||||
Type: jsonschema.StringType,
|
// s,
|
||||||
// TODO: Narrow down the scope of the regex match.
|
// {
|
||||||
// Also likely need to rename this variable.
|
// Type: jsonschema.StringType,
|
||||||
Pattern: dynvar.VariableRegex,
|
// // TODO: Narrow down the scope of the regex match.
|
||||||
},
|
// // Also likely need to rename this variable.
|
||||||
},
|
// Pattern: dynvar.ReferenceRegex,
|
||||||
}
|
// },
|
||||||
}
|
// },
|
||||||
return s
|
// }
|
||||||
})
|
// }
|
||||||
if err != nil {
|
// return s
|
||||||
return nil, err
|
// })
|
||||||
}
|
// if err != nil {
|
||||||
return &s, nil
|
// return nil, err
|
||||||
|
// }
|
||||||
|
// return &s, nil
|
||||||
|
|
||||||
// tracker := newTracker()
|
// tracker := newTracker()
|
||||||
// schema, err := safeToSchema(golangType, docs, "", tracker)
|
// schema, err := safeToSchema(golangType, docs, "", tracker)
|
||||||
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"github.com/databricks/cli/libs/dyn"
|
"github.com/databricks/cli/libs/dyn"
|
||||||
)
|
)
|
||||||
|
|
||||||
const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`
|
const ReferenceRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`
|
||||||
|
|
||||||
var re = regexp.MustCompile(VariableRegex)
|
var re = regexp.MustCompile(ReferenceRegex)
|
||||||
|
|
||||||
// ref represents a variable reference.
|
// ref represents a variable reference.
|
||||||
// It is a string [dyn.Value] contained in a larger [dyn.Value].
|
// It is a string [dyn.Value] contained in a larger [dyn.Value].
|
||||||
|
|
|
@ -3,17 +3,13 @@ package jsonschema
|
||||||
import (
|
import (
|
||||||
"container/list"
|
"container/list"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
"path"
|
"path"
|
||||||
"reflect"
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO: Maybe can be removed?
|
|
||||||
var InvalidSchema = Schema{
|
|
||||||
Type: InvalidType,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fields tagged "readonly" should not be emitted in the schema as they are
|
// Fields tagged "readonly" should not be emitted in the schema as they are
|
||||||
// computed at runtime, and should not be assigned a value by the bundle author.
|
// computed at runtime, and should not be assigned a value by the bundle author.
|
||||||
const readonlyTag = "readonly"
|
const readonlyTag = "readonly"
|
||||||
|
@ -26,19 +22,17 @@ const internalTag = "internal"
|
||||||
// Fields tagged as "deprecated" are removed/omitted from the generated schema.
|
// Fields tagged as "deprecated" are removed/omitted from the generated schema.
|
||||||
const deprecatedTag = "deprecated"
|
const deprecatedTag = "deprecated"
|
||||||
|
|
||||||
// TODO: Test what happens with invalid cycles? Do integration tests fail?
|
|
||||||
// TODO: Call out in the PR description that recursive types like "for_each_task"
|
|
||||||
// are now supported.
|
|
||||||
|
|
||||||
type constructor struct {
|
type constructor struct {
|
||||||
// Map of typ.PkgPath() + "." + typ.Name() to the schema for that type.
|
// Map of typ.PkgPath() + "." + typ.Name() to the schema for that type.
|
||||||
// Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings
|
// Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings
|
||||||
definitions map[string]Schema
|
definitions map[string]Schema
|
||||||
|
|
||||||
seen map[string]struct{}
|
// Map of typ.PkgPath() + "." + typ.Name() to the corresponding type. Used to
|
||||||
|
// track types that have been seen to avoid infinite recursion.
|
||||||
|
seen map[string]reflect.Type
|
||||||
|
|
||||||
// Transformation function to apply after generating a node in the schema.
|
// The root type for which the schema is being generated.
|
||||||
fn func(s Schema) Schema
|
root reflect.Type
|
||||||
}
|
}
|
||||||
|
|
||||||
// The $defs block in a JSON schema cannot contain "/", otherwise it will not be
|
// The $defs block in a JSON schema cannot contain "/", otherwise it will not be
|
||||||
|
@ -47,13 +41,19 @@ type constructor struct {
|
||||||
//
|
//
|
||||||
// For example:
|
// For example:
|
||||||
// {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}}
|
// {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}}
|
||||||
func (c *constructor) nestedDefinitions() any {
|
func (c *constructor) Definitions() any {
|
||||||
if len(c.definitions) == 0 {
|
defs := maps.Clone(c.definitions)
|
||||||
|
|
||||||
|
// Remove the root type from the definitions. No need to include it in the
|
||||||
|
// definitions.
|
||||||
|
delete(defs, typePath(c.root))
|
||||||
|
|
||||||
|
if len(defs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
res := make(map[string]any)
|
res := make(map[string]any)
|
||||||
for k, v := range c.definitions {
|
for k, v := range defs {
|
||||||
parts := strings.Split(k, "/")
|
parts := strings.Split(k, "/")
|
||||||
cur := res
|
cur := res
|
||||||
for i, p := range parts {
|
for i, p := range parts {
|
||||||
|
@ -72,47 +72,74 @@ func (c *constructor) nestedDefinitions() any {
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Skip generating schema for interface fields.
|
// FromType converts a reflect.Type to a jsonschema.Schema. Nodes in the final JSON
|
||||||
func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) {
|
// schema are guaranteed to be one level deep, which is done using defining $defs
|
||||||
|
// for every Go type and referring them using $ref in the corresponding node in
|
||||||
|
// the JSON schema.
|
||||||
|
//
|
||||||
|
// fns is a list of transformation functions that will be applied to all $defs
|
||||||
|
// in the schema.
|
||||||
|
func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (Schema, error) {
|
||||||
c := constructor{
|
c := constructor{
|
||||||
definitions: make(map[string]Schema),
|
definitions: make(map[string]Schema),
|
||||||
seen: make(map[string]struct{}),
|
seen: make(map[string]reflect.Type),
|
||||||
fn: fn,
|
root: typ,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := c.walk(typ)
|
err := c.walk(typ)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return InvalidSchema, err
|
return Schema{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for k, v := range c.definitions {
|
||||||
|
for _, fn := range fns {
|
||||||
|
c.definitions[k] = fn(c.seen[k], v)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res := c.definitions[typePath(typ)]
|
res := c.definitions[typePath(typ)]
|
||||||
// No need to include the root type in the definitions.
|
res.Definitions = c.Definitions()
|
||||||
delete(c.definitions, typePath(typ))
|
|
||||||
res.Definitions = c.nestedDefinitions()
|
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// typePath computes a unique string representation of the type. $ref in the generated
|
||||||
|
// JSON schema will refer to this path. See TestTypePath for examples outputs.
|
||||||
func typePath(typ reflect.Type) string {
|
func typePath(typ reflect.Type) string {
|
||||||
// Pointers have a typ.Name() of "". Dereference them to get the underlying type.
|
// Pointers have a typ.Name() of "". Dereference them to get the underlying type.
|
||||||
for typ.Kind() == reflect.Ptr {
|
for typ.Kind() == reflect.Ptr {
|
||||||
typ = typ.Elem()
|
typ = typ.Elem()
|
||||||
}
|
}
|
||||||
|
|
||||||
// typ.Name() resolves to "" for any type.
|
|
||||||
if typ.Kind() == reflect.Interface {
|
if typ.Kind() == reflect.Interface {
|
||||||
return "interface"
|
return "interface"
|
||||||
}
|
}
|
||||||
|
|
||||||
// For built-in types, return the type name directly.
|
// Recursively call typePath, to handle slices of slices / maps.
|
||||||
if typ.PkgPath() == "" {
|
if typ.Kind() == reflect.Slice {
|
||||||
return typ.Name()
|
return path.Join("slice", typePath(typ.Elem()))
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.Join([]string{typ.PkgPath(), typ.Name()}, ".")
|
if typ.Kind() == reflect.Map {
|
||||||
|
if typ.Key().Kind() != reflect.String {
|
||||||
|
panic(fmt.Sprintf("found map with non-string key: %v", typ.Key()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively call typePath, to handle maps of maps / slices.
|
||||||
|
return path.Join("map", typePath(typ.Elem()))
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case typ.PkgPath() != "" && typ.Name() != "":
|
||||||
|
return typ.PkgPath() + "." + typ.Name()
|
||||||
|
case typ.Name() != "":
|
||||||
|
return typ.Name()
|
||||||
|
default:
|
||||||
|
panic("unexpected empty type name for type: " + typ.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: would a worked based model fit better here? Is this internal API not
|
// Walk the Go type, generating $defs for every type encountered, and populating
|
||||||
// the right fit?
|
// the corresponding $ref in the JSON schema.
|
||||||
func (c *constructor) walk(typ reflect.Type) error {
|
func (c *constructor) walk(typ reflect.Type) error {
|
||||||
// Dereference pointers if necessary.
|
// Dereference pointers if necessary.
|
||||||
for typ.Kind() == reflect.Ptr {
|
for typ.Kind() == reflect.Ptr {
|
||||||
|
@ -121,10 +148,11 @@ func (c *constructor) walk(typ reflect.Type) error {
|
||||||
|
|
||||||
typPath := typePath(typ)
|
typPath := typePath(typ)
|
||||||
|
|
||||||
// Keep track of seen types to avoid infinite recursion.
|
// Return early if the type has already been seen, to avoid infinite recursion.
|
||||||
if _, ok := c.seen[typPath]; !ok {
|
if _, ok := c.seen[typPath]; ok {
|
||||||
c.seen[typPath] = struct{}{}
|
return nil
|
||||||
}
|
}
|
||||||
|
c.seen[typPath] = typ
|
||||||
|
|
||||||
// Return early directly if it's already been processed.
|
// Return early directly if it's already been processed.
|
||||||
if _, ok := c.definitions[typPath]; ok {
|
if _, ok := c.definitions[typPath]; ok {
|
||||||
|
@ -134,7 +162,6 @@ func (c *constructor) walk(typ reflect.Type) error {
|
||||||
var s Schema
|
var s Schema
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// TODO: Narrow / widen down the number of Go types handled here.
|
|
||||||
switch typ.Kind() {
|
switch typ.Kind() {
|
||||||
case reflect.Struct:
|
case reflect.Struct:
|
||||||
s, err = c.fromTypeStruct(typ)
|
s, err = c.fromTypeStruct(typ)
|
||||||
|
@ -142,20 +169,20 @@ func (c *constructor) walk(typ reflect.Type) error {
|
||||||
s, err = c.fromTypeSlice(typ)
|
s, err = c.fromTypeSlice(typ)
|
||||||
case reflect.Map:
|
case reflect.Map:
|
||||||
s, err = c.fromTypeMap(typ)
|
s, err = c.fromTypeMap(typ)
|
||||||
// TODO: Should the primitive functions below be inlined?
|
|
||||||
case reflect.String:
|
case reflect.String:
|
||||||
s = Schema{Type: StringType}
|
s = Schema{Type: StringType}
|
||||||
case reflect.Bool:
|
case reflect.Bool:
|
||||||
s = Schema{Type: BooleanType}
|
s = Schema{Type: BooleanType}
|
||||||
// TODO: Add comment about reduced coverage of primitive Go types in the code paths here.
|
case reflect.Int, reflect.Int32, reflect.Int64:
|
||||||
case reflect.Int:
|
|
||||||
s = Schema{Type: IntegerType}
|
s = Schema{Type: IntegerType}
|
||||||
case reflect.Float32, reflect.Float64:
|
case reflect.Float32, reflect.Float64:
|
||||||
s = Schema{Type: NumberType}
|
s = Schema{Type: NumberType}
|
||||||
case reflect.Interface:
|
case reflect.Interface:
|
||||||
// An interface value can never be serialized from text, and thus is explicitly
|
// Interface or any types are not serialized to JSON by the default JSON
|
||||||
// set to null and disallowed in the schema.
|
// unmarshaller (json.Unmarshal). They likely thus are parsed by the
|
||||||
s = Schema{Type: NullType}
|
// dynamic configuration tree and we should support arbitary values here.
|
||||||
|
// Eg: variables.default can be anything.
|
||||||
|
s = Schema{}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported type: %s", typ.Kind())
|
return fmt.Errorf("unsupported type: %s", typ.Kind())
|
||||||
}
|
}
|
||||||
|
@ -163,13 +190,7 @@ func (c *constructor) walk(typ reflect.Type) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.fn != nil {
|
// Store the computed JSON schema for the type.
|
||||||
s = c.fn(s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Store definition for the type if it's part of a Go package and not a built-in type.
|
|
||||||
// TODO: Apply transformation at the end, to all definitions instead of
|
|
||||||
// during recursive traversal?
|
|
||||||
c.definitions[typPath] = s
|
c.definitions[typPath] = s
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -206,20 +227,15 @@ func getStructFields(typ reflect.Type) []reflect.StructField {
|
||||||
return fields
|
return fields
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: get rid of the errors here and panic instead?
|
|
||||||
func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
|
func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
|
||||||
if typ.Kind() != reflect.Struct {
|
if typ.Kind() != reflect.Struct {
|
||||||
return InvalidSchema, fmt.Errorf("expected struct, got %s", typ.Kind())
|
return Schema{}, fmt.Errorf("expected struct, got %s", typ.Kind())
|
||||||
}
|
}
|
||||||
|
|
||||||
res := Schema{
|
res := Schema{
|
||||||
Type: ObjectType,
|
Type: ObjectType,
|
||||||
|
Properties: make(map[string]*Schema),
|
||||||
Properties: make(map[string]*Schema),
|
Required: []string{},
|
||||||
|
|
||||||
// TODO: Confirm that empty arrays are not serialized.
|
|
||||||
Required: []string{},
|
|
||||||
|
|
||||||
AdditionalProperties: false,
|
AdditionalProperties: false,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,6 +256,7 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
|
||||||
if jsonTags[0] == "" || jsonTags[0] == "-" || !structField.IsExported() {
|
if jsonTags[0] == "" || jsonTags[0] == "-" || !structField.IsExported() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// "omitempty" tags in the Go SDK structs represent fields that not are
|
// "omitempty" tags in the Go SDK structs represent fields that not are
|
||||||
// required to be present in the API payload. Thus its absence in the
|
// required to be present in the API payload. Thus its absence in the
|
||||||
// tags list indicates that the field is required.
|
// tags list indicates that the field is required.
|
||||||
|
@ -247,19 +264,16 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
|
||||||
res.Required = append(res.Required, jsonTags[0])
|
res.Required = append(res.Required, jsonTags[0])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Walk the fields of the struct.
|
||||||
typPath := typePath(structField.Type)
|
typPath := typePath(structField.Type)
|
||||||
// Only walk if the type has not been seen yet.
|
err := c.walk(structField.Type)
|
||||||
if _, ok := c.seen[typPath]; !ok {
|
if err != nil {
|
||||||
// Trigger call to fromType, to recursively generate definitions for
|
return Schema{}, err
|
||||||
// the struct field.
|
|
||||||
err := c.walk(structField.Type)
|
|
||||||
if err != nil {
|
|
||||||
return InvalidSchema, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For every property in the struct, add a $ref to the corresponding
|
||||||
|
// $defs block.
|
||||||
refPath := path.Join("#/$defs", typPath)
|
refPath := path.Join("#/$defs", typPath)
|
||||||
// For non-built-in types, refer to the definition.
|
|
||||||
res.Properties[jsonTags[0]] = &Schema{
|
res.Properties[jsonTags[0]] = &Schema{
|
||||||
Reference: &refPath,
|
Reference: &refPath,
|
||||||
}
|
}
|
||||||
|
@ -268,11 +282,9 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Add comments explaining the translation between struct, map, slice and
|
|
||||||
// the JSON schema representation.
|
|
||||||
func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) {
|
func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) {
|
||||||
if typ.Kind() != reflect.Slice {
|
if typ.Kind() != reflect.Slice {
|
||||||
return InvalidSchema, fmt.Errorf("expected slice, got %s", typ.Kind())
|
return Schema{}, fmt.Errorf("expected slice, got %s", typ.Kind())
|
||||||
}
|
}
|
||||||
|
|
||||||
res := Schema{
|
res := Schema{
|
||||||
|
@ -280,19 +292,16 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
typPath := typePath(typ.Elem())
|
typPath := typePath(typ.Elem())
|
||||||
// Only walk if the type has not been seen yet.
|
|
||||||
if _, ok := c.seen[typPath]; !ok {
|
// Walk the slice element type.
|
||||||
// Trigger call to fromType, to recursively generate definitions for
|
err := c.walk(typ.Elem())
|
||||||
// the slice element.
|
if err != nil {
|
||||||
err := c.walk(typ.Elem())
|
return Schema{}, err
|
||||||
if err != nil {
|
|
||||||
return InvalidSchema, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
refPath := path.Join("#/$defs", typPath)
|
refPath := path.Join("#/$defs", typPath)
|
||||||
|
|
||||||
// For non-built-in types, refer to the definition
|
// Add a $ref to the corresponding $defs block for the slice element type.
|
||||||
res.Items = &Schema{
|
res.Items = &Schema{
|
||||||
Reference: &refPath,
|
Reference: &refPath,
|
||||||
}
|
}
|
||||||
|
@ -301,11 +310,11 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) {
|
||||||
|
|
||||||
func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) {
|
func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) {
|
||||||
if typ.Kind() != reflect.Map {
|
if typ.Kind() != reflect.Map {
|
||||||
return InvalidSchema, fmt.Errorf("expected map, got %s", typ.Kind())
|
return Schema{}, fmt.Errorf("expected map, got %s", typ.Kind())
|
||||||
}
|
}
|
||||||
|
|
||||||
if typ.Key().Kind() != reflect.String {
|
if typ.Key().Kind() != reflect.String {
|
||||||
return InvalidSchema, fmt.Errorf("found map with non-string key: %v", typ.Key())
|
return Schema{}, fmt.Errorf("found map with non-string key: %v", typ.Key())
|
||||||
}
|
}
|
||||||
|
|
||||||
res := Schema{
|
res := Schema{
|
||||||
|
@ -313,19 +322,16 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
typPath := typePath(typ.Elem())
|
typPath := typePath(typ.Elem())
|
||||||
// Only walk if the type has not been seen yet.
|
|
||||||
if _, ok := c.seen[typPath]; !ok {
|
// Walk the map value type.
|
||||||
// Trigger call to fromType, to recursively generate definitions for
|
err := c.walk(typ.Elem())
|
||||||
// the map value.
|
if err != nil {
|
||||||
err := c.walk(typ.Elem())
|
return Schema{}, err
|
||||||
if err != nil {
|
|
||||||
return InvalidSchema, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
refPath := path.Join("#/$defs", typPath)
|
refPath := path.Join("#/$defs", typPath)
|
||||||
|
|
||||||
// For non-built-in types, refer to the definition
|
// Add a $ref to the corresponding $defs block for the map value type.
|
||||||
res.AdditionalProperties = &Schema{
|
res.AdditionalProperties = &Schema{
|
||||||
Reference: &refPath,
|
Reference: &refPath,
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,9 +67,7 @@ func TestFromTypeBasic(t *testing.T) {
|
||||||
expected: Schema{
|
expected: Schema{
|
||||||
Type: "object",
|
Type: "object",
|
||||||
Definitions: map[string]any{
|
Definitions: map[string]any{
|
||||||
"interface": Schema{
|
"interface": Schema{},
|
||||||
Type: "null",
|
|
||||||
},
|
|
||||||
"string": Schema{
|
"string": Schema{
|
||||||
Type: "string",
|
Type: "string",
|
||||||
},
|
},
|
||||||
|
@ -160,6 +158,8 @@ func TestGetStructFields(t *testing.T) {
|
||||||
assert.Equal(t, "B", fields[2].Name)
|
assert.Equal(t, "B", fields[2].Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: Add other case coverage for all the tests below
|
||||||
|
|
||||||
func TestFromTypeNested(t *testing.T) {
|
func TestFromTypeNested(t *testing.T) {
|
||||||
type Inner struct {
|
type Inner struct {
|
||||||
S string `json:"s"`
|
S string `json:"s"`
|
||||||
|
@ -222,7 +222,7 @@ func TestFromTypeNested(t *testing.T) {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "struct as a map value",
|
name: "struct as a map value",
|
||||||
typ: reflect.TypeOf(map[string]Inner{}),
|
typ: reflect.TypeOf(map[string]*Inner{}),
|
||||||
expected: Schema{
|
expected: Schema{
|
||||||
Type: "object",
|
Type: "object",
|
||||||
Definitions: expectedDefinitions,
|
Definitions: expectedDefinitions,
|
||||||
|
@ -252,7 +252,6 @@ func TestFromTypeNested(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Call out in the PR description that recursive Go types are supported.
|
|
||||||
func TestFromTypeRecursive(t *testing.T) {
|
func TestFromTypeRecursive(t *testing.T) {
|
||||||
fooRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Foo"
|
fooRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Foo"
|
||||||
barRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Bar"
|
barRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Bar"
|
||||||
|
@ -353,8 +352,76 @@ func TestFromTypeSelfReferential(t *testing.T) {
|
||||||
assert.Equal(t, expected, s)
|
assert.Equal(t, expected, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: Add coverage for all errors returned by FromType.
|
||||||
func TestFromTypeError(t *testing.T) {
|
func TestFromTypeError(t *testing.T) {
|
||||||
type mapOfInts map[int]int
|
type mapOfInts map[int]int
|
||||||
_, err := FromType(reflect.TypeOf(mapOfInts{}), nil)
|
|
||||||
assert.EqualError(t, err, "found map with non-string key: int")
|
assert.PanicsWithValue(t, "found map with non-string key: int", func() {
|
||||||
|
FromType(reflect.TypeOf(mapOfInts{}), nil)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Add test that the fn argument ot from_type works as expected.
|
||||||
|
|
||||||
|
func TestTypePath(t *testing.T) {
|
||||||
|
type myStruct struct{}
|
||||||
|
|
||||||
|
tcases := []struct {
|
||||||
|
typ reflect.Type
|
||||||
|
path string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(""),
|
||||||
|
path: "string",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(int(0)),
|
||||||
|
path: "int",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(true),
|
||||||
|
path: "bool",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(float64(0)),
|
||||||
|
path: "float64",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(myStruct{}),
|
||||||
|
path: "github.com/databricks/cli/libs/jsonschema.myStruct",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf([]int{}),
|
||||||
|
path: "slice/int",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(map[string]int{}),
|
||||||
|
path: "map/int",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf([]myStruct{}),
|
||||||
|
path: "slice/github.com/databricks/cli/libs/jsonschema.myStruct",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf([][]map[string]map[string]myStruct{}),
|
||||||
|
path: "slice/slice/map/map/github.com/databricks/cli/libs/jsonschema.myStruct",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
typ: reflect.TypeOf(map[string]myStruct{}),
|
||||||
|
path: "map/github.com/databricks/cli/libs/jsonschema.myStruct",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: support arbitary depth of maps and slices. Also add validation
|
||||||
|
// in this function that non-string keys are not allowed.
|
||||||
|
for _, tc := range tcases {
|
||||||
|
t.Run(tc.typ.String(), func(t *testing.T) {
|
||||||
|
assert.Equal(t, tc.path, typePath(tc.typ))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Maps with non-string keys should panic.
|
||||||
|
assert.PanicsWithValue(t, "found map with non-string key: int", func() {
|
||||||
|
typePath(reflect.TypeOf(map[int]int{}))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue