From 27eb0c40725aa978191ce3fe43eb07993fd6cfe4 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 6 Feb 2025 20:27:55 +0100 Subject: [PATCH] Allow 'any' examples in JSON schema (#2289) ## Changes 1. Allow `any` examples in json-schema type since we have many of them in open api spec 2. Fix issue with missing overrides annotations when re-generating the schema ## Tests --- bundle/docsgen/main.go | 2 +- bundle/docsgen/nodes.go | 4 +- bundle/docsgen/refs.go | 12 +- .../schema/annotations_openapi_overrides.yml | 161 +++++++++--------- bundle/internal/schema/parser.go | 10 +- libs/jsonschema/schema.go | 2 +- 6 files changed, 101 insertions(+), 90 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index ad737feea..84bf4779f 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.MarkdownDescription = a.MarkdownDescription } if a.MarkdownExamples != "" { - s.Examples = []any{a.MarkdownExamples} + s.Examples = []string{a.MarkdownExamples} } } diff --git a/bundle/docsgen/nodes.go b/bundle/docsgen/nodes.go index 68ed86450..61d2c21cc 100644 --- a/bundle/docsgen/nodes.go +++ b/bundle/docsgen/nodes.go @@ -220,9 +220,9 @@ func isCycleField(field string) bool { } func getExample(v *jsonschema.Schema) string { - examples := v.Examples + examples := getExamples(v.Examples) if len(examples) == 0 { return "" } - return examples[0].(string) + return examples[0] } diff --git a/bundle/docsgen/refs.go b/bundle/docsgen/refs.go index ca45e6ab2..7a4451129 100644 --- a/bundle/docsgen/refs.go +++ b/bundle/docsgen/refs.go @@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j node := s description := s.Description markdownDescription := s.MarkdownDescription - examples := s.Examples + examples := getExamples(s.Examples) for node.Reference != nil { ref := getRefType(node) @@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j markdownDescription = newNode.MarkdownDescription } if len(examples) == 0 { - examples = newNode.Examples + examples = getExamples(newNode.Examples) } node = newNode @@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j return &newNode } +func getExamples(examples any) []string { + typedExamples, ok := examples.([]string) + if !ok { + return []string{} + } + return typedExamples +} + func getRefType(node *jsonschema.Schema) string { if node.Reference == nil { return "" diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 912a4fda0..585886313 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create). - "markdown_examples": |- The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`: @@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard: If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate). In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy). - "embed_credentials": "description": |- PLACEHOLDER @@ -356,7 +354,6 @@ github.com/databricks/cli/bundle/config/resources.Volume: - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments. - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets). - "markdown_examples": |- The following example creates a volume with the key `my_volume`: @@ -376,6 +373,42 @@ github.com/databricks/cli/bundle/config/resources.Volume: "volume_type": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: + "create_time": + "description": |- + PLACEHOLDER + "creator": + "description": |- + PLACEHOLDER + "deployment_artifacts": + "description": |- + PLACEHOLDER + "deployment_id": + "description": |- + PLACEHOLDER + "mode": + "description": |- + PLACEHOLDER + "source_code_path": + "description": |- + PLACEHOLDER + "status": + "description": |- + PLACEHOLDER + "update_time": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: + "source_code_path": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/apps.AppResource: "job": "description": |- @@ -389,6 +422,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource: "sql_warehouse": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: + "key": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER + "scope": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: + "name": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: + "id": + "description": |- + PLACEHOLDER + "permission": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: + "message": + "description": |- + PLACEHOLDER + "state": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: + "message": + "description": |- + PLACEHOLDER + "state": {} github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes: "availability": "description": |- @@ -473,85 +549,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeployment: - "create_time": - "description": |- - PLACEHOLDER - "creator": - "description": |- - PLACEHOLDER - "deployment_artifacts": - "description": |- - PLACEHOLDER - "deployment_id": - "description": |- - PLACEHOLDER - "mode": - "description": |- - PLACEHOLDER - "source_code_path": - "description": |- - PLACEHOLDER - "status": - "description": |- - PLACEHOLDER - "update_time": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts: - "source_code_path": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret: - "key": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER - "scope": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint: - "name": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse: - "id": - "description": |- - PLACEHOLDER - "permission": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus: - "message": - "description": |- - PLACEHOLDER - "state": - "description": |- - PLACEHOLDER -github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus: - "message": - "description": |- - PLACEHOLDER - "state": github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index 50e69e7c8..ca8c27d4c 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "os" @@ -9,8 +10,9 @@ import ( "strings" "github.com/databricks/cli/bundle/internal/annotation" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/jsonschema" - "gopkg.in/yaml.v3" ) type Components struct { @@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid if err != nil { return err } - err = yaml.Unmarshal(b, &overrides) + overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b)) + if err != nil { + return err + } + err = convert.ToTyped(&overrides, overridesDyn) if err != nil { return err } diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 5028bb0d7..85f6a0328 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -79,7 +79,7 @@ type Schema struct { // Examples of the value for properties in the schema. // https://json-schema.org/understanding-json-schema/reference/annotations - Examples []any `json:"examples,omitempty"` + Examples any `json:"examples,omitempty"` } // Default value defined in a JSON Schema, represented as a string.