Move bundle schema update to an internal module (#1012)

## Changes

This PR:
1. Move code to load bundle JSON Schema descriptions from the OpenAPI
spec to an internal Go module
2. Remove command line flags from the `bundle schema` command. These
flags were meant for internal processes and at no point were meant for
customer use.
3. Regenerate `bundle_descriptions.json`
4. Add support for `bundle: "deprecated"`. The `environments` field is
tagged as deprecated in this PR and consequently will no longer be a
part of the bundle schema.

## Tests
Tested by regenerating the CLI against its current OpenAPI spec (as
defined in `__openapi_sha`). The `bundle_descriptions.json` in this PR
was generated from the code generator.

Manually checked that the autocompletion / descriptions from the new
bundle schema are correct.
This commit is contained in:
shreyas-goenka 2023-12-06 16:15:18 +05:30 committed by GitHub
parent a6752a5388
commit 6002f49c87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 2834 additions and 2000 deletions

View File

@ -8,6 +8,12 @@
".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go" ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
}, },
"toolchain": { "toolchain": {
"required": ["go"] "required": ["go"],
"post_generate": [
"go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
"echo 'go.sum linguist-generated=true' >> ./.gitattributes",
"echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
]
} }
} }

3
.gitattributes vendored
View File

@ -83,3 +83,6 @@ cmd/workspace/warehouses/warehouses.go linguist-generated=true
cmd/workspace/workspace-bindings/workspace-bindings.go linguist-generated=true cmd/workspace/workspace-bindings/workspace-bindings.go linguist-generated=true
cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
cmd/workspace/workspace/workspace.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true
bundle/internal/tf/schema/\*.go linguist-generated=true
go.sum linguist-generated=true
bundle/schema/docs/bundle_descriptions.json linguist-generated=true

View File

@ -48,7 +48,7 @@ type Root struct {
Targets map[string]*Target `json:"targets,omitempty"` Targets map[string]*Target `json:"targets,omitempty"`
// DEPRECATED. Left for backward compatibility with Targets // DEPRECATED. Left for backward compatibility with Targets
Environments map[string]*Target `json:"environments,omitempty"` Environments map[string]*Target `json:"environments,omitempty" bundle:"deprecated"`
// Sync section specifies options for files synchronization // Sync section specifies options for files synchronization
Sync Sync `json:"sync,omitempty"` Sync Sync `json:"sync,omitempty"`

View File

@ -0,0 +1,42 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"github.com/databricks/cli/bundle/schema"
)
func main() {
if len(os.Args) != 2 {
fmt.Println("Usage: go run main.go <output-file>")
os.Exit(1)
}
// Output file, to write the generated schema descriptions to.
outputFile := os.Args[1]
// Input file, the databricks openapi spec.
inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
if inputFile == "" {
log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
}
// Generate the schema descriptions.
docs, err := schema.UpdateBundleDescriptions(inputFile)
if err != nil {
log.Fatal(err)
}
result, err := json.MarshalIndent(docs, "", " ")
if err != nil {
log.Fatal(err)
}
// Write the schema descriptions to the output file.
err = os.WriteFile(outputFile, result, 0644)
if err != nil {
log.Fatal(err)
}
}

View File

@ -13,15 +13,6 @@ These descriptions are rendered in the inline documentation in an IDE
### SOP: Add schema descriptions for new fields in bundle config ### SOP: Add schema descriptions for new fields in bundle config
1. You can autogenerate empty descriptions for the new fields by running Manually edit bundle_descriptions.json to add your descriptions. Note that the
`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` descriptions in `resources` block is generated from the OpenAPI spec, and thus
2. Manually edit bundle_descriptions.json to add your descriptions any changes there will be overwritten.
3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
5. push to repo
### SOP: Update descriptions in resources from a newer openapi spec
1. Run `databricks bundle schema --only-docs --openapi PATH_TO_SPEC > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
2. push to repo

View File

@ -23,39 +23,6 @@ type Docs struct {
//go:embed docs/bundle_descriptions.json //go:embed docs/bundle_descriptions.json
var bundleDocs []byte var bundleDocs []byte
func BundleDocs(openapiSpecPath string) (*Docs, error) {
docs, err := initializeBundleDocs()
if err != nil {
return nil, err
}
if openapiSpecPath != "" {
openapiSpec, err := os.ReadFile(openapiSpecPath)
if err != nil {
return nil, err
}
spec := &openapi.Specification{}
err = json.Unmarshal(openapiSpec, spec)
if err != nil {
return nil, err
}
openapiReader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
}
resourcesDocs, err := openapiReader.ResourcesDocs()
if err != nil {
return nil, err
}
resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
if err != nil {
return nil, err
}
docs.Properties["resources"] = schemaToDocs(resourceSchema)
}
docs.refreshTargetsDocs()
return docs, nil
}
func (docs *Docs) refreshTargetsDocs() error { func (docs *Docs) refreshTargetsDocs() error {
targetsDocs, ok := docs.Properties["targets"] targetsDocs, ok := docs.Properties["targets"]
if !ok || targetsDocs.AdditionalProperties == nil || if !ok || targetsDocs.AdditionalProperties == nil ||
@ -70,21 +37,53 @@ func (docs *Docs) refreshTargetsDocs() error {
return nil return nil
} }
func initializeBundleDocs() (*Docs, error) { func LoadBundleDescriptions() (*Docs, error) {
// load embedded descriptions
embedded := Docs{} embedded := Docs{}
err := json.Unmarshal(bundleDocs, &embedded) err := json.Unmarshal(bundleDocs, &embedded)
return &embedded, err
}
func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) {
embedded, err := LoadBundleDescriptions()
if err != nil { if err != nil {
return nil, err return nil, err
} }
// generate schema with the embedded descriptions
schema, err := New(reflect.TypeOf(config.Root{}), &embedded) // Generate schema from the embedded descriptions, and convert it back to docs.
// This creates empty descriptions for any properties that were missing in the
// embedded descriptions.
schema, err := New(reflect.TypeOf(config.Root{}), embedded)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// converting the schema back to docs. This creates empty descriptions
// for any properties that were missing in the embedded descriptions
docs := schemaToDocs(schema) docs := schemaToDocs(schema)
// Load the Databricks OpenAPI spec
openapiSpec, err := os.ReadFile(openapiSpecPath)
if err != nil {
return nil, err
}
spec := &openapi.Specification{}
err = json.Unmarshal(openapiSpec, spec)
if err != nil {
return nil, err
}
openapiReader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
}
// Generate descriptions for the "resources" field
resourcesDocs, err := openapiReader.ResourcesDocs()
if err != nil {
return nil, err
}
resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
if err != nil {
return nil, err
}
docs.Properties["resources"] = schemaToDocs(resourceSchema)
docs.refreshTargetsDocs()
return docs, nil return docs, nil
} }

File diff suppressed because it is too large Load Diff

View File

@ -17,6 +17,10 @@ const readonlyTag = "readonly"
// Fields can be tagged as "internal" to remove them from the generated schema. // Fields can be tagged as "internal" to remove them from the generated schema.
const internalTag = "internal" const internalTag = "internal"
// Annotation for bundle fields that have been deprecated.
// Fields tagged as "deprecated" are removed/omitted from the generated schema.
const deprecatedTag = "deprecated"
// This function translates golang types into json schema. Here is the mapping // This function translates golang types into json schema. Here is the mapping
// between json schema types and golang types // between json schema types and golang types
// //
@ -205,7 +209,9 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschem
required := []string{} required := []string{}
for _, child := range children { for _, child := range children {
bundleTag := child.Tag.Get("bundle") bundleTag := child.Tag.Get("bundle")
if bundleTag == readonlyTag || bundleTag == internalTag { // Fields marked as "readonly", "internal" or "deprecated" are skipped
// while generating the schema
if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag {
continue continue
} }

View File

@ -2,7 +2,6 @@ package bundle
import ( import (
"encoding/json" "encoding/json"
"os"
"reflect" "reflect"
"github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config"
@ -16,47 +15,24 @@ func newSchemaCommand() *cobra.Command {
Short: "Generate JSON Schema for bundle configuration", Short: "Generate JSON Schema for bundle configuration",
} }
var openapi string
var outputFile string
var onlyDocs bool
cmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
cmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
cmd.Flags().StringVar(&outputFile, "output-file", "", "File path to write the schema to. If not specified, the schema will be written to stdout.")
cmd.RunE = func(cmd *cobra.Command, args []string) error { cmd.RunE = func(cmd *cobra.Command, args []string) error {
// If no openapi spec is provided, try to use the environment variable. // Load embedded schema descriptions.
// This environment variable is set during CLI code generation. docs, err := schema.LoadBundleDescriptions()
if openapi == "" {
openapi = os.Getenv("DATABRICKS_OPENAPI_SPEC")
}
docs, err := schema.BundleDocs(openapi)
if err != nil { if err != nil {
return err return err
} }
// Generate the JSON schema from the bundle configuration struct in Go.
schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) schema, err := schema.New(reflect.TypeOf(config.Root{}), docs)
if err != nil { if err != nil {
return err return err
} }
// Print the JSON schema to stdout.
result, err := json.MarshalIndent(schema, "", " ") result, err := json.MarshalIndent(schema, "", " ")
if err != nil { if err != nil {
return err return err
} }
if onlyDocs {
result, err = json.MarshalIndent(docs, "", " ")
if err != nil {
return err
}
}
// If outputFile is provided, write to that file.
if outputFile != "" {
f, err := os.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer f.Close()
cmd.SetOut(f)
}
cmd.OutOrStdout().Write(result) cmd.OutOrStdout().Write(result)
return nil return nil
} }