Move bundle schema update to an internal module (#1012)

## Changes

This PR:
1. Move code to load bundle JSON Schema descriptions from the OpenAPI
spec to an internal Go module
2. Remove command line flags from the `bundle schema` command. These
flags were meant for internal processes and at no point were meant for
customer use.
3. Regenerate `bundle_descriptions.json`
4. Add support for `bundle: "deprecated"`. The `environments` field is
tagged as deprecated in this PR and consequently will no longer be a
part of the bundle schema.

## Tests
Tested by regenerating the CLI against its current OpenAPI spec (as
defined in `__openapi_sha`). The `bundle_descriptions.json` in this PR
was generated from the code generator.

Manually checked that the autocompletion / descriptions from the new
bundle schema are correct.

.codegen.json

@@ -8,6 +8,12 @@
         ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
     },
     "toolchain": {
-        "required": ["go"]
+        "required": ["go"],
+        "post_generate": [
+            "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
+            "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
+            "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
+            "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
+        ]
     }
 }

.gitattributes

@@ -83,3 +83,6 @@ cmd/workspace/warehouses/warehouses.go linguist-generated=true
 cmd/workspace/workspace-bindings/workspace-bindings.go linguist-generated=true
 cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
 cmd/workspace/workspace/workspace.go linguist-generated=true
+bundle/internal/tf/schema/\*.go linguist-generated=true
+go.sum linguist-generated=true
+bundle/schema/docs/bundle_descriptions.json linguist-generated=true

bundle/config/root.go

@@ -48,7 +48,7 @@ type Root struct {
 	Targets map[string]*Target `json:"targets,omitempty"`
 
 	// DEPRECATED. Left for backward compatibility with Targets
-	Environments map[string]*Target `json:"environments,omitempty"`
+	Environments map[string]*Target `json:"environments,omitempty" bundle:"deprecated"`
 
 	// Sync section specifies options for files synchronization
 	Sync Sync `json:"sync,omitempty"`

bundle/internal/bundle/schema/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/databricks/cli/bundle/schema"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Println("Usage: go run main.go <output-file>")
+		os.Exit(1)
+	}
+
+	// Output file, to write the generated schema descriptions to.
+	outputFile := os.Args[1]
+
+	// Input file, the databricks openapi spec.
+	inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
+	if inputFile == "" {
+		log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
+	}
+
+	// Generate the schema descriptions.
+	docs, err := schema.UpdateBundleDescriptions(inputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	result, err := json.MarshalIndent(docs, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Write the schema descriptions to the output file.
+	err = os.WriteFile(outputFile, result, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

bundle/schema/README.md

@@ -13,15 +13,6 @@ These descriptions are rendered in the inline documentation in an IDE
 
 ### SOP: Add schema descriptions for new fields in bundle config
 
-1. You can autogenerate empty descriptions for the new fields by running
+Manually edit bundle_descriptions.json to add your descriptions. Note that the
-`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
+descriptions in `resources` block is generated from the OpenAPI spec, and thus
-2. Manually edit bundle_descriptions.json to add your descriptions
+any changes there will be overwritten.
-3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
-4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
-5. push to repo
-
-
-### SOP: Update descriptions in resources from a newer openapi spec
-
-1. Run `databricks bundle schema --only-docs --openapi PATH_TO_SPEC > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
-2. push to repo

bundle/schema/docs.go

@@ -23,39 +23,6 @@ type Docs struct {
 //go:embed docs/bundle_descriptions.json
 var bundleDocs []byte
 
-func BundleDocs(openapiSpecPath string) (*Docs, error) {
-	docs, err := initializeBundleDocs()
-	if err != nil {
-		return nil, err
-	}
-	if openapiSpecPath != "" {
-		openapiSpec, err := os.ReadFile(openapiSpecPath)
-		if err != nil {
-			return nil, err
-		}
-		spec := &openapi.Specification{}
-		err = json.Unmarshal(openapiSpec, spec)
-		if err != nil {
-			return nil, err
-		}
-		openapiReader := &OpenapiReader{
-			OpenapiSpec: spec,
-			Memo:        make(map[string]*jsonschema.Schema),
-		}
-		resourcesDocs, err := openapiReader.ResourcesDocs()
-		if err != nil {
-			return nil, err
-		}
-		resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
-		if err != nil {
-			return nil, err
-		}
-		docs.Properties["resources"] = schemaToDocs(resourceSchema)
-	}
-	docs.refreshTargetsDocs()
-	return docs, nil
-}
-
 func (docs *Docs) refreshTargetsDocs() error {
 	targetsDocs, ok := docs.Properties["targets"]
 	if !ok || targetsDocs.AdditionalProperties == nil ||
@@ -70,21 +37,53 @@ func (docs *Docs) refreshTargetsDocs() error {
 	return nil
 }
 
-func initializeBundleDocs() (*Docs, error) {
+func LoadBundleDescriptions() (*Docs, error) {
-	// load embedded descriptions
 	embedded := Docs{}
 	err := json.Unmarshal(bundleDocs, &embedded)
+	return &embedded, err
+}
+
+func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) {
+	embedded, err := LoadBundleDescriptions()
 	if err != nil {
 		return nil, err
 	}
-	// generate schema with the embedded descriptions
+
-	schema, err := New(reflect.TypeOf(config.Root{}), &embedded)
+	// Generate schema from the embedded descriptions, and convert it back to docs.
+	// This creates empty descriptions for any properties that were missing in the
+	// embedded descriptions.
+	schema, err := New(reflect.TypeOf(config.Root{}), embedded)
 	if err != nil {
 		return nil, err
 	}
-	// converting the schema back to docs. This creates empty descriptions
-	// for any properties that were missing in the embedded descriptions
 	docs := schemaToDocs(schema)
+
+	// Load the Databricks OpenAPI spec
+	openapiSpec, err := os.ReadFile(openapiSpecPath)
+	if err != nil {
+		return nil, err
+	}
+	spec := &openapi.Specification{}
+	err = json.Unmarshal(openapiSpec, spec)
+	if err != nil {
+		return nil, err
+	}
+	openapiReader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*jsonschema.Schema),
+	}
+
+	// Generate descriptions for the "resources" field
+	resourcesDocs, err := openapiReader.ResourcesDocs()
+	if err != nil {
+		return nil, err
+	}
+	resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
+	if err != nil {
+		return nil, err
+	}
+	docs.Properties["resources"] = schemaToDocs(resourceSchema)
+	docs.refreshTargetsDocs()
 	return docs, nil
 }
 

bundle/schema/schema.go

@@ -17,6 +17,10 @@ const readonlyTag = "readonly"
 // Fields can be tagged as "internal" to remove them from the generated schema.
 const internalTag = "internal"
 
+// Annotation for bundle fields that have been deprecated.
+// Fields tagged as "deprecated" are removed/omitted from the generated schema.
+const deprecatedTag = "deprecated"
+
 // This function translates golang types into json schema. Here is the mapping
 // between json schema types and golang types
 //
@@ -205,7 +209,9 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschem
 		required := []string{}
 		for _, child := range children {
 			bundleTag := child.Tag.Get("bundle")
-			if bundleTag == readonlyTag || bundleTag == internalTag {
+			// Fields marked as "readonly", "internal" or "deprecated" are skipped
+			// while generating the schema
+			if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag {
 				continue
 			}
 

cmd/bundle/schema.go

@@ -2,7 +2,6 @@ package bundle
 
 import (
 	"encoding/json"
-	"os"
 	"reflect"
 
 	"github.com/databricks/cli/bundle/config"
@@ -16,47 +15,24 @@ func newSchemaCommand() *cobra.Command {
 		Short: "Generate JSON Schema for bundle configuration",
 	}
 
-	var openapi string
-	var outputFile string
-	var onlyDocs bool
-	cmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
-	cmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
-	cmd.Flags().StringVar(&outputFile, "output-file", "", "File path to write the schema to. If not specified, the schema will be written to stdout.")
-
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		// If no openapi spec is provided, try to use the environment variable.
+		// Load embedded schema descriptions.
-		// This environment variable is set during CLI code generation.
+		docs, err := schema.LoadBundleDescriptions()
-		if openapi == "" {
-			openapi = os.Getenv("DATABRICKS_OPENAPI_SPEC")
-		}
-		docs, err := schema.BundleDocs(openapi)
 		if err != nil {
 			return err
 		}
+
+		// Generate the JSON schema from the bundle configuration struct in Go.
 		schema, err := schema.New(reflect.TypeOf(config.Root{}), docs)
 		if err != nil {
 			return err
 		}
+
+		// Print the JSON schema to stdout.
 		result, err := json.MarshalIndent(schema, "", "  ")
 		if err != nil {
 			return err
 		}
-		if onlyDocs {
-			result, err = json.MarshalIndent(docs, "", "  ")
-			if err != nil {
-				return err
-			}
-		}
-
-		// If outputFile is provided, write to that file.
-		if outputFile != "" {
-			f, err := os.OpenFile(outputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			cmd.SetOut(f)
-		}
 		cmd.OutOrStdout().Write(result)
 		return nil
 	}