tmp: Documentation autogeneration script

2024-12-18 17:07:09 +01:00 · 2024-12-18 17:07:09 +01:00 · 8c58d485b5
parent 042c8d88c6
commit 8c58d485b5
7 changed files with 2355 additions and 2 deletions
--- a/8
+++ b/8
@ -29,11 +29,15 @@ snapshot:
 vendor:
 	@echo "✓ Filling vendor folder with library code ..."
 	@go mod vendor
-  
+
 schema:
 	@echo "✓ Generating json-schema ..."
 	@go run ./bundle/internal/schema ./bundle/internal/schema ./bundle/schema/jsonschema.json

+docs:
+	@echo "✓ Generating docs ..."
+	@go run ./bundle/internal/docs ./bundle/internal/schema ./bundle/internal/docs/docs.md
+
 INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h

 integration:
@ -42,4 +46,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short

-.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck test testonly coverage build snapshot vendor schema integration integration-short docs
--- a/bundle/internal/docs/docs.go
+++ b/bundle/internal/docs/docs.go
@ -0,0 +1,235 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/databricks/cli/libs/jsonschema"
+
+	md "github.com/nao1215/markdown"
+)
+
+type rootNode struct {
+	Title               string
+	Description         string
+	Attributes          []attributeNode
+	Example             string
+	ObjectKeyAttributes []attributeNode
+	ArrayItemAttributes []attributeNode
+	TopLevel            bool
+}
+
+type attributeNode struct {
+	Title       string
+	Type        string
+	Description string
+}
+
+type rootProp struct {
+	k        string
+	v        *jsonschema.Schema
+	topLevel bool
+}
+
+const (
+	AdditionalPropertiesMessage              = "Each item has the following attributes:"
+	AdditionalPropertiesAttributeTitle       = "<name>"
+	AdditionalPropertiesAttributeDescription = "The definition of the item"
+)
+
+func getNodes(s jsonschema.Schema, refs map[string]jsonschema.Schema, a annotationFile) []rootNode {
+	rootProps := []rootProp{}
+	for k, v := range s.Properties {
+		rootProps = append(rootProps, rootProp{k, v, true})
+	}
+	nodes := make([]rootNode, 0, len(rootProps))
+
+	for i := 0; i < len(rootProps); i++ {
+		k := rootProps[i].k
+		v := rootProps[i].v
+		v = resolveRefs(v, refs)
+		node := rootNode{
+			Title:       k,
+			Description: getDescription(v),
+			TopLevel:    rootProps[i].topLevel,
+		}
+
+		node.Attributes = getAttributes(v.Properties, refs)
+		rootProps = append(rootProps, extractNodes(k, v.Properties, refs, a)...)
+
+		additionalProps, ok := v.AdditionalProperties.(*jsonschema.Schema)
+		if ok {
+			objectKeyType := resolveRefs(additionalProps, refs)
+			node.ObjectKeyAttributes = getAttributes(objectKeyType.Properties, refs)
+			rootProps = append(rootProps, extractNodes(k, objectKeyType.Properties, refs, a)...)
+		}
+
+		if v.Items != nil {
+			arrayItemType := resolveRefs(v.Items, refs)
+			node.ArrayItemAttributes = getAttributes(arrayItemType.Properties, refs)
+		}
+
+		nodes = append(nodes, node)
+	}
+
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].Title < nodes[j].Title
+	})
+	return nodes
+}
+
+func buildMarkdown(nodes []rootNode, outputFile string) error {
+	f, err := os.Create(outputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	m := md.NewMarkdown(f)
+	for _, node := range nodes {
+		if node.TopLevel {
+			m = m.H2(node.Title)
+		} else {
+			m = m.H3(node.Title)
+		}
+		m = m.PlainText(node.Description)
+
+		if len(node.ObjectKeyAttributes) > 0 {
+			m = buildAttributeTable(m, []attributeNode{
+				{Title: AdditionalPropertiesAttributeTitle, Type: "Map", Description: AdditionalPropertiesAttributeDescription},
+			})
+			m = m.PlainText("Each item has the following attributes:")
+			m = buildAttributeTable(m, node.ObjectKeyAttributes)
+
+		} else if len(node.ArrayItemAttributes) > 0 {
+			m = m.PlainText(fmt.Sprintf("Each item of `%s` has the following attributes:", node.Title))
+			m = buildAttributeTable(m, node.ArrayItemAttributes)
+		} else if len(node.Attributes) > 0 {
+			m = m.H4("Attributes")
+			m = buildAttributeTable(m, node.Attributes)
+		}
+	}
+
+	err = m.Build()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
+func buildAttributeTable(m *md.Markdown, attributes []attributeNode) *md.Markdown {
+	rows := [][]string{}
+	for _, n := range attributes {
+		rows = append(rows, []string{fmt.Sprintf("`%s`", n.Title), n.Type, formatDescription(n.Description)})
+	}
+	m = m.CustomTable(md.TableSet{
+		Header: []string{"Key", "Type", "Description"},
+		Rows:   rows,
+	}, md.TableOptions{AutoWrapText: false, AutoFormatHeaders: false})
+
+	return m
+}
+
+func formatDescription(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return strings.ReplaceAll(s, "\n", " ")
+}
+
+// Build a custom table which we use in Databricks website
+func buildCustomAttributeTable(m *md.Markdown, attributes []attributeNode) *md.Markdown {
+	m = m.PlainText(".. list-table::")
+	m = m.PlainText("   :header-rows: 1")
+
+	m = m.PlainText("   * - Key")
+	m = m.PlainText("     - Type")
+	m = m.PlainText("     - Description")
+
+	for _, a := range attributes {
+		m = m.PlainText("   * - " + a.Title)
+		m = m.PlainText("     - " + a.Type)
+		m = m.PlainText("     - " + a.Description)
+	}
+	return m
+}
+
+func getAttributes(props map[string]*jsonschema.Schema, refs map[string]jsonschema.Schema) []attributeNode {
+	typesMapping := map[string]string{
+		"string":  "String",
+		"integer": "Integer",
+		"boolean": "Boolean",
+		"array":   "Sequence",
+		"object":  "Map",
+	}
+
+	attributes := []attributeNode{}
+	for k, v := range props {
+		v = resolveRefs(v, refs)
+		typeString := typesMapping[string(v.Type)]
+		if typeString == "" {
+			typeString = "Any"
+		}
+		attributes = append(attributes, attributeNode{
+			Title:       k,
+			Type:        typeString,
+			Description: getDescription(v),
+		})
+	}
+	sort.Slice(attributes, func(i, j int) bool {
+		return attributes[i].Title < attributes[j].Title
+	})
+	return attributes
+}
+
+func getDescription(s *jsonschema.Schema) string {
+	if s.MarkdownDescription != "" {
+		return s.MarkdownDescription
+	}
+	return s.Description
+}
+
+func resolveRefs(s *jsonschema.Schema, schemas map[string]jsonschema.Schema) *jsonschema.Schema {
+	node := s
+
+	description := s.Description
+	markdownDescription := s.MarkdownDescription
+
+	for node.Reference != nil {
+		ref := strings.TrimPrefix(*node.Reference, "#/$defs/")
+		newNode, ok := schemas[ref]
+		if !ok {
+			log.Printf("schema %s not found", ref)
+		}
+
+		if description == "" {
+			description = newNode.Description
+		}
+		if markdownDescription == "" {
+			markdownDescription = newNode.MarkdownDescription
+		}
+
+		node = &newNode
+	}
+
+	node.Description = description
+	node.MarkdownDescription = markdownDescription
+
+	return node
+}
+
+func extractNodes(prefix string, props map[string]*jsonschema.Schema, refs map[string]jsonschema.Schema, a annotationFile) []rootProp {
+	nodes := []rootProp{}
+	for k, v := range props {
+		v = resolveRefs(v, refs)
+		if v.Type == "object" {
+			nodes = append(nodes, rootProp{prefix + "." + k, v, false})
+		}
+		v.MarkdownDescription = ""
+	}
+	return nodes
+}
--- a/bundle/internal/docs/docs.md
+++ b/bundle/internal/docs/docs.md
--- a/bundle/internal/docs/main.go
+++ b/bundle/internal/docs/main.go
@ -0,0 +1,183 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/merge"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
+	"github.com/databricks/cli/libs/jsonschema"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+)
+
+const Placeholder = "PLACEHOLDER"
+
+func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	switch typ {
+	case reflect.TypeOf(resources.Job{}):
+		// This field has been deprecated in jobs API v2.1 and is always set to
+		// "MULTI_TASK" in the backend. We should not expose it to the user.
+		delete(s.Properties, "format")
+
+		// These fields are only meant to be set by the DABs client (ie the CLI)
+		// and thus should not be exposed to the user. These are used to annotate
+		// jobs that were created by DABs.
+		delete(s.Properties, "deployment")
+		delete(s.Properties, "edit_mode")
+
+	case reflect.TypeOf(jobs.GitSource{}):
+		// These fields are readonly and are not meant to be set by the user.
+		delete(s.Properties, "job_source")
+		delete(s.Properties, "git_snapshot")
+
+	default:
+		// Do nothing
+	}
+
+	return s
+}
+
+// While volume_type is required in the volume create API, DABs automatically sets
+// it's value to "MANAGED" if it's not provided. Thus, we make it optional
+// in the bundle schema.
+func makeVolumeTypeOptional(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	if typ != reflect.TypeOf(resources.Volume{}) {
+		return s
+	}
+
+	res := []string{}
+	for _, r := range s.Required {
+		if r != "volume_type" {
+			res = append(res, r)
+		}
+	}
+	s.Required = res
+	return s
+}
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println("Usage: go run main.go <annotation-file> <output-file>")
+		os.Exit(1)
+	}
+
+	annotationFile := os.Args[1]
+	outputFile := os.Args[2]
+
+	err := generateDocs(annotationFile, outputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+type annotationFile map[string]map[string]annotation
+
+type annotation struct {
+	Description         string `json:"description,omitempty"`
+	MarkdownDescription string `json:"markdown_description,omitempty"`
+	Title               string `json:"title,omitempty"`
+	Default             any    `json:"default,omitempty"`
+	Enum                []any  `json:"enum,omitempty"`
+}
+
+func generateDocs(workdir, outputPath string) error {
+	annotationsPath := filepath.Join(workdir, "annotations.yml")
+	annotationsOpenApiPath := filepath.Join(workdir, "annotations_openapi.yml")
+	annotationsOpenApiOverridesPath := filepath.Join(workdir, "annotations_openapi_overrides.yml")
+
+	annotations, err := LoadAndMergeAnnotations([]string{annotationsPath, annotationsOpenApiPath, annotationsOpenApiOverridesPath})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	schemas := map[string]jsonschema.Schema{}
+
+	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+		removeJobsFields,
+		makeVolumeTypeOptional,
+
+		func(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+			schemas[jsonschema.TypePath(typ)] = s
+
+			refPath := getPath(typ)
+			shouldHandle := strings.HasPrefix(refPath, "github.com")
+			if !shouldHandle {
+				return s
+			}
+
+			a := annotations[refPath]
+			if a == nil {
+				a = map[string]annotation{}
+			}
+
+			rootTypeAnnotation, ok := a["_"]
+			if ok {
+				assignAnnotation(&s, rootTypeAnnotation)
+			}
+
+			for k, v := range s.Properties {
+				assignAnnotation(v, a[k])
+			}
+
+			return s
+		},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	nodes := getNodes(s, schemas, annotations)
+	err = buildMarkdown(nodes, outputPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func getPath(typ reflect.Type) string {
+	return typ.PkgPath() + "." + typ.Name()
+}
+
+func assignAnnotation(s *jsonschema.Schema, a annotation) {
+	if a.Description != "" && a.Description != Placeholder {
+		s.Description = a.Description
+	}
+	if a.MarkdownDescription != "" {
+		s.MarkdownDescription = a.MarkdownDescription
+	}
+}
+
+func LoadAndMergeAnnotations(sources []string) (annotationFile, error) {
+	prev := dyn.NilValue
+	for _, path := range sources {
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		generated, err := yamlloader.LoadYAML(path, bytes.NewBuffer(b))
+		if err != nil {
+			return nil, err
+		}
+		prev, err = merge.Merge(prev, generated)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var data annotationFile
+
+	err := convert.ToTyped(&data, prev)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
--- a/go.mod
+++ b/go.mod
@ -53,8 +53,13 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/nao1215/markdown v0.6.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/radovskyb/watcher v1.0.7 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/zclconf/go-cty v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
--- a/go.sum
+++ b/go.sum
@ -115,6 +115,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
+github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
@ -127,8 +129,14 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/nao1215/markdown v0.6.0 h1:kqhrC47K434YA1jMTUwJwSV/hla8ifN3NzehMEffI/E=
+github.com/nao1215/markdown v0.6.0/go.mod h1:ObBhnNduWwPN+bu4dtv4JoLRt57ONla7l//03iHIVhY=
 github.com/nwidger/jsoncolor v0.3.2 h1:rVJJlwAWDJShnbTYOQ5RM7yTA20INyKXlJ/fg4JMhHQ=
 github.com/nwidger/jsoncolor v0.3.2/go.mod h1:Cs34umxLbJvgBMnVNVqhji9BhoT/N/KinHqZptQ7cf4=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
@ -136,6 +144,8 @@ github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzL
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
+github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
--- a/libs/jsonschema/from_type.go
+++ b/libs/jsonschema/from_type.go
@ -111,6 +111,10 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (
 	return res, nil
 }

+func TypePath(typ reflect.Type) string {
+	return typePath(typ)
+}
+
 // typePath computes a unique string representation of the type. $ref in the generated
 // JSON schema will refer to this path. See TestTypePath for examples outputs.
 func typePath(typ reflect.Type) string {