merge

bundle/docsgen/markdown.go

@@ -12,10 +12,11 @@ func buildMarkdown(nodes []rootNode, outputFile, header string) error {
 	m = m.PlainText(header)
 	for _, node := range nodes {
 		m = m.LF()
+		title := escapeBrackets(node.Title)
 		if node.TopLevel {
-			m = m.H2(node.Title)
+			m = m.H2(title)
 		} else {
-			m = m.H3(node.Title)
+			m = m.H3(title)
 		}
 		m = m.LF()
 
@@ -93,7 +94,23 @@ func formatDescription(a attributeNode) string {
 		} else if s != "" {
 			s += ". "
 		}
-		s += fmt.Sprintf("See [_](#%s).", a.Link)
+		s += fmt.Sprintf("See [_](#%s).", cleanAnchor(a.Link))
 	}
 	return s
 }
+
+// Docs framework does not allow special characters in anchor links and strip them out by default
+// We need to clean them up to make sure the links pass the validation
+func cleanAnchor(s string) string {
+	s = strings.ReplaceAll(s, "<", "")
+	s = strings.ReplaceAll(s, ">", "")
+	s = strings.ReplaceAll(s, ".", "")
+
+	return s
+}
+
+func escapeBrackets(s string) string {
+	s = strings.ReplaceAll(s, "<", "\\<")
+	s = strings.ReplaceAll(s, ">", "\\>")
+	return s
+}

bundle/docsgen/markdown_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildMarkdownAnchors(t *testing.T) {
+	nodes := []rootNode{
+		{
+			Title:       "some_field",
+			TopLevel:    true,
+			Type:        "Map",
+			Description: "This is a description",
+			Attributes: []attributeNode{
+				{
+					Title:       "my_attribute",
+					Type:        "Map",
+					Description: "Desc with link",
+					Link:        "some_field.<name>.my_attribute",
+				},
+			},
+		},
+		{
+			Title:       "some_field.<name>.my_attribute",
+			TopLevel:    false,
+			Type:        "Boolean",
+			Description: "Another description",
+		},
+	}
+	tmpDir := t.TempDir()
+	path := filepath.Join(tmpDir, "output.md")
+
+	err := buildMarkdown(nodes, path, "Header")
+	require.NoError(t, err)
+
+	expected := testutil.ReadFile(t, "testdata/anchors.md")
+	testutil.AssertFileContents(t, path, expected)
+}

bundle/docsgen/nodes.go

@@ -65,7 +65,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel
 		v = resolveRefs(v, refs)
 		node := rootNode{
 			Title:       k,
-			Description: getDescription(v, item.topLevel),
+			Description: getDescription(v),
 			TopLevel:    item.topLevel,
 			Example:     getExample(v),
 			Type:        getHumanReadableType(v.Type),
@@ -78,7 +78,7 @@ func buildNodes(s jsonschema.Schema, refs map[string]*jsonschema.Schema, ownFiel
 
 		mapValueType := getMapValueType(v, refs)
 		if mapValueType != nil {
-			d := getDescription(mapValueType, true)
+			d := getDescription(mapValueType)
 			if d != "" {
 				node.Description = d
 			}
@@ -174,7 +174,7 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri
 		attributes = append(attributes, attributeNode{
 			Title:       k,
 			Type:        typeString,
-			Description: getDescription(v, true),
+			Description: getDescription(v),
 			Link:        reference,
 		})
 	}
@@ -184,8 +184,8 @@ func getAttributes(props, refs map[string]*jsonschema.Schema, ownFields map[stri
 	return attributes
 }
 
-func getDescription(s *jsonschema.Schema, allowMarkdown bool) string {
+func getDescription(s *jsonschema.Schema) string {
-	if allowMarkdown && s.MarkdownDescription != "" {
+	if s.MarkdownDescription != "" {
 		return s.MarkdownDescription
 	}
 	return s.Description

bundle/docsgen/output/reference.md

@@ -43,7 +43,7 @@ artifacts:
   
    * - `files`
      - Sequence
-     - The source files for the artifact. See [_](#artifacts.<name>.files).
+     - The source files for the artifact. See [_](#artifactsnamefiles).
   
    * - `path`
      - String
@@ -64,7 +64,7 @@ artifacts:
     path: .
 ```
   
-### artifacts.<name>.files
+### artifacts.\<name\>.files
   
 **`Type: Sequence`**
   
@@ -113,11 +113,11 @@ The bundle attributes when deploying to this target,
   
    * - `deployment`
      - Map
-     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundle.deployment).
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#bundledeployment).
   
    * - `git`
      - Map
-     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundle.git).
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#bundlegit).
   
    * - `name`
      - String
@@ -132,7 +132,7 @@ The bundle attributes when deploying to this target,
   
 **`Type: Map`**
   
-The definition of the bundle deployment
+The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md).
   
   
   
@@ -149,7 +149,7 @@ The definition of the bundle deployment
   
    * - `lock`
      - Map
-     - The deployment lock attributes. See [_](#bundle.deployment.lock).
+     - The deployment lock attributes. See [_](#bundledeploymentlock).
   
   
 ### bundle.deployment.lock
@@ -180,7 +180,7 @@ The deployment lock attributes.
   
 **`Type: Map`**
   
-The Git version control details that are associated with your bundle.
+The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git).
   
   
   
@@ -217,11 +217,11 @@ Defines attributes for experimental features.
   
    * - `pydabs`
      - Map
-     - The PyDABs configuration. See [_](#experimental.pydabs).
+     - The PyDABs configuration. See [_](#experimentalpydabs).
   
    * - `python`
      - Map
-     - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimental.python).
+     - Configures loading of Python code defined with 'databricks-bundles' package. See [_](#experimentalpython).
   
    * - `python_wheel_wrapper`
      - Boolean
@@ -530,11 +530,11 @@ targets:
   
    * - `artifacts`
      - Map
-     - The artifacts to include in the target deployment. See [_](#targets.<name>.artifacts).
+     - The artifacts to include in the target deployment. See [_](#targetsnameartifacts).
   
    * - `bundle`
      - Map
-     - The bundle attributes when deploying to this target. See [_](#targets.<name>.bundle).
+     - The bundle attributes when deploying to this target. See [_](#targetsnamebundle).
   
    * - `cluster_id`
      - String
@@ -550,7 +550,7 @@ targets:
   
    * - `git`
      - Map
-     - The Git version control settings for the target. See [_](#targets.<name>.git).
+     - The Git version control settings for the target. See [_](#targetsnamegit).
   
    * - `mode`
      - String
@@ -558,34 +558,34 @@ targets:
   
    * - `permissions`
      - Sequence
-     - The permissions for deploying and running the bundle in the target. See [_](#targets.<name>.permissions).
+     - The permissions for deploying and running the bundle in the target. See [_](#targetsnamepermissions).
   
    * - `presets`
      - Map
-     - The deployment presets for the target. See [_](#targets.<name>.presets).
+     - The deployment presets for the target. See [_](#targetsnamepresets).
   
    * - `resources`
      - Map
-     - The resource definitions for the target. See [_](#targets.<name>.resources).
+     - The resource definitions for the target. See [_](#targetsnameresources).
   
    * - `run_as`
      - Map
-     - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targets.<name>.run_as).
+     - The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md). See [_](#targetsnamerun_as).
   
    * - `sync`
      - Map
-     - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targets.<name>.sync).
+     - The local paths to sync to the target workspace when a bundle is run or deployed. See [_](#targetsnamesync).
   
    * - `variables`
      - Map
-     - The custom variable definitions for the target. See [_](#targets.<name>.variables).
+     - The custom variable definitions for the target. See [_](#targetsnamevariables).
   
    * - `workspace`
      - Map
-     - The Databricks workspace for the target. See [_](#targets.<name>.workspace).
+     - The Databricks workspace for the target. See [_](#targetsnameworkspace).
   
   
-### targets.<name>.artifacts
+### targets.\<name\>.artifacts
   
 **`Type: Map`**
   
@@ -615,7 +615,7 @@ artifacts:
   
    * - `files`
      - Sequence
-     - The source files for the artifact. See [_](#targets.<name>.artifacts.<name>.files).
+     - The source files for the artifact. See [_](#targetsnameartifactsnamefiles).
   
    * - `path`
      - String
@@ -626,7 +626,7 @@ artifacts:
      - Required. The type of the artifact. Valid values are `whl`.
   
   
-### targets.<name>.artifacts.<name>.files
+### targets.\<name\>.artifacts.\<name\>.files
   
 **`Type: Sequence`**
   
@@ -646,7 +646,7 @@ The source files for the artifact.
      - Required. The path of the files used to build the artifact.
   
   
-### targets.<name>.bundle
+### targets.\<name\>.bundle
   
 **`Type: Map`**
   
@@ -675,11 +675,11 @@ The bundle attributes when deploying to this target.
   
    * - `deployment`
      - Map
-     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targets.<name>.bundle.deployment).
+     - The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md). See [_](#targetsnamebundledeployment).
   
    * - `git`
      - Map
-     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targets.<name>.bundle.git).
+     - The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git). See [_](#targetsnamebundlegit).
   
    * - `name`
      - String
@@ -690,11 +690,11 @@ The bundle attributes when deploying to this target.
      - Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).
   
   
-### targets.<name>.bundle.deployment
+### targets.\<name\>.bundle.deployment
   
 **`Type: Map`**
   
-The definition of the bundle deployment
+The definition of the bundle deployment. For supported attributes see [_](/dev-tools/bundles/deployment-modes.md).
   
   
   
@@ -711,10 +711,10 @@ The definition of the bundle deployment
   
    * - `lock`
      - Map
-     - The deployment lock attributes. See [_](#targets.<name>.bundle.deployment.lock).
+     - The deployment lock attributes. See [_](#targetsnamebundledeploymentlock).
   
   
-### targets.<name>.bundle.deployment.lock
+### targets.\<name\>.bundle.deployment.lock
   
 **`Type: Map`**
   
@@ -738,11 +738,11 @@ The deployment lock attributes.
      - Whether to force this lock if it is enabled.
   
   
-### targets.<name>.bundle.git
+### targets.\<name\>.bundle.git
   
 **`Type: Map`**
   
-The Git version control details that are associated with your bundle.
+The Git version control details that are associated with your bundle. For supported attributes see [_](/dev-tools/bundles/settings.md#git).
   
   
   
@@ -762,7 +762,7 @@ The Git version control details that are associated with your bundle.
      - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
   
   
-### targets.<name>.git
+### targets.\<name\>.git
   
 **`Type: Map`**
   
@@ -786,7 +786,7 @@ The Git version control settings for the target.
      - The origin URL of the repository. See [_](/dev-tools/bundles/settings.md#git).
   
   
-### targets.<name>.permissions
+### targets.\<name\>.permissions
   
 **`Type: Sequence`**
   
@@ -818,7 +818,7 @@ The permissions for deploying and running the bundle in the target.
      - The name of the user that has the permission set in level.
   
   
-### targets.<name>.presets
+### targets.\<name\>.presets
   
 **`Type: Map`**
   
@@ -858,7 +858,7 @@ The deployment presets for the target.
      - A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED.
   
   
-### targets.<name>.resources
+### targets.\<name\>.resources
   
 **`Type: Map`**
   
@@ -922,11 +922,11 @@ The resource definitions for the target.
      - The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
   
   
-### targets.<name>.run_as
+### targets.\<name\>.run_as
   
 **`Type: Map`**
   
-The identity to use to run the bundle.
+The identity to use to run the bundle, see [_](/dev-tools/bundles/run-as.md).
   
   
   
@@ -946,7 +946,7 @@ The identity to use to run the bundle.
      - The email of an active workspace user. Non-admin users can only set this field to their own email.
   
   
-### targets.<name>.sync
+### targets.\<name\>.sync
   
 **`Type: Map`**
   
@@ -974,7 +974,7 @@ The local paths to sync to the target workspace when a bundle is run or deployed
      - The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed.
   
   
-### targets.<name>.variables
+### targets.\<name\>.variables
   
 **`Type: Map`**
   
@@ -1004,14 +1004,14 @@ variables:
   
    * - `lookup`
      - Map
-     - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targets.<name>.variables.<name>.lookup).
+     - The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID. See [_](#targetsnamevariablesnamelookup).
   
    * - `type`
      - String
      - The type of the variable.
   
   
-### targets.<name>.variables.<name>.lookup
+### targets.\<name\>.variables.\<name\>.lookup
   
 **`Type: Map`**
   
@@ -1075,7 +1075,7 @@ The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, m
      - 
   
   
-### targets.<name>.workspace
+### targets.\<name\>.workspace
   
 **`Type: Map`**
   
@@ -1185,18 +1185,18 @@ variables:
   
    * - `lookup`
      - Map
-     - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variables.<name>.lookup).
+     - The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID. See [_](#variablesnamelookup).
   
    * - `type`
      - String
      - The type of the variable.
   
   
-### variables.<name>.lookup
+### variables.\<name\>.lookup
   
 **`Type: Map`**
   
-The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.
+The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID.
   
   
   

bundle/docsgen/testdata/anchors.md

@@ -0,0 +1,28 @@
+Header
+  
+## some_field
+  
+**`Type: Map`**
+  
+This is a description
+  
+  
+  
+.. list-table::
+   :header-rows: 1
+  
+   * - Key
+     - Type
+     - Description
+  
+   * - `my_attribute`
+     - Map
+     - Desc with link. See [_](#some_fieldnamemy_attribute).
+  
+  
+### some_field.\<name\>.my_attribute
+  
+**`Type: Boolean`**
+  
+Another description
+  

bundle/internal/schema/annotations.yml

@@ -414,16 +414,6 @@ github.com/databricks/cli/bundle/config/resources.Permission:
   "user_name":
     "description": |-
       The name of the user that has the permission set in level.
-github.com/databricks/cli/bundle/config/resources.Pipeline:
-  "allow_duplicate_names":
-    "description": |-
-      PLACEHOLDER
-  "dry_run":
-    "description": |-
-      PLACEHOLDER
-  "run_as":
-    "description": |-
-      PLACEHOLDER
 github.com/databricks/cli/bundle/config/variable.Lookup:
   "alert":
     "description": |-

bundle/internal/schema/main_test.go

@@ -124,3 +124,23 @@ func getAnnotations(path string) (annotation.File, error) {
 	err = yaml.Unmarshal(b, &data)
 	return data, err
 }
+
+func TestNoDuplicatedAnnotations(t *testing.T) {
+	// Check for duplicated annotations in annotation files
+	files := []string{
+		"annotations_openapi_overrides.yml",
+		"annotations.yml",
+	}
+
+	annotations := map[string]bool{}
+	for _, file := range files {
+		annotationsFile, err := getAnnotations(file)
+		assert.NoError(t, err)
+		for k := range annotationsFile {
+			if _, ok := annotations[k]; ok {
+				t.Errorf("Annotation `%s` is duplicated in %s", k, file)
+			}
+			annotations[k] = true
+		}
+	}
+}

libs/daemon/daemon.go

@@ -11,7 +11,8 @@ import (
 const DatabricksCliParentPid = "DATABRICKS_CLI_PARENT_PID"
 
 type Daemon struct {
-	// If provided, the child process will create a pid file at this path.
+	// If provided, the child process's pid will be written in the file at this
+	// path.
 	PidFilePath string
 
 	// Environment variables to set in the child process.
@@ -100,6 +101,8 @@ func (d *Daemon) Release() error {
 		}
 	}
 
+	// Note that the child process will stream it's output directly to the log file.
+	// So it's safe to close this file handle even if the child process is still running.
 	if d.logFile != nil {
 		err := d.logFile.Close()
 		if err != nil {

libs/daemon/daemon_test.go

@@ -0,0 +1,51 @@
+package daemon
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDaemon(t *testing.T) {
+	tmpDir := t.TempDir()
+	cmd := exec.Command("go", "run", "internal/parent_process/main.go", tmpDir)
+
+	// cmd.Run() will block until the parent process exits.
+	err := cmd.Run()
+	require.NoError(t, err)
+
+	// Assert that a PID file was created for the child process.
+	assert.FileExists(t, filepath.Join(tmpDir, "child.pid"))
+
+	// Wait 10 seconds for the server to start and to write the port number to
+	// a file.
+	portFilePath := filepath.Join(tmpDir, "port.txt")
+	assert.Eventually(t, func() bool {
+		_, err := os.Stat(portFilePath)
+		return err == nil
+	}, 10*time.Second, 100*time.Millisecond)
+
+	port, err := strconv.Atoi(testutil.ReadFile(t, portFilePath))
+	require.NoError(t, err)
+
+	// Query the local server, which should be alive even after the parent process
+	// has exited.
+	r, err := http.Get("http://localhost:" + strconv.Itoa(port))
+	require.NoError(t, err)
+	defer r.Body.Close()
+
+	// The server should respond with "child says hi".
+	assert.Equal(t, http.StatusOK, r.StatusCode)
+	b, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	assert.Equal(t, "child says hi", string(b))
+}

libs/daemon/internal/parent_process/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/databricks/cli/libs/daemon"
+)
+
+func main() {
+	tmpDir := os.Args[1]
+
+	d := daemon.Daemon{
+		PidFilePath: filepath.Join(tmpDir, "child.pid"),
+		Executable:  "python3",
+		// The server script writes the port number the server is listening on
+		// to the specified file.
+		Args: []string{"./internal/parent_process/server.py", filepath.Join(tmpDir, "port.txt")},
+	}
+
+	err := d.Start()
+	if err != nil {
+		panic(err)
+	}
+
+	err = d.Release()
+	if err != nil {
+		panic(err)
+	}
+}

libs/daemon/internal/parent_process/server.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import sys
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+if len(sys.argv) < 2:
+    print("Usage: python script.py <port_file_path>")
+    sys.exit(1)
+
+port_file_path = sys.argv[1]
+
+
+class SimpleHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        # Send HTTP 200 response with plain text content
+        self.send_response(200)
+        self.send_header("Content-type", "text/plain")
+        self.end_headers()
+        self.wfile.write(b"child says hi")
+
+
+# Bind to localhost on port 0 to let the OS pick an available port.
+server_address = ("localhost", 0)
+httpd = HTTPServer(server_address, SimpleHandler)
+
+# Retrieve the assigned port.
+assigned_port = httpd.server_address[1]
+
+# Write the port number to the provided file path.
+with open(port_file_path, "w") as f:
+    f.write(str(assigned_port))
+
+try:
+    # Automatically shut down the server after 2 minutes. This is a precaution to
+    # prevent the server from running indefinitely incase the GET API is never called.
+    httpd.timeout = 120
+
+    # This server will exit after one request.
+    httpd.handle_request()
+except KeyboardInterrupt:
+    print("\nServer is shutting down.")