Remove deprecated or readonly fields from the bundle schema (#1809)

## Changes
We do not need the user to specify these fields in their bundle
configuration, so we remove them from the JSON schema.

## Tests
Manually and end to end tests. The JSON schema has also been regenerated
after these changes.

bundle/internal/schema/main.go

@@ -8,8 +8,10 @@ import (
 	"reflect"
 
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/config/variable"
 	"github.com/databricks/cli/libs/jsonschema"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
 )
 
 func interpolationPattern(s string) string {
@@ -66,6 +68,31 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.
 	}
 }
 
+func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	switch typ {
+	case reflect.TypeOf(resources.Job{}):
+		// This field has been deprecated in jobs API v2.1 and is always set to
+		// "MULTI_TASK" in the backend. We should not expose it to the user.
+		delete(s.Properties, "format")
+
+		// These fields are only meant to be set by the DABs client (ie the CLI)
+		// and thus should not be exposed to the user. These are used to annotate
+		// jobs that were created by DABs.
+		delete(s.Properties, "deployment")
+		delete(s.Properties, "edit_mode")
+
+	case reflect.TypeOf(jobs.GitSource{}):
+		// These fields are readonly and are not meant to be set by the user.
+		delete(s.Properties, "job_source")
+		delete(s.Properties, "git_snapshot")
+
+	default:
+		// Do nothing
+	}
+
+	return s
+}
+
 func main() {
 	if len(os.Args) != 2 {
 		fmt.Println("Usage: go run main.go <output-file>")
@@ -90,6 +117,7 @@ func main() {
 	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
 		p.addDescriptions,
 		p.addEnums,
+		removeJobsFields,
 		addInterpolationPatterns,
 	})
 	if err != nil {

bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml

@@ -0,0 +1,4 @@
+resources:
+  jobs:
+    foo:
+      format: SINGLE_TASK

bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml

@@ -0,0 +1,6 @@
+resources:
+  jobs:
+    foo:
+      deployment:
+        kind: BUNDLE
+        metadata_file_path: /a/b/c

bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml

@@ -0,0 +1,6 @@
+targets:
+  foo:
+    resources:
+      jobs:
+        bar:
+          edit_mode: whatever

bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml

@@ -0,0 +1,8 @@
+resources:
+  jobs:
+    foo:
+      git_source:
+        git_provider: GITHUB
+        git_url: www.whatever.com
+        git_snapshot:
+          used_commit: abcdef

bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml

@@ -0,0 +1,9 @@
+resources:
+  jobs:
+    foo:
+      git_source:
+        git_provider: GITHUB
+        git_url: www.whatever.com
+        job_source:
+          import_from_git_branch: master
+          job_config_path: def

bundle/internal/schema/testdata/pass/job.yml

@@ -32,7 +32,6 @@ resources:
       name: myjob
       continuous:
         pause_status: PAUSED
-      edit_mode: EDITABLE
       max_concurrent_runs: 10
       description: "my job description"
       email_notifications:
@@ -43,10 +42,12 @@ resources:
             dependencies:
               - python=3.7
             client: "myclient"
-      format: MULTI_TASK
       tags:
         foo: bar
         bar: baz
+      git_source:
+        git_provider: gitHub
+        git_url: www.github.com/a/b
       tasks:
         - task_key: mytask
           notebook_task:

bundle/schema/embed_test.go

@@ -39,7 +39,7 @@ func TestJsonSchema(t *testing.T) {
 
 	// Assert job fields have their descriptions loaded.
 	resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job")
-	fields := []string{"name", "continuous", "deployment", "tasks", "trigger"}
+	fields := []string{"name", "continuous", "tasks", "trigger"}
 	for _, field := range fields {
 		assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description)
 	}
@@ -53,7 +53,7 @@ func TestJsonSchema(t *testing.T) {
 
 	// Assert descriptions are loaded for pipelines
 	pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline")
-	fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"}
+	fields = []string{"name", "catalog", "clusters", "channel", "continuous", "development"}
 	for _, field := range fields {
 		assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description)
 	}

bundle/schema/jsonschema.json

@@ -213,18 +213,10 @@
                         "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous"
                       },
-                      "deployment": {
-                        "description": "Deployment information for jobs managed by external sources.",
-                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment"
-                      },
                       "description": {
                         "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.",
                         "$ref": "#/$defs/string"
                       },
-                      "edit_mode": {
-                        "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.",
-                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode"
-                      },
                       "email_notifications": {
                         "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications"
@@ -233,10 +225,6 @@
                         "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.",
                         "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment"
                       },
-                      "format": {
-                        "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.",
-                        "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format"
-                      },
                       "git_source": {
                         "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource"
@@ -2547,9 +2535,6 @@
                       "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider"
                     },
-                    "git_snapshot": {
-                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot"
-                    },
                     "git_tag": {
                       "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.",
                       "$ref": "#/$defs/string"
@@ -2557,10 +2542,6 @@
                     "git_url": {
                       "description": "URL of the repository to be cloned by this job.",
                       "$ref": "#/$defs/string"
-                    },
-                    "job_source": {
-                      "description": "The source of the job specification in the remote repository when the job is source controlled.",
-                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource"
                     }
                   },
                   "additionalProperties": false,