From df8d39bf164283210e51715b01fa059441a6d882 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 20 Sep 2022 09:39:04 +0200 Subject: [PATCH] wip --- cmd/deploy/deploy.go | 14 +- cmd/sync/snapshot.go | 86 +- cmd/sync/sync.go | 24 +- example-project/.bricks/schema.json | 10844 ++++++++++++++++++++++++++ example-project/.gitignore | 2 + example-project/databricks.yml | 17 + example-project/my_first_job.py | 7 + example-project/resources.yml | 45 + project/environment.go | 1 + 9 files changed, 10982 insertions(+), 58 deletions(-) create mode 100644 example-project/.bricks/schema.json create mode 100644 example-project/.gitignore create mode 100644 example-project/databricks.yml create mode 100644 example-project/my_first_job.py create mode 100644 example-project/resources.yml diff --git a/cmd/deploy/deploy.go b/cmd/deploy/deploy.go index 49473072..577ef652 100644 --- a/cmd/deploy/deploy.go +++ b/cmd/deploy/deploy.go @@ -43,7 +43,10 @@ var deployCmd = &cobra.Command{ return err } - env := "dev" + env := cmd.Flag("environment").Value.String() + if env == "" { + env = "development" + } var out = map[string]interface{}{ "terraform": map[string]interface{}{ @@ -113,12 +116,19 @@ var deployCmd = &cobra.Command{ panic(err) } + for _, job := range out["resource"].(map[string]interface{})["databricks_job"].(map[string]interface{}) { + spark_python_task := job.(map[string]interface{})["spark_python_task"].(map[string]interface{}) + python_file := spark_python_task["python_file"].(string) + spark_python_task["python_file"] = filepath.Join(prj.Environment().Workspace.Root, python_file) + } } // Perform any string interpolation / string templating // TODO Make sure dist/env exists... + os.MkdirAll(filepath.Join(prj.Root(), "dist", env), 0755) + f, err := os.Create(filepath.Join(prj.Root(), "dist", env, "main.tf.json")) if err != nil { return err @@ -143,7 +153,7 @@ var deployCmd = &cobra.Command{ runtf := true if runtf { execPath := "/opt/homebrew/bin/terraform" - log.Printf("[INFO] tf exec path: %s", execPath) + // log.Printf("[INFO] tf exec path: %s", execPath) workingDir := filepath.Join(prj.Root(), "dist", env) tf, err := tfexec.NewTerraform(workingDir, execPath) diff --git a/cmd/sync/snapshot.go b/cmd/sync/snapshot.go index da7646e0..d355fcc8 100644 --- a/cmd/sync/snapshot.go +++ b/cmd/sync/snapshot.go @@ -1,11 +1,7 @@ package sync import ( - "encoding/json" "fmt" - "io" - "os" - "path/filepath" "strings" "time" @@ -23,54 +19,54 @@ const SyncSnapshotFile = "repo_snapshot.json" const BricksDir = ".bricks" func (s *snapshot) storeSnapshot(root string) error { - // create snapshot file - configDir := filepath.Join(root, BricksDir) - if _, err := os.Stat(configDir); os.IsNotExist(err) { - err = os.Mkdir(configDir, os.ModeDir|os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create config directory: %s", err) - } - } - persistedSnapshotPath := filepath.Join(configDir, SyncSnapshotFile) - f, err := os.OpenFile(persistedSnapshotPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) - if err != nil { - return fmt.Errorf("failed to create/open persisted sync snapshot file: %s", err) - } - defer f.Close() + // // create snapshot file + // configDir := filepath.Join(root, BricksDir) + // if _, err := os.Stat(configDir); os.IsNotExist(err) { + // err = os.Mkdir(configDir, os.ModeDir|os.ModePerm) + // if err != nil { + // return fmt.Errorf("failed to create config directory: %s", err) + // } + // } + // persistedSnapshotPath := filepath.Join(configDir, SyncSnapshotFile) + // f, err := os.OpenFile(persistedSnapshotPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + // if err != nil { + // return fmt.Errorf("failed to create/open persisted sync snapshot file: %s", err) + // } + // defer f.Close() - // persist snapshot to disk - bytes, err := json.MarshalIndent(s, "", " ") - if err != nil { - return fmt.Errorf("failed to json marshal in-memory snapshot: %s", err) - } - _, err = f.Write(bytes) - if err != nil { - return fmt.Errorf("failed to write sync snapshot to disk: %s", err) - } + // // persist snapshot to disk + // bytes, err := json.MarshalIndent(s, "", " ") + // if err != nil { + // return fmt.Errorf("failed to json marshal in-memory snapshot: %s", err) + // } + // _, err = f.Write(bytes) + // if err != nil { + // return fmt.Errorf("failed to write sync snapshot to disk: %s", err) + // } return nil } func (s *snapshot) loadSnapshot(root string) error { - persistedSnapshotPath := filepath.Join(root, BricksDir, SyncSnapshotFile) - if _, err := os.Stat(persistedSnapshotPath); os.IsNotExist(err) { - return nil - } + // persistedSnapshotPath := filepath.Join(root, BricksDir, SyncSnapshotFile) + // if _, err := os.Stat(persistedSnapshotPath); os.IsNotExist(err) { + // return nil + // } - f, err := os.Open(persistedSnapshotPath) - if err != nil { - return fmt.Errorf("failed to open persisted sync snapshot file: %s", err) - } - defer f.Close() + // f, err := os.Open(persistedSnapshotPath) + // if err != nil { + // return fmt.Errorf("failed to open persisted sync snapshot file: %s", err) + // } + // defer f.Close() - bytes, err := io.ReadAll(f) - if err != nil { - // clean up these error messages a bit - return fmt.Errorf("failed to read sync snapshot from disk: %s", err) - } - err = json.Unmarshal(bytes, s) - if err != nil { - return fmt.Errorf("failed to json unmarshal persisted snapshot: %s", err) - } + // bytes, err := io.ReadAll(f) + // if err != nil { + // // clean up these error messages a bit + // return fmt.Errorf("failed to read sync snapshot from disk: %s", err) + // } + // err = json.Unmarshal(bytes, s) + // if err != nil { + // return fmt.Errorf("failed to json unmarshal persisted snapshot: %s", err) + // } return nil } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 25cc4e5f..478eb777 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -2,7 +2,6 @@ package sync import ( "fmt" - "log" "time" "github.com/databricks/bricks/cmd/root" @@ -22,6 +21,10 @@ var syncCmd = &cobra.Command{ prj := project.Get(ctx) wsc := prj.WorkspacesClient() + if prj.Environment().Workspace.Root != "" { + *remotePath = prj.Environment().Workspace.Root + } + if *remotePath == "" { me, err := prj.Me() if err != nil { @@ -33,15 +36,15 @@ var syncCmd = &cobra.Command{ } *remotePath = fmt.Sprintf("/Repos/%s/%s", me.UserName, repositoryName) } - - log.Printf("[INFO] Remote file sync location: %v", *remotePath) - repoExists, err := git.RepoExists(*remotePath, ctx, wsc) - if err != nil { - return err - } - if !repoExists { - return fmt.Errorf("repo not found, please ensure %s exists", *remotePath) - } + var err error + // log.Printf("[INFO] Remote file sync location: %v", *remotePath) + // repoExists, err := git.RepoExists(*remotePath, ctx, wsc) + // if err != nil { + // return err + // } + // if !repoExists { + // return fmt.Errorf("repo not found, please ensure %s exists", *remotePath) + // } root := prj.Root() fileSet := git.NewFileSet(root) @@ -64,5 +67,4 @@ func init() { interval = syncCmd.Flags().Duration("interval", 1*time.Second, "project files polling interval") remotePath = syncCmd.Flags().String("remote-path", "", "remote path to store repo in. eg: /Repos/me@example.com/test-repo") syncCmd.Flags().StringP("environment", "e", "", "Environment to use") - } diff --git a/example-project/.bricks/schema.json b/example-project/.bricks/schema.json new file mode 100644 index 00000000..985490ef --- /dev/null +++ b/example-project/.bricks/schema.json @@ -0,0 +1,10844 @@ +{ + "properties": { + "environments": { + "additionalProperties": { + "properties": { + "resources": { + "properties": { + "aws_s3_mount": { + "additionalProperties": { + "properties": { + "cluster_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_profile": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "s3_bucket_name": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "mount_name", + "s3_bucket_name" + ] + }, + "type": "object" + }, + "azure_adls_gen1_mount": { + "additionalProperties": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "spark_conf_prefix": { + "type": "string" + }, + "storage_resource_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "tenant_id", + "mount_name", + "storage_resource_name", + "client_id", + "client_secret_key", + "client_secret_scope" + ] + }, + "type": "object" + }, + "azure_adls_gen2_mount": { + "additionalProperties": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "initialize_file_system": { + "type": "boolean" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "initialize_file_system", + "mount_name", + "tenant_id", + "client_secret_key", + "container_name", + "storage_account_name", + "client_id", + "client_secret_scope" + ] + }, + "type": "object" + }, + "azure_blob_mount": { + "additionalProperties": { + "properties": { + "auth_type": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "token_secret_key": { + "type": "string" + }, + "token_secret_scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "container_name", + "token_secret_key", + "auth_type", + "mount_name", + "storage_account_name", + "token_secret_scope" + ] + }, + "type": "object" + }, + "catalog": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "cluster": { + "additionalProperties": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "default_tags": { + "type": "object" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "is_pinned": { + "type": "boolean" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "state": { + "type": "string" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "delete": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "type": "object" + }, + "cluster_policy": { + "additionalProperties": { + "properties": { + "definition": { + "type": "string", + "description": "Policy definition JSON document expressed in\nDatabricks Policy Definition Language." + }, + "id": { + "type": "string" + }, + "name": { + "type": "string", + "description": "Cluster policy name. This must be unique.\nLength must be between 1 and 100 characters." + }, + "policy_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "dbfs_file": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "dbfs_path": { + "type": "string" + }, + "file_size": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "directory": { + "additionalProperties": { + "properties": { + "delete_recursive": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "object_id": { + "type": "integer" + }, + "path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "entitlements": { + "additionalProperties": { + "properties": { + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "service_principal_id": { + "type": "string" + }, + "user_id": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "external_location": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "credential_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "skip_validation": { + "type": "boolean" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url", + "credential_name", + "name" + ] + }, + "type": "object" + }, + "git_credential": { + "additionalProperties": { + "properties": { + "force": { + "type": "boolean" + }, + "git_provider": { + "type": "string" + }, + "git_username": { + "type": "string" + }, + "id": { + "type": "string" + }, + "personal_access_token": { + "type": "string" + } + }, + "type": "object", + "required": [ + "git_provider", + "git_username", + "personal_access_token" + ] + }, + "type": "object" + }, + "global_init_script": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "name": { + "type": "string" + }, + "position": { + "type": "integer" + }, + "source": { + "type": "string" + }, + "timeouts": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "grants": { + "additionalProperties": { + "properties": { + "catalog": { + "type": "string" + }, + "external_location": { + "type": "string" + }, + "function": { + "type": "string" + }, + "grant": { + "properties": { + "principal": { + "type": "string" + }, + "privileges": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "principal", + "privileges" + ] + }, + "id": { + "type": "string" + }, + "metastore": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "storage_credential": { + "type": "string" + }, + "table": { + "type": "string" + }, + "view": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "group": { + "additionalProperties": { + "properties": { + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "display_name" + ] + }, + "type": "object" + }, + "group_instance_profile": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_profile_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "group_id", + "instance_profile_id" + ] + }, + "type": "object" + }, + "group_member": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "member_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "member_id", + "group_id" + ] + }, + "type": "object" + }, + "group_role": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "role": { + "type": "string" + } + }, + "type": "object", + "required": [ + "group_id", + "role" + ] + }, + "type": "object" + }, + "instance_pool": { + "additionalProperties": { + "properties": { + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "custom_tags": { + "type": "object" + }, + "disk_spec": { + "properties": { + "disk_count": { + "type": "integer" + }, + "disk_size": { + "type": "integer" + }, + "disk_type": { + "properties": { + "azure_disk_volume_type": { + "type": "string" + }, + "ebs_volume_type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "idle_instance_autotermination_minutes": { + "type": "integer" + }, + "instance_pool_fleet_attributes": { + "properties": { + "fleet_on_demand_option": { + "properties": { + "allocation_strategy": { + "type": "string" + }, + "instance_pools_to_use_count": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "allocation_strategy" + ] + }, + "fleet_spot_option": { + "properties": { + "allocation_strategy": { + "type": "string" + }, + "instance_pools_to_use_count": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "allocation_strategy" + ] + }, + "launch_template_override": { + "properties": { + "availability_zone": { + "type": "string" + }, + "instance_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "availability_zone", + "instance_type" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "instance_pool_name": { + "type": "string" + }, + "max_capacity": { + "type": "integer" + }, + "min_idle_instances": { + "type": "integer" + }, + "node_type_id": { + "type": "string" + }, + "preloaded_docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "preloaded_spark_versions": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "idle_instance_autotermination_minutes", + "instance_pool_name" + ] + }, + "type": "object" + }, + "instance_profile": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "is_meta_instance_profile": { + "type": "boolean" + }, + "skip_validation": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "ip_access_list": { + "additionalProperties": { + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "ip_addresses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "label": { + "type": "string" + }, + "list_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ip_addresses", + "label", + "list_type" + ] + }, + "type": "object" + }, + "job": { + "additionalProperties": { + "properties": { + "always_running": { + "type": "boolean" + }, + "dbt_task": { + "properties": { + "commands": { + "items": { + "type": "string" + }, + "type": "array" + }, + "profiles_directory": { + "type": "string" + }, + "project_directory": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "commands" + ] + }, + "email_notifications": { + "properties": { + "alert_on_last_attempt": { + "type": "boolean" + }, + "no_alert_for_skipped_runs": { + "type": "boolean" + }, + "on_failure": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_start": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_success": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "existing_cluster_id": { + "type": "string" + }, + "format": { + "type": "string" + }, + "git_source": { + "properties": { + "branch": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "id": { + "type": "string" + }, + "job_cluster": { + "properties": { + "job_cluster_key": { + "type": "string" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + } + }, + "type": "object" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "max_concurrent_runs": { + "type": "integer" + }, + "max_retries": { + "type": "integer" + }, + "min_retry_interval_millis": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "notebook_task": { + "properties": { + "base_parameters": { + "type": "object" + }, + "notebook_path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "notebook_path" + ] + }, + "pipeline_task": { + "properties": { + "pipeline_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "pipeline_id" + ] + }, + "python_wheel_task": { + "properties": { + "entry_point": { + "type": "string" + }, + "named_parameters": { + "type": "object" + }, + "package_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "retry_on_timeout": { + "type": "boolean" + }, + "schedule": { + "properties": { + "pause_status": { + "type": "string" + }, + "quartz_cron_expression": { + "type": "string" + }, + "timezone_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + "spark_jar_task": { + "properties": { + "jar_uri": { + "type": "string" + }, + "main_class_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "spark_python_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + }, + "python_file": { + "type": "string" + } + }, + "type": "object", + "required": [ + "python_file" + ] + }, + "spark_submit_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "tags": { + "type": "object" + }, + "task": { + "properties": { + "dbt_task": { + "properties": { + "commands": { + "items": { + "type": "string" + }, + "type": "array" + }, + "profiles_directory": { + "type": "string" + }, + "project_directory": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "commands" + ] + }, + "depends_on": { + "properties": { + "task_key": { + "type": "string" + } + }, + "type": "object" + }, + "description": { + "type": "string" + }, + "email_notifications": { + "properties": { + "alert_on_last_attempt": { + "type": "boolean" + }, + "no_alert_for_skipped_runs": { + "type": "boolean" + }, + "on_failure": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_start": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_success": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "existing_cluster_id": { + "type": "string" + }, + "job_cluster_key": { + "type": "string" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "max_retries": { + "type": "integer" + }, + "min_retry_interval_millis": { + "type": "integer" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "notebook_task": { + "properties": { + "base_parameters": { + "type": "object" + }, + "notebook_path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "notebook_path" + ] + }, + "pipeline_task": { + "properties": { + "pipeline_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "pipeline_id" + ] + }, + "python_wheel_task": { + "properties": { + "entry_point": { + "type": "string" + }, + "named_parameters": { + "type": "object" + }, + "package_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "retry_on_timeout": { + "type": "boolean" + }, + "spark_jar_task": { + "properties": { + "jar_uri": { + "type": "string" + }, + "main_class_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "spark_python_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + }, + "python_file": { + "type": "string" + } + }, + "type": "object", + "required": [ + "python_file" + ] + }, + "spark_submit_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "sql_task": { + "properties": { + "alert": { + "properties": { + "alert_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "alert_id" + ] + }, + "dashboard": { + "properties": { + "dashboard_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dashboard_id" + ] + }, + "parameters": { + "type": "object" + }, + "query": { + "properties": { + "query_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "query_id" + ] + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object" + }, + "task_key": { + "type": "string" + }, + "timeout_seconds": { + "type": "integer" + } + }, + "type": "object" + }, + "timeout_seconds": { + "type": "integer" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "library": { + "additionalProperties": { + "properties": { + "cluster_id": { + "type": "string" + }, + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "id": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object", + "required": [ + "cluster_id" + ] + }, + "type": "object" + }, + "metastore": { + "additionalProperties": { + "properties": { + "cloud": { + "type": "string" + }, + "created_at": { + "type": "integer" + }, + "created_by": { + "type": "string" + }, + "default_data_access_config_id": { + "type": "string" + }, + "delta_sharing_organization_name": { + "type": "string" + }, + "delta_sharing_recipient_token_lifetime_in_seconds": { + "type": "integer" + }, + "delta_sharing_scope": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "global_metastore_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "region": { + "type": "string" + }, + "storage_root": { + "type": "string" + }, + "updated_at": { + "type": "integer" + }, + "updated_by": { + "type": "string" + } + }, + "type": "object", + "required": [ + "storage_root", + "name" + ] + }, + "type": "object" + }, + "metastore_assignment": { + "additionalProperties": { + "properties": { + "default_catalog_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "metastore_id", + "workspace_id" + ] + }, + "type": "object" + }, + "metastore_data_access": { + "additionalProperties": { + "properties": { + "aws_iam_role": { + "properties": { + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role_arn" + ] + }, + "azure_managed_identity": { + "properties": { + "access_connector_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_connector_id" + ] + }, + "azure_service_principal": { + "properties": { + "application_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "directory_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "directory_id", + "application_id", + "client_secret" + ] + }, + "configuration_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "is_default": { + "type": "boolean" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "metastore_id", + "name" + ] + }, + "type": "object" + }, + "mlflow_experiment": { + "additionalProperties": { + "properties": { + "artifact_location": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "experiment_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_update_time": { + "type": "integer" + }, + "lifecycle_stage": { + "type": "string" + }, + "name": { + "type": "string" + }, + "timeouts": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "mlflow_model": { + "additionalProperties": { + "properties": { + "creation_timestamp": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_updated_timestamp": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "registered_model_id": { + "type": "string" + }, + "tags": { + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "key", + "value" + ] + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "mlflow_webhook": { + "additionalProperties": { + "properties": { + "description": { + "type": "string" + }, + "events": { + "items": { + "type": "string" + }, + "type": "array" + }, + "http_url_spec": { + "properties": { + "authorization": { + "type": "string" + }, + "enable_ssl_verification": { + "type": "boolean" + }, + "string": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "id": { + "type": "string" + }, + "job_spec": { + "properties": { + "access_token": { + "type": "string" + }, + "job_id": { + "type": "string" + }, + "workspace_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_token", + "job_id" + ] + }, + "model_name": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object", + "required": [ + "events" + ] + }, + "type": "object" + }, + "mount": { + "additionalProperties": { + "properties": { + "abfs": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "initialize_file_system": { + "type": "boolean" + }, + "storage_account_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_secret_key", + "client_secret_scope", + "initialize_file_system", + "client_id" + ] + }, + "adl": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "spark_conf_prefix": { + "type": "string" + }, + "storage_resource_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_id", + "client_secret_key", + "client_secret_scope" + ] + }, + "cluster_id": { + "type": "string" + }, + "encryption_type": { + "type": "string" + }, + "extra_configs": { + "type": "object" + }, + "gs": { + "properties": { + "bucket_name": { + "type": "string" + }, + "service_account": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_id": { + "type": "string" + }, + "s3": { + "properties": { + "bucket_name": { + "type": "string" + }, + "instance_profile": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name" + ] + }, + "source": { + "type": "string" + }, + "timeouts": { + "properties": { + "default": { + "type": "string" + } + }, + "type": "object" + }, + "uri": { + "type": "string" + }, + "wasb": { + "properties": { + "auth_type": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "token_secret_key": { + "type": "string" + }, + "token_secret_scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "auth_type", + "token_secret_key", + "token_secret_scope" + ] + } + }, + "type": "object" + }, + "type": "object" + }, + "mws_credentials": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "credentials_id": { + "type": "string" + }, + "credentials_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "account_id", + "credentials_name", + "role_arn" + ] + }, + "type": "object" + }, + "mws_customer_managed_keys": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_key_info": { + "properties": { + "key_alias": { + "type": "string" + }, + "key_arn": { + "type": "string" + }, + "key_region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "key_alias", + "key_arn" + ] + }, + "creation_time": { + "type": "integer" + }, + "customer_managed_key_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "use_cases": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "account_id", + "use_cases" + ] + }, + "type": "object" + }, + "mws_log_delivery": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "config_id": { + "type": "string" + }, + "config_name": { + "type": "string" + }, + "credentials_id": { + "type": "string" + }, + "delivery_path_prefix": { + "type": "string" + }, + "delivery_start_time": { + "type": "string" + }, + "id": { + "type": "string" + }, + "log_type": { + "type": "string" + }, + "output_format": { + "type": "string" + }, + "status": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "workspace_ids_filter": { + "items": { + "type": "integer" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "log_type", + "output_format", + "storage_configuration_id", + "credentials_id", + "account_id" + ] + }, + "type": "object" + }, + "mws_networks": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "error_messages": { + "properties": { + "error_message": { + "type": "string" + }, + "error_type": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "network_id": { + "type": "string" + }, + "network_name": { + "type": "string" + }, + "security_group_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "subnet_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "vpc_endpoints": { + "properties": { + "dataplane_relay": { + "items": { + "type": "string" + }, + "type": "array" + }, + "rest_api": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "dataplane_relay", + "rest_api" + ] + }, + "vpc_id": { + "type": "string" + }, + "vpc_status": { + "type": "string" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "account_id", + "security_group_ids", + "vpc_id", + "network_name", + "subnet_ids" + ] + }, + "type": "object" + }, + "mws_permission_assignment": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permissions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "principal_id": { + "type": "integer" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "workspace_id", + "permissions", + "principal_id" + ] + }, + "type": "object" + }, + "mws_private_access_settings": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "allowed_vpc_endpoint_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "private_access_level": { + "type": "string" + }, + "private_access_settings_id": { + "type": "string" + }, + "private_access_settings_name": { + "type": "string" + }, + "public_access_enabled": { + "type": "boolean" + }, + "region": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object", + "required": [ + "private_access_settings_name", + "region" + ] + }, + "type": "object" + }, + "mws_storage_configurations": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "bucket_name": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "storage_configuration_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name", + "storage_configuration_name", + "account_id" + ] + }, + "type": "object" + }, + "mws_vpc_endpoint": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_account_id": { + "type": "string" + }, + "aws_endpoint_service_id": { + "type": "string" + }, + "aws_vpc_endpoint_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "state": { + "type": "string" + }, + "use_case": { + "type": "string" + }, + "vpc_endpoint_id": { + "type": "string" + }, + "vpc_endpoint_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "vpc_endpoint_name", + "aws_vpc_endpoint_id", + "region" + ] + }, + "type": "object" + }, + "mws_workspaces": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_region": { + "type": "string" + }, + "cloud": { + "type": "string" + }, + "cloud_resource_bucket": { + "properties": { + "gcp": { + "properties": { + "project_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "project_id" + ] + } + }, + "type": "object" + }, + "creation_time": { + "type": "integer" + }, + "credentials_id": { + "type": "string" + }, + "deployment_name": { + "type": "string" + }, + "external_customer_info": { + "properties": { + "authoritative_user_email": { + "type": "string" + }, + "authoritative_user_full_name": { + "type": "string" + }, + "customer_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "authoritative_user_full_name", + "customer_name", + "authoritative_user_email" + ] + }, + "id": { + "type": "string" + }, + "is_no_public_ip_enabled": { + "type": "boolean" + }, + "location": { + "type": "string" + }, + "managed_services_customer_managed_key_id": { + "type": "string" + }, + "network": { + "properties": { + "gcp_common_network_config": { + "properties": { + "gke_cluster_master_ip_range": { + "type": "string" + }, + "gke_connectivity_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "gke_cluster_master_ip_range", + "gke_connectivity_type" + ] + }, + "gcp_managed_network_config": { + "properties": { + "gke_cluster_pod_ip_range": { + "type": "string" + }, + "gke_cluster_service_ip_range": { + "type": "string" + }, + "subnet_cidr": { + "type": "string" + } + }, + "type": "object", + "required": [ + "gke_cluster_pod_ip_range", + "gke_cluster_service_ip_range", + "subnet_cidr" + ] + }, + "network_id": { + "type": "string" + } + }, + "type": "object" + }, + "network_id": { + "type": "string" + }, + "pricing_tier": { + "type": "string" + }, + "private_access_settings_id": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "storage_customer_managed_key_id": { + "type": "string" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "read": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "token": { + "properties": { + "comment": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_id": { + "type": "string" + }, + "token_value": { + "type": "string" + } + }, + "type": "object" + }, + "workspace_id": { + "type": "integer" + }, + "workspace_name": { + "type": "string" + }, + "workspace_status": { + "type": "string" + }, + "workspace_status_message": { + "type": "string" + }, + "workspace_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "account_id", + "workspace_name" + ] + }, + "type": "object" + }, + "notebook": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "format": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "obo_token": { + "additionalProperties": { + "properties": { + "application_id": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "application_id", + "comment", + "lifetime_seconds" + ] + }, + "type": "object" + }, + "permission_assignment": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permissions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "principal_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "permissions", + "principal_id" + ] + }, + "type": "object" + }, + "permissions": { + "additionalProperties": { + "properties": { + "access_control": { + "properties": { + "group_name": { + "type": "string" + }, + "permission_level": { + "type": "string" + }, + "service_principal_name": { + "type": "string" + }, + "user_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "permission_level" + ] + }, + "authorization": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "cluster_policy_id": { + "type": "string" + }, + "directory_id": { + "type": "string" + }, + "directory_path": { + "type": "string" + }, + "experiment_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_pool_id": { + "type": "string" + }, + "job_id": { + "type": "string" + }, + "notebook_id": { + "type": "string" + }, + "notebook_path": { + "type": "string" + }, + "object_type": { + "type": "string" + }, + "pipeline_id": { + "type": "string" + }, + "registered_model_id": { + "type": "string" + }, + "repo_id": { + "type": "string" + }, + "repo_path": { + "type": "string" + }, + "sql_alert_id": { + "type": "string" + }, + "sql_dashboard_id": { + "type": "string" + }, + "sql_endpoint_id": { + "type": "string" + }, + "sql_query_id": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "pipeline": { + "additionalProperties": { + "properties": { + "allow_duplicate_names": { + "type": "boolean" + }, + "channel": { + "type": "string" + }, + "cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "aws_attributes": { + "properties": { + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "custom_tags": { + "type": "object" + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "gcp_attributes": { + "properties": { + "google_service_account": { + "type": "string" + } + }, + "type": "object" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "configuration": { + "type": "object" + }, + "continuous": { + "type": "boolean" + }, + "development": { + "type": "boolean" + }, + "edition": { + "type": "string" + }, + "filters": { + "properties": { + "exclude": { + "items": { + "type": "string" + }, + "type": "array" + }, + "include": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "library": { + "properties": { + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "notebook": { + "properties": { + "path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "photon": { + "type": "boolean" + }, + "storage": { + "type": "string" + }, + "target": { + "type": "string" + }, + "timeouts": { + "properties": { + "default": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "recipient": { + "additionalProperties": { + "properties": { + "authentication_type": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "data_recipient_global_metastore_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "ip_access_list": { + "properties": { + "allowed_ip_addresses": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "allowed_ip_addresses" + ] + }, + "name": { + "type": "string" + }, + "sharing_code": { + "type": "string" + }, + "tokens": { + "properties": { + "activation_url": { + "type": "string" + }, + "created_at": { + "type": "integer" + }, + "created_by": { + "type": "string" + }, + "expiration_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "updated_at": { + "type": "integer" + }, + "updated_by": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "authentication_type", + "name" + ] + }, + "type": "object" + }, + "repo": { + "additionalProperties": { + "properties": { + "branch": { + "type": "string" + }, + "commit_hash": { + "type": "string" + }, + "git_provider": { + "type": "string" + }, + "id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "type": "object" + }, + "schema": { + "additionalProperties": { + "properties": { + "catalog_name": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "catalog_name" + ] + }, + "type": "object" + }, + "secret": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "last_updated_timestamp": { + "type": "integer" + }, + "scope": { + "type": "string" + }, + "string_value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "string_value", + "key", + "scope" + ] + }, + "type": "object" + }, + "secret_acl": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permission": { + "type": "string" + }, + "principal": { + "type": "string" + }, + "scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "permission", + "principal", + "scope" + ] + }, + "type": "object" + }, + "secret_scope": { + "additionalProperties": { + "properties": { + "backend_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "initial_manage_principal": { + "type": "string" + }, + "keyvault_metadata": { + "properties": { + "dns_name": { + "type": "string" + }, + "resource_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dns_name", + "resource_id" + ] + }, + "name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "service_principal": { + "additionalProperties": { + "properties": { + "active": { + "type": "boolean" + }, + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "application_id": { + "type": "string" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "service_principal_role": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "role": { + "type": "string" + }, + "service_principal_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role", + "service_principal_id" + ] + }, + "type": "object" + }, + "sql_dashboard": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "sql_endpoint": { + "additionalProperties": { + "properties": { + "auto_stop_mins": { + "type": "integer" + }, + "channel": { + "properties": { + "name": { + "type": "string" + } + }, + "type": "object" + }, + "cluster_size": { + "type": "string" + }, + "data_source_id": { + "type": "string" + }, + "enable_photon": { + "type": "boolean" + }, + "enable_serverless_compute": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "jdbc_url": { + "type": "string" + }, + "max_num_clusters": { + "type": "integer" + }, + "min_num_clusters": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "num_clusters": { + "type": "integer" + }, + "odbc_params": { + "properties": { + "host": { + "type": "string" + }, + "hostname": { + "type": "string" + }, + "path": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string" + } + }, + "type": "object", + "required": [ + "port", + "protocol", + "path" + ] + }, + "spot_instance_policy": { + "type": "string" + }, + "state": { + "type": "string" + }, + "tags": { + "properties": { + "custom_tags": { + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value", + "key" + ] + } + }, + "type": "object" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "cluster_size" + ] + }, + "type": "object" + }, + "sql_global_config": { + "additionalProperties": { + "properties": { + "data_access_config": { + "type": "object" + }, + "enable_serverless_compute": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "security_policy": { + "type": "string" + }, + "sql_config_params": { + "type": "object" + } + }, + "type": "object" + }, + "type": "object" + }, + "sql_permissions": { + "additionalProperties": { + "properties": { + "anonymous_function": { + "type": "boolean" + }, + "any_file": { + "type": "boolean" + }, + "catalog": { + "type": "boolean" + }, + "cluster_id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "id": { + "type": "string" + }, + "privilege_assignments": { + "properties": { + "principal": { + "type": "string" + }, + "privileges": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "principal", + "privileges" + ] + }, + "table": { + "type": "string" + }, + "view": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "sql_query": { + "additionalProperties": { + "properties": { + "data_source_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "parameter": { + "properties": { + "date": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "date_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetime": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetime_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetimesec": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetimesec_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "enum": { + "properties": { + "multiple": { + "properties": { + "prefix": { + "type": "string" + }, + "separator": { + "type": "string" + }, + "suffix": { + "type": "string" + } + }, + "type": "object", + "required": [ + "prefix", + "separator", + "suffix" + ] + }, + "options": { + "items": { + "type": "string" + }, + "type": "array" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "options" + ] + }, + "name": { + "type": "string" + }, + "number": { + "properties": { + "value": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "query": { + "properties": { + "multiple": { + "properties": { + "prefix": { + "type": "string" + }, + "separator": { + "type": "string" + }, + "suffix": { + "type": "string" + } + }, + "type": "object", + "required": [ + "prefix", + "separator", + "suffix" + ] + }, + "query_id": { + "type": "string" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "query_id" + ] + }, + "text": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "title": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "query": { + "type": "string" + }, + "run_as_role": { + "type": "string" + }, + "schedule": { + "properties": { + "continuous": { + "properties": { + "interval_seconds": { + "type": "integer" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "interval_seconds" + ] + }, + "daily": { + "properties": { + "interval_days": { + "type": "integer" + }, + "time_of_day": { + "type": "string" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "interval_days", + "time_of_day" + ] + }, + "weekly": { + "properties": { + "day_of_week": { + "type": "string" + }, + "interval_weeks": { + "type": "integer" + }, + "time_of_day": { + "type": "string" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "day_of_week", + "interval_weeks", + "time_of_day" + ] + } + }, + "type": "object" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "query", + "data_source_id" + ] + }, + "type": "object" + }, + "sql_visualization": { + "additionalProperties": { + "properties": { + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "options": { + "type": "string" + }, + "query_id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "visualization_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "options", + "query_id", + "type" + ] + }, + "type": "object" + }, + "sql_widget": { + "additionalProperties": { + "properties": { + "dashboard_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "parameter": { + "properties": { + "map_to": { + "type": "string" + }, + "name": { + "type": "string" + }, + "title": { + "type": "string" + }, + "type": { + "type": "string" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "type" + ] + }, + "position": { + "properties": { + "auto_height": { + "type": "boolean" + }, + "pos_x": { + "type": "integer" + }, + "pos_y": { + "type": "integer" + }, + "size_x": { + "type": "integer" + }, + "size_y": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "size_x", + "size_y" + ] + }, + "text": { + "type": "string" + }, + "title": { + "type": "string" + }, + "visualization_id": { + "type": "string" + }, + "widget_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dashboard_id" + ] + }, + "type": "object" + }, + "storage_credential": { + "additionalProperties": { + "properties": { + "aws_iam_role": { + "properties": { + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role_arn" + ] + }, + "azure_managed_identity": { + "properties": { + "access_connector_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_connector_id" + ] + }, + "azure_service_principal": { + "properties": { + "application_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "directory_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_secret", + "directory_id", + "application_id" + ] + }, + "comment": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "table": { + "additionalProperties": { + "properties": { + "catalog_name": { + "type": "string" + }, + "column": { + "properties": { + "comment": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nullable": { + "type": "boolean" + }, + "partition_index": { + "type": "integer" + }, + "position": { + "type": "integer" + }, + "type_interval_type": { + "type": "string" + }, + "type_json": { + "type": "string" + }, + "type_name": { + "type": "string" + }, + "type_precision": { + "type": "integer" + }, + "type_scale": { + "type": "integer" + }, + "type_text": { + "type": "string" + } + }, + "type": "object", + "required": [ + "position", + "type_text", + "name", + "type_name" + ] + }, + "comment": { + "type": "string" + }, + "data_source_format": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + }, + "schema_name": { + "type": "string" + }, + "storage_credential_name": { + "type": "string" + }, + "storage_location": { + "type": "string" + }, + "table_type": { + "type": "string" + }, + "view_definition": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "schema_name", + "data_source_format", + "table_type", + "catalog_name" + ] + }, + "type": "object" + }, + "token": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "expiry_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_id": { + "type": "string" + }, + "token_value": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "user": { + "additionalProperties": { + "properties": { + "active": { + "type": "boolean" + }, + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "user_name" + ] + }, + "type": "object" + }, + "user_instance_profile": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "instance_profile_id": { + "type": "string" + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "instance_profile_id", + "user_id" + ] + }, + "type": "object" + }, + "user_role": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "role": { + "type": "string" + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role", + "user_id" + ] + }, + "type": "object" + }, + "workspace_conf": { + "additionalProperties": { + "properties": { + "custom_config": { + "type": "object" + }, + "id": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + } + }, + "type": "object", + "title": "resources" + } + }, + "type": "object" + }, + "type": "object" + }, + "resources": { + "properties": { + "aws_s3_mount": { + "additionalProperties": { + "properties": { + "cluster_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_profile": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "s3_bucket_name": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "mount_name", + "s3_bucket_name" + ] + }, + "type": "object" + }, + "azure_adls_gen1_mount": { + "additionalProperties": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "spark_conf_prefix": { + "type": "string" + }, + "storage_resource_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "tenant_id", + "mount_name", + "storage_resource_name", + "client_id", + "client_secret_key", + "client_secret_scope" + ] + }, + "type": "object" + }, + "azure_adls_gen2_mount": { + "additionalProperties": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "initialize_file_system": { + "type": "boolean" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "initialize_file_system", + "mount_name", + "tenant_id", + "client_secret_key", + "container_name", + "storage_account_name", + "client_id", + "client_secret_scope" + ] + }, + "type": "object" + }, + "azure_blob_mount": { + "additionalProperties": { + "properties": { + "auth_type": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "id": { + "type": "string" + }, + "mount_name": { + "type": "string" + }, + "source": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "token_secret_key": { + "type": "string" + }, + "token_secret_scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "container_name", + "token_secret_key", + "auth_type", + "mount_name", + "storage_account_name", + "token_secret_scope" + ] + }, + "type": "object" + }, + "catalog": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "cluster": { + "additionalProperties": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "default_tags": { + "type": "object" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "is_pinned": { + "type": "boolean" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "state": { + "type": "string" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "delete": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "type": "object" + }, + "cluster_policy": { + "additionalProperties": { + "properties": { + "definition": { + "type": "string", + "description": "Policy definition JSON document expressed in\nDatabricks Policy Definition Language." + }, + "id": { + "type": "string" + }, + "name": { + "type": "string", + "description": "Cluster policy name. This must be unique.\nLength must be between 1 and 100 characters." + }, + "policy_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "dbfs_file": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "dbfs_path": { + "type": "string" + }, + "file_size": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "directory": { + "additionalProperties": { + "properties": { + "delete_recursive": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "object_id": { + "type": "integer" + }, + "path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "entitlements": { + "additionalProperties": { + "properties": { + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "service_principal_id": { + "type": "string" + }, + "user_id": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "external_location": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "credential_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "skip_validation": { + "type": "boolean" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url", + "credential_name", + "name" + ] + }, + "type": "object" + }, + "git_credential": { + "additionalProperties": { + "properties": { + "force": { + "type": "boolean" + }, + "git_provider": { + "type": "string" + }, + "git_username": { + "type": "string" + }, + "id": { + "type": "string" + }, + "personal_access_token": { + "type": "string" + } + }, + "type": "object", + "required": [ + "git_provider", + "git_username", + "personal_access_token" + ] + }, + "type": "object" + }, + "global_init_script": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "name": { + "type": "string" + }, + "position": { + "type": "integer" + }, + "source": { + "type": "string" + }, + "timeouts": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "grants": { + "additionalProperties": { + "properties": { + "catalog": { + "type": "string" + }, + "external_location": { + "type": "string" + }, + "function": { + "type": "string" + }, + "grant": { + "properties": { + "principal": { + "type": "string" + }, + "privileges": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "principal", + "privileges" + ] + }, + "id": { + "type": "string" + }, + "metastore": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "storage_credential": { + "type": "string" + }, + "table": { + "type": "string" + }, + "view": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "group": { + "additionalProperties": { + "properties": { + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "display_name" + ] + }, + "type": "object" + }, + "group_instance_profile": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_profile_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "group_id", + "instance_profile_id" + ] + }, + "type": "object" + }, + "group_member": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "member_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "member_id", + "group_id" + ] + }, + "type": "object" + }, + "group_role": { + "additionalProperties": { + "properties": { + "group_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "role": { + "type": "string" + } + }, + "type": "object", + "required": [ + "group_id", + "role" + ] + }, + "type": "object" + }, + "instance_pool": { + "additionalProperties": { + "properties": { + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "custom_tags": { + "type": "object" + }, + "disk_spec": { + "properties": { + "disk_count": { + "type": "integer" + }, + "disk_size": { + "type": "integer" + }, + "disk_type": { + "properties": { + "azure_disk_volume_type": { + "type": "string" + }, + "ebs_volume_type": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "idle_instance_autotermination_minutes": { + "type": "integer" + }, + "instance_pool_fleet_attributes": { + "properties": { + "fleet_on_demand_option": { + "properties": { + "allocation_strategy": { + "type": "string" + }, + "instance_pools_to_use_count": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "allocation_strategy" + ] + }, + "fleet_spot_option": { + "properties": { + "allocation_strategy": { + "type": "string" + }, + "instance_pools_to_use_count": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "allocation_strategy" + ] + }, + "launch_template_override": { + "properties": { + "availability_zone": { + "type": "string" + }, + "instance_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "availability_zone", + "instance_type" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "instance_pool_name": { + "type": "string" + }, + "max_capacity": { + "type": "integer" + }, + "min_idle_instances": { + "type": "integer" + }, + "node_type_id": { + "type": "string" + }, + "preloaded_docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "preloaded_spark_versions": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "idle_instance_autotermination_minutes", + "instance_pool_name" + ] + }, + "type": "object" + }, + "instance_profile": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "is_meta_instance_profile": { + "type": "boolean" + }, + "skip_validation": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "ip_access_list": { + "additionalProperties": { + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "ip_addresses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "label": { + "type": "string" + }, + "list_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ip_addresses", + "label", + "list_type" + ] + }, + "type": "object" + }, + "job": { + "additionalProperties": { + "properties": { + "always_running": { + "type": "boolean" + }, + "dbt_task": { + "properties": { + "commands": { + "items": { + "type": "string" + }, + "type": "array" + }, + "profiles_directory": { + "type": "string" + }, + "project_directory": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "commands" + ] + }, + "email_notifications": { + "properties": { + "alert_on_last_attempt": { + "type": "boolean" + }, + "no_alert_for_skipped_runs": { + "type": "boolean" + }, + "on_failure": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_start": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_success": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "existing_cluster_id": { + "type": "string" + }, + "format": { + "type": "string" + }, + "git_source": { + "properties": { + "branch": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "id": { + "type": "string" + }, + "job_cluster": { + "properties": { + "job_cluster_key": { + "type": "string" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + } + }, + "type": "object" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "max_concurrent_runs": { + "type": "integer" + }, + "max_retries": { + "type": "integer" + }, + "min_retry_interval_millis": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "notebook_task": { + "properties": { + "base_parameters": { + "type": "object" + }, + "notebook_path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "notebook_path" + ] + }, + "pipeline_task": { + "properties": { + "pipeline_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "pipeline_id" + ] + }, + "python_wheel_task": { + "properties": { + "entry_point": { + "type": "string" + }, + "named_parameters": { + "type": "object" + }, + "package_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "retry_on_timeout": { + "type": "boolean" + }, + "schedule": { + "properties": { + "pause_status": { + "type": "string" + }, + "quartz_cron_expression": { + "type": "string" + }, + "timezone_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + "spark_jar_task": { + "properties": { + "jar_uri": { + "type": "string" + }, + "main_class_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "spark_python_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + }, + "python_file": { + "type": "string" + } + }, + "type": "object", + "required": [ + "python_file" + ] + }, + "spark_submit_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "tags": { + "type": "object" + }, + "task": { + "properties": { + "dbt_task": { + "properties": { + "commands": { + "items": { + "type": "string" + }, + "type": "array" + }, + "profiles_directory": { + "type": "string" + }, + "project_directory": { + "type": "string" + }, + "schema": { + "type": "string" + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "commands" + ] + }, + "depends_on": { + "properties": { + "task_key": { + "type": "string" + } + }, + "type": "object" + }, + "description": { + "type": "string" + }, + "email_notifications": { + "properties": { + "alert_on_last_attempt": { + "type": "boolean" + }, + "no_alert_for_skipped_runs": { + "type": "boolean" + }, + "on_failure": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_start": { + "items": { + "type": "string" + }, + "type": "array" + }, + "on_success": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "existing_cluster_id": { + "type": "string" + }, + "job_cluster_key": { + "type": "string" + }, + "library": { + "properties": { + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "max_retries": { + "type": "integer" + }, + "min_retry_interval_millis": { + "type": "integer" + }, + "new_cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "autotermination_minutes": { + "type": "integer" + }, + "aws_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "ebs_volume_count": { + "type": "integer" + }, + "ebs_volume_size": { + "type": "integer" + }, + "ebs_volume_type": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "spot_bid_price_percent": { + "type": "integer" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "azure_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "first_on_demand": { + "type": "integer" + }, + "spot_bid_max_price": { + "type": "integer" + } + }, + "type": "object" + }, + "cluster_id": { + "type": "string" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "cluster_name": { + "type": "string" + }, + "custom_tags": { + "type": "object" + }, + "data_security_mode": { + "type": "string" + }, + "docker_image": { + "properties": { + "basic_auth": { + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object", + "required": [ + "password", + "username" + ] + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "enable_elastic_disk": { + "type": "boolean" + }, + "enable_local_disk_encryption": { + "type": "boolean" + }, + "gcp_attributes": { + "properties": { + "availability": { + "type": "string" + }, + "boot_disk_size": { + "type": "integer" + }, + "google_service_account": { + "type": "string" + }, + "use_preemptible_executors": { + "type": "boolean" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "idempotency_token": { + "type": "string" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "single_user_name": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "spark_version": { + "type": "string" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + }, + "workload_type": { + "properties": { + "clients": { + "properties": { + "jobs": { + "type": "boolean" + }, + "notebooks": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "spark_version" + ] + }, + "notebook_task": { + "properties": { + "base_parameters": { + "type": "object" + }, + "notebook_path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "notebook_path" + ] + }, + "pipeline_task": { + "properties": { + "pipeline_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "pipeline_id" + ] + }, + "python_wheel_task": { + "properties": { + "entry_point": { + "type": "string" + }, + "named_parameters": { + "type": "object" + }, + "package_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "retry_on_timeout": { + "type": "boolean" + }, + "spark_jar_task": { + "properties": { + "jar_uri": { + "type": "string" + }, + "main_class_name": { + "type": "string" + }, + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "spark_python_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + }, + "python_file": { + "type": "string" + } + }, + "type": "object", + "required": [ + "python_file" + ] + }, + "spark_submit_task": { + "properties": { + "parameters": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "sql_task": { + "properties": { + "alert": { + "properties": { + "alert_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "alert_id" + ] + }, + "dashboard": { + "properties": { + "dashboard_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dashboard_id" + ] + }, + "parameters": { + "type": "object" + }, + "query": { + "properties": { + "query_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "query_id" + ] + }, + "warehouse_id": { + "type": "string" + } + }, + "type": "object" + }, + "task_key": { + "type": "string" + }, + "timeout_seconds": { + "type": "integer" + } + }, + "type": "object" + }, + "timeout_seconds": { + "type": "integer" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "library": { + "additionalProperties": { + "properties": { + "cluster_id": { + "type": "string" + }, + "cran": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "egg": { + "type": "string" + }, + "id": { + "type": "string" + }, + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "pypi": { + "properties": { + "package": { + "type": "string" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "package" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object", + "required": [ + "cluster_id" + ] + }, + "type": "object" + }, + "metastore": { + "additionalProperties": { + "properties": { + "cloud": { + "type": "string" + }, + "created_at": { + "type": "integer" + }, + "created_by": { + "type": "string" + }, + "default_data_access_config_id": { + "type": "string" + }, + "delta_sharing_organization_name": { + "type": "string" + }, + "delta_sharing_recipient_token_lifetime_in_seconds": { + "type": "integer" + }, + "delta_sharing_scope": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "global_metastore_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "region": { + "type": "string" + }, + "storage_root": { + "type": "string" + }, + "updated_at": { + "type": "integer" + }, + "updated_by": { + "type": "string" + } + }, + "type": "object", + "required": [ + "storage_root", + "name" + ] + }, + "type": "object" + }, + "metastore_assignment": { + "additionalProperties": { + "properties": { + "default_catalog_name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "metastore_id", + "workspace_id" + ] + }, + "type": "object" + }, + "metastore_data_access": { + "additionalProperties": { + "properties": { + "aws_iam_role": { + "properties": { + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role_arn" + ] + }, + "azure_managed_identity": { + "properties": { + "access_connector_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_connector_id" + ] + }, + "azure_service_principal": { + "properties": { + "application_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "directory_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "directory_id", + "application_id", + "client_secret" + ] + }, + "configuration_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "is_default": { + "type": "boolean" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "metastore_id", + "name" + ] + }, + "type": "object" + }, + "mlflow_experiment": { + "additionalProperties": { + "properties": { + "artifact_location": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "experiment_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_update_time": { + "type": "integer" + }, + "lifecycle_stage": { + "type": "string" + }, + "name": { + "type": "string" + }, + "timeouts": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "mlflow_model": { + "additionalProperties": { + "properties": { + "creation_timestamp": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_updated_timestamp": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "registered_model_id": { + "type": "string" + }, + "tags": { + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "key", + "value" + ] + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "mlflow_webhook": { + "additionalProperties": { + "properties": { + "description": { + "type": "string" + }, + "events": { + "items": { + "type": "string" + }, + "type": "array" + }, + "http_url_spec": { + "properties": { + "authorization": { + "type": "string" + }, + "enable_ssl_verification": { + "type": "boolean" + }, + "string": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "id": { + "type": "string" + }, + "job_spec": { + "properties": { + "access_token": { + "type": "string" + }, + "job_id": { + "type": "string" + }, + "workspace_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_token", + "job_id" + ] + }, + "model_name": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object", + "required": [ + "events" + ] + }, + "type": "object" + }, + "mount": { + "additionalProperties": { + "properties": { + "abfs": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "initialize_file_system": { + "type": "boolean" + }, + "storage_account_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_secret_key", + "client_secret_scope", + "initialize_file_system", + "client_id" + ] + }, + "adl": { + "properties": { + "client_id": { + "type": "string" + }, + "client_secret_key": { + "type": "string" + }, + "client_secret_scope": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "spark_conf_prefix": { + "type": "string" + }, + "storage_resource_name": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_id", + "client_secret_key", + "client_secret_scope" + ] + }, + "cluster_id": { + "type": "string" + }, + "encryption_type": { + "type": "string" + }, + "extra_configs": { + "type": "object" + }, + "gs": { + "properties": { + "bucket_name": { + "type": "string" + }, + "service_account": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name" + ] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "resource_id": { + "type": "string" + }, + "s3": { + "properties": { + "bucket_name": { + "type": "string" + }, + "instance_profile": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name" + ] + }, + "source": { + "type": "string" + }, + "timeouts": { + "properties": { + "default": { + "type": "string" + } + }, + "type": "object" + }, + "uri": { + "type": "string" + }, + "wasb": { + "properties": { + "auth_type": { + "type": "string" + }, + "container_name": { + "type": "string" + }, + "directory": { + "type": "string" + }, + "storage_account_name": { + "type": "string" + }, + "token_secret_key": { + "type": "string" + }, + "token_secret_scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "auth_type", + "token_secret_key", + "token_secret_scope" + ] + } + }, + "type": "object" + }, + "type": "object" + }, + "mws_credentials": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "credentials_id": { + "type": "string" + }, + "credentials_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "account_id", + "credentials_name", + "role_arn" + ] + }, + "type": "object" + }, + "mws_customer_managed_keys": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_key_info": { + "properties": { + "key_alias": { + "type": "string" + }, + "key_arn": { + "type": "string" + }, + "key_region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "key_alias", + "key_arn" + ] + }, + "creation_time": { + "type": "integer" + }, + "customer_managed_key_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "use_cases": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "account_id", + "use_cases" + ] + }, + "type": "object" + }, + "mws_log_delivery": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "config_id": { + "type": "string" + }, + "config_name": { + "type": "string" + }, + "credentials_id": { + "type": "string" + }, + "delivery_path_prefix": { + "type": "string" + }, + "delivery_start_time": { + "type": "string" + }, + "id": { + "type": "string" + }, + "log_type": { + "type": "string" + }, + "output_format": { + "type": "string" + }, + "status": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "workspace_ids_filter": { + "items": { + "type": "integer" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "log_type", + "output_format", + "storage_configuration_id", + "credentials_id", + "account_id" + ] + }, + "type": "object" + }, + "mws_networks": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "error_messages": { + "properties": { + "error_message": { + "type": "string" + }, + "error_type": { + "type": "string" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "network_id": { + "type": "string" + }, + "network_name": { + "type": "string" + }, + "security_group_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "subnet_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "vpc_endpoints": { + "properties": { + "dataplane_relay": { + "items": { + "type": "string" + }, + "type": "array" + }, + "rest_api": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "dataplane_relay", + "rest_api" + ] + }, + "vpc_id": { + "type": "string" + }, + "vpc_status": { + "type": "string" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "account_id", + "security_group_ids", + "vpc_id", + "network_name", + "subnet_ids" + ] + }, + "type": "object" + }, + "mws_permission_assignment": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permissions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "principal_id": { + "type": "integer" + }, + "workspace_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "workspace_id", + "permissions", + "principal_id" + ] + }, + "type": "object" + }, + "mws_private_access_settings": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "allowed_vpc_endpoint_ids": { + "items": { + "type": "string" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "private_access_level": { + "type": "string" + }, + "private_access_settings_id": { + "type": "string" + }, + "private_access_settings_name": { + "type": "string" + }, + "public_access_enabled": { + "type": "boolean" + }, + "region": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object", + "required": [ + "private_access_settings_name", + "region" + ] + }, + "type": "object" + }, + "mws_storage_configurations": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "bucket_name": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "storage_configuration_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "bucket_name", + "storage_configuration_name", + "account_id" + ] + }, + "type": "object" + }, + "mws_vpc_endpoint": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_account_id": { + "type": "string" + }, + "aws_endpoint_service_id": { + "type": "string" + }, + "aws_vpc_endpoint_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "region": { + "type": "string" + }, + "state": { + "type": "string" + }, + "use_case": { + "type": "string" + }, + "vpc_endpoint_id": { + "type": "string" + }, + "vpc_endpoint_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "vpc_endpoint_name", + "aws_vpc_endpoint_id", + "region" + ] + }, + "type": "object" + }, + "mws_workspaces": { + "additionalProperties": { + "properties": { + "account_id": { + "type": "string" + }, + "aws_region": { + "type": "string" + }, + "cloud": { + "type": "string" + }, + "cloud_resource_bucket": { + "properties": { + "gcp": { + "properties": { + "project_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "project_id" + ] + } + }, + "type": "object" + }, + "creation_time": { + "type": "integer" + }, + "credentials_id": { + "type": "string" + }, + "deployment_name": { + "type": "string" + }, + "external_customer_info": { + "properties": { + "authoritative_user_email": { + "type": "string" + }, + "authoritative_user_full_name": { + "type": "string" + }, + "customer_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "authoritative_user_full_name", + "customer_name", + "authoritative_user_email" + ] + }, + "id": { + "type": "string" + }, + "is_no_public_ip_enabled": { + "type": "boolean" + }, + "location": { + "type": "string" + }, + "managed_services_customer_managed_key_id": { + "type": "string" + }, + "network": { + "properties": { + "gcp_common_network_config": { + "properties": { + "gke_cluster_master_ip_range": { + "type": "string" + }, + "gke_connectivity_type": { + "type": "string" + } + }, + "type": "object", + "required": [ + "gke_cluster_master_ip_range", + "gke_connectivity_type" + ] + }, + "gcp_managed_network_config": { + "properties": { + "gke_cluster_pod_ip_range": { + "type": "string" + }, + "gke_cluster_service_ip_range": { + "type": "string" + }, + "subnet_cidr": { + "type": "string" + } + }, + "type": "object", + "required": [ + "gke_cluster_pod_ip_range", + "gke_cluster_service_ip_range", + "subnet_cidr" + ] + }, + "network_id": { + "type": "string" + } + }, + "type": "object" + }, + "network_id": { + "type": "string" + }, + "pricing_tier": { + "type": "string" + }, + "private_access_settings_id": { + "type": "string" + }, + "storage_configuration_id": { + "type": "string" + }, + "storage_customer_managed_key_id": { + "type": "string" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + }, + "read": { + "type": "string" + }, + "update": { + "type": "string" + } + }, + "type": "object" + }, + "token": { + "properties": { + "comment": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_id": { + "type": "string" + }, + "token_value": { + "type": "string" + } + }, + "type": "object" + }, + "workspace_id": { + "type": "integer" + }, + "workspace_name": { + "type": "string" + }, + "workspace_status": { + "type": "string" + }, + "workspace_status_message": { + "type": "string" + }, + "workspace_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "account_id", + "workspace_name" + ] + }, + "type": "object" + }, + "notebook": { + "additionalProperties": { + "properties": { + "content_base64": { + "type": "string" + }, + "format": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + }, + "md5": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "type": "object" + }, + "obo_token": { + "additionalProperties": { + "properties": { + "application_id": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "application_id", + "comment", + "lifetime_seconds" + ] + }, + "type": "object" + }, + "permission_assignment": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permissions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "principal_id": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "permissions", + "principal_id" + ] + }, + "type": "object" + }, + "permissions": { + "additionalProperties": { + "properties": { + "access_control": { + "properties": { + "group_name": { + "type": "string" + }, + "permission_level": { + "type": "string" + }, + "service_principal_name": { + "type": "string" + }, + "user_name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "permission_level" + ] + }, + "authorization": { + "type": "string" + }, + "cluster_id": { + "type": "string" + }, + "cluster_policy_id": { + "type": "string" + }, + "directory_id": { + "type": "string" + }, + "directory_path": { + "type": "string" + }, + "experiment_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instance_pool_id": { + "type": "string" + }, + "job_id": { + "type": "string" + }, + "notebook_id": { + "type": "string" + }, + "notebook_path": { + "type": "string" + }, + "object_type": { + "type": "string" + }, + "pipeline_id": { + "type": "string" + }, + "registered_model_id": { + "type": "string" + }, + "repo_id": { + "type": "string" + }, + "repo_path": { + "type": "string" + }, + "sql_alert_id": { + "type": "string" + }, + "sql_dashboard_id": { + "type": "string" + }, + "sql_endpoint_id": { + "type": "string" + }, + "sql_query_id": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "pipeline": { + "additionalProperties": { + "properties": { + "allow_duplicate_names": { + "type": "boolean" + }, + "channel": { + "type": "string" + }, + "cluster": { + "properties": { + "apply_policy_default_values": { + "type": "boolean" + }, + "autoscale": { + "properties": { + "max_workers": { + "type": "integer" + }, + "min_workers": { + "type": "integer" + } + }, + "type": "object" + }, + "aws_attributes": { + "properties": { + "first_on_demand": { + "type": "integer" + }, + "instance_profile_arn": { + "type": "string" + }, + "zone_id": { + "type": "string" + } + }, + "type": "object" + }, + "cluster_log_conf": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "custom_tags": { + "type": "object" + }, + "driver_instance_pool_id": { + "type": "string" + }, + "driver_node_type_id": { + "type": "string" + }, + "gcp_attributes": { + "properties": { + "google_service_account": { + "type": "string" + } + }, + "type": "object" + }, + "init_scripts": { + "properties": { + "dbfs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + }, + "file": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "gcs": { + "properties": { + "destination": { + "type": "string" + } + }, + "type": "object" + }, + "s3": { + "properties": { + "canned_acl": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "enable_encryption": { + "type": "boolean" + }, + "encryption_type": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "kms_key": { + "type": "string" + }, + "region": { + "type": "string" + } + }, + "type": "object", + "required": [ + "destination" + ] + } + }, + "type": "object" + }, + "instance_pool_id": { + "type": "string" + }, + "label": { + "type": "string" + }, + "node_type_id": { + "type": "string" + }, + "num_workers": { + "type": "integer" + }, + "policy_id": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "spark_env_vars": { + "type": "object" + }, + "ssh_public_keys": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "configuration": { + "type": "object" + }, + "continuous": { + "type": "boolean" + }, + "development": { + "type": "boolean" + }, + "edition": { + "type": "string" + }, + "filters": { + "properties": { + "exclude": { + "items": { + "type": "string" + }, + "type": "array" + }, + "include": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "id": { + "type": "string" + }, + "library": { + "properties": { + "jar": { + "type": "string" + }, + "maven": { + "properties": { + "coordinates": { + "type": "string" + }, + "exclusions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo": { + "type": "string" + } + }, + "type": "object", + "required": [ + "coordinates" + ] + }, + "notebook": { + "properties": { + "path": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "whl": { + "type": "string" + } + }, + "type": "object" + }, + "name": { + "type": "string" + }, + "photon": { + "type": "boolean" + }, + "storage": { + "type": "string" + }, + "target": { + "type": "string" + }, + "timeouts": { + "properties": { + "default": { + "type": "string" + } + }, + "type": "object" + }, + "url": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "recipient": { + "additionalProperties": { + "properties": { + "authentication_type": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "data_recipient_global_metastore_id": { + "type": "string" + }, + "id": { + "type": "string" + }, + "ip_access_list": { + "properties": { + "allowed_ip_addresses": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "allowed_ip_addresses" + ] + }, + "name": { + "type": "string" + }, + "sharing_code": { + "type": "string" + }, + "tokens": { + "properties": { + "activation_url": { + "type": "string" + }, + "created_at": { + "type": "integer" + }, + "created_by": { + "type": "string" + }, + "expiration_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "updated_at": { + "type": "integer" + }, + "updated_by": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "authentication_type", + "name" + ] + }, + "type": "object" + }, + "repo": { + "additionalProperties": { + "properties": { + "branch": { + "type": "string" + }, + "commit_hash": { + "type": "string" + }, + "git_provider": { + "type": "string" + }, + "id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "type": "object" + }, + "schema": { + "additionalProperties": { + "properties": { + "catalog_name": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "force_destroy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "catalog_name" + ] + }, + "type": "object" + }, + "secret": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "key": { + "type": "string" + }, + "last_updated_timestamp": { + "type": "integer" + }, + "scope": { + "type": "string" + }, + "string_value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "string_value", + "key", + "scope" + ] + }, + "type": "object" + }, + "secret_acl": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "permission": { + "type": "string" + }, + "principal": { + "type": "string" + }, + "scope": { + "type": "string" + } + }, + "type": "object", + "required": [ + "permission", + "principal", + "scope" + ] + }, + "type": "object" + }, + "secret_scope": { + "additionalProperties": { + "properties": { + "backend_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "initial_manage_principal": { + "type": "string" + }, + "keyvault_metadata": { + "properties": { + "dns_name": { + "type": "string" + }, + "resource_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dns_name", + "resource_id" + ] + }, + "name": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "service_principal": { + "additionalProperties": { + "properties": { + "active": { + "type": "boolean" + }, + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "application_id": { + "type": "string" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "object" + }, + "service_principal_role": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "role": { + "type": "string" + }, + "service_principal_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role", + "service_principal_id" + ] + }, + "type": "object" + }, + "sql_dashboard": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "sql_endpoint": { + "additionalProperties": { + "properties": { + "auto_stop_mins": { + "type": "integer" + }, + "channel": { + "properties": { + "name": { + "type": "string" + } + }, + "type": "object" + }, + "cluster_size": { + "type": "string" + }, + "data_source_id": { + "type": "string" + }, + "enable_photon": { + "type": "boolean" + }, + "enable_serverless_compute": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "jdbc_url": { + "type": "string" + }, + "max_num_clusters": { + "type": "integer" + }, + "min_num_clusters": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "num_clusters": { + "type": "integer" + }, + "odbc_params": { + "properties": { + "host": { + "type": "string" + }, + "hostname": { + "type": "string" + }, + "path": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string" + } + }, + "type": "object", + "required": [ + "port", + "protocol", + "path" + ] + }, + "spot_instance_policy": { + "type": "string" + }, + "state": { + "type": "string" + }, + "tags": { + "properties": { + "custom_tags": { + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value", + "key" + ] + } + }, + "type": "object" + }, + "timeouts": { + "properties": { + "create": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "name", + "cluster_size" + ] + }, + "type": "object" + }, + "sql_global_config": { + "additionalProperties": { + "properties": { + "data_access_config": { + "type": "object" + }, + "enable_serverless_compute": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "instance_profile_arn": { + "type": "string" + }, + "security_policy": { + "type": "string" + }, + "sql_config_params": { + "type": "object" + } + }, + "type": "object" + }, + "type": "object" + }, + "sql_permissions": { + "additionalProperties": { + "properties": { + "anonymous_function": { + "type": "boolean" + }, + "any_file": { + "type": "boolean" + }, + "catalog": { + "type": "boolean" + }, + "cluster_id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "id": { + "type": "string" + }, + "privilege_assignments": { + "properties": { + "principal": { + "type": "string" + }, + "privileges": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "principal", + "privileges" + ] + }, + "table": { + "type": "string" + }, + "view": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "sql_query": { + "additionalProperties": { + "properties": { + "data_source_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "parameter": { + "properties": { + "date": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "date_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetime": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetime_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetimesec": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "datetimesec_range": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "enum": { + "properties": { + "multiple": { + "properties": { + "prefix": { + "type": "string" + }, + "separator": { + "type": "string" + }, + "suffix": { + "type": "string" + } + }, + "type": "object", + "required": [ + "prefix", + "separator", + "suffix" + ] + }, + "options": { + "items": { + "type": "string" + }, + "type": "array" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "options" + ] + }, + "name": { + "type": "string" + }, + "number": { + "properties": { + "value": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "query": { + "properties": { + "multiple": { + "properties": { + "prefix": { + "type": "string" + }, + "separator": { + "type": "string" + }, + "suffix": { + "type": "string" + } + }, + "type": "object", + "required": [ + "prefix", + "separator", + "suffix" + ] + }, + "query_id": { + "type": "string" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "query_id" + ] + }, + "text": { + "properties": { + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "value" + ] + }, + "title": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "query": { + "type": "string" + }, + "run_as_role": { + "type": "string" + }, + "schedule": { + "properties": { + "continuous": { + "properties": { + "interval_seconds": { + "type": "integer" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "interval_seconds" + ] + }, + "daily": { + "properties": { + "interval_days": { + "type": "integer" + }, + "time_of_day": { + "type": "string" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "interval_days", + "time_of_day" + ] + }, + "weekly": { + "properties": { + "day_of_week": { + "type": "string" + }, + "interval_weeks": { + "type": "integer" + }, + "time_of_day": { + "type": "string" + }, + "until_date": { + "type": "string" + } + }, + "type": "object", + "required": [ + "day_of_week", + "interval_weeks", + "time_of_day" + ] + } + }, + "type": "object" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "query", + "data_source_id" + ] + }, + "type": "object" + }, + "sql_visualization": { + "additionalProperties": { + "properties": { + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "options": { + "type": "string" + }, + "query_id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "visualization_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "options", + "query_id", + "type" + ] + }, + "type": "object" + }, + "sql_widget": { + "additionalProperties": { + "properties": { + "dashboard_id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "parameter": { + "properties": { + "map_to": { + "type": "string" + }, + "name": { + "type": "string" + }, + "title": { + "type": "string" + }, + "type": { + "type": "string" + }, + "value": { + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "type" + ] + }, + "position": { + "properties": { + "auto_height": { + "type": "boolean" + }, + "pos_x": { + "type": "integer" + }, + "pos_y": { + "type": "integer" + }, + "size_x": { + "type": "integer" + }, + "size_y": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "size_x", + "size_y" + ] + }, + "text": { + "type": "string" + }, + "title": { + "type": "string" + }, + "visualization_id": { + "type": "string" + }, + "widget_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "dashboard_id" + ] + }, + "type": "object" + }, + "storage_credential": { + "additionalProperties": { + "properties": { + "aws_iam_role": { + "properties": { + "role_arn": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role_arn" + ] + }, + "azure_managed_identity": { + "properties": { + "access_connector_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "access_connector_id" + ] + }, + "azure_service_principal": { + "properties": { + "application_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "directory_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "client_secret", + "directory_id", + "application_id" + ] + }, + "comment": { + "type": "string" + }, + "id": { + "type": "string" + }, + "metastore_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "object" + }, + "table": { + "additionalProperties": { + "properties": { + "catalog_name": { + "type": "string" + }, + "column": { + "properties": { + "comment": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nullable": { + "type": "boolean" + }, + "partition_index": { + "type": "integer" + }, + "position": { + "type": "integer" + }, + "type_interval_type": { + "type": "string" + }, + "type_json": { + "type": "string" + }, + "type_name": { + "type": "string" + }, + "type_precision": { + "type": "integer" + }, + "type_scale": { + "type": "integer" + }, + "type_text": { + "type": "string" + } + }, + "type": "object", + "required": [ + "position", + "type_text", + "name", + "type_name" + ] + }, + "comment": { + "type": "string" + }, + "data_source_format": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "properties": { + "type": "object" + }, + "schema_name": { + "type": "string" + }, + "storage_credential_name": { + "type": "string" + }, + "storage_location": { + "type": "string" + }, + "table_type": { + "type": "string" + }, + "view_definition": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "schema_name", + "data_source_format", + "table_type", + "catalog_name" + ] + }, + "type": "object" + }, + "token": { + "additionalProperties": { + "properties": { + "comment": { + "type": "string" + }, + "creation_time": { + "type": "integer" + }, + "expiry_time": { + "type": "integer" + }, + "id": { + "type": "string" + }, + "lifetime_seconds": { + "type": "integer" + }, + "token_id": { + "type": "string" + }, + "token_value": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "user": { + "additionalProperties": { + "properties": { + "active": { + "type": "boolean" + }, + "allow_cluster_create": { + "type": "boolean" + }, + "allow_instance_pool_create": { + "type": "boolean" + }, + "databricks_sql_access": { + "type": "boolean" + }, + "display_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "force": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "workspace_access": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "user_name" + ] + }, + "type": "object" + }, + "user_instance_profile": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "instance_profile_id": { + "type": "string" + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "instance_profile_id", + "user_id" + ] + }, + "type": "object" + }, + "user_role": { + "additionalProperties": { + "properties": { + "id": { + "type": "string" + }, + "role": { + "type": "string" + }, + "user_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "role", + "user_id" + ] + }, + "type": "object" + }, + "workspace_conf": { + "additionalProperties": { + "properties": { + "custom_config": { + "type": "object" + }, + "id": { + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + } + }, + "type": "object", + "title": "resources" + } + }, + "type": "object", + "title": "resource file" +} diff --git a/example-project/.gitignore b/example-project/.gitignore new file mode 100644 index 00000000..2f24ad7c --- /dev/null +++ b/example-project/.gitignore @@ -0,0 +1,2 @@ +/dist +repo_snapshot.json diff --git a/example-project/databricks.yml b/example-project/databricks.yml new file mode 100644 index 00000000..1a814fba --- /dev/null +++ b/example-project/databricks.yml @@ -0,0 +1,17 @@ +name: example-project + +environments: + development: + workspace: + profile: e2-dogfood + root: /Repos/pieter.noordhuis@databricks.com/bricks-demos/development + + staging: + workspace: + profile: e2-dogfood + root: /Repos/pieter.noordhuis@databricks.com/bricks-demos/staging + + production: + workspace: + profile: e2-dogfood + root: /Repos/pieter.noordhuis@databricks.com/bricks-demos/production diff --git a/example-project/my_first_job.py b/example-project/my_first_job.py new file mode 100644 index 00000000..9e934f1a --- /dev/null +++ b/example-project/my_first_job.py @@ -0,0 +1,7 @@ +import sys + +print("hello world!") +print(sys.argv) + +for x in spark.range(10).collect(): + print(x) diff --git a/example-project/resources.yml b/example-project/resources.yml new file mode 100644 index 00000000..fb59ff16 --- /dev/null +++ b/example-project/resources.yml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=./.bricks/schema.json + +resources: + job: + my_first_job: + spark_python_task: + python_file: "./my_first_job.py" + parameters: + - "hello" + - "world" + tags: + key: value + + +environments: + development: + resources: + job: + my_first_job: + name: "[development] my first job" + + # Use autoscaling on e2-dogfood. + existing_cluster_id: "0923-164208-meows279" + + staging: + resources: + job: + my_first_job: + name: "[staging] my first job" + new_cluster: + spark_version: 10.4.x-scala2.12 + node_type_id: "i3.xlarge" + num_workers: 2 + + production: + resources: + job: + my_first_job: + name: "[production] my first job" + new_cluster: + spark_version: 10.4.x-scala2.12 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 4 + max_workers: 8 diff --git a/project/environment.go b/project/environment.go index 91ab2589..efbdbcb2 100644 --- a/project/environment.go +++ b/project/environment.go @@ -13,6 +13,7 @@ const defaultEnvironment = "development" // Workspace defines configurables at the workspace level. type Workspace struct { Profile string `json:"profile,omitempty"` + Root string `json:"root,omitempty"` } // Environment defines all configurables for a single environment.