From 84b47745e451f6552465243665ad6c897c55ae5e Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Fri, 23 Aug 2024 12:13:21 +0200 Subject: [PATCH 01/78] Ignore CLI version check on development builds of the CLI (#1714) ## Changes This changes makes sure we ignore CLI version check on development builds of the CLI. Before: ``` $ cat databricks.yml | grep cli_version databricks_cli_version: ">= 0.223.1" $ cli bundle deploy Error: Databricks CLI version constraint not satisfied. Required: >= 0.223.1, current: 0.0.0-dev+06b169284737 ``` after ``` ... $ cli bundle deploy ... Warning: Ignoring Databricks CLI version constraint for development build. Required: >= 0.223.1, current: 0.0.0-dev+d52d6f08fcd5 ``` ## Tests --- bundle/config/mutator/verify_cli_version.go | 4 ++++ bundle/config/mutator/verify_cli_version_test.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/verify_cli_version.go b/bundle/config/mutator/verify_cli_version.go index 9c32fcc9..279af44e 100644 --- a/bundle/config/mutator/verify_cli_version.go +++ b/bundle/config/mutator/verify_cli_version.go @@ -40,6 +40,10 @@ func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia } if !c.Check(version) { + if version.Prerelease() == "dev" && version.Major() == 0 { + return diag.Warningf("Ignoring Databricks CLI version constraint for development build. Required: %s, current: %s", constraint, currentVersion) + } + return diag.Errorf("Databricks CLI version constraint not satisfied. Required: %s, current: %s", constraint, currentVersion) } diff --git a/bundle/config/mutator/verify_cli_version_test.go b/bundle/config/mutator/verify_cli_version_test.go index 24f65674..02546129 100644 --- a/bundle/config/mutator/verify_cli_version_test.go +++ b/bundle/config/mutator/verify_cli_version_test.go @@ -107,6 +107,11 @@ func TestVerifyCliVersion(t *testing.T) { constraint: "^0.100", expectedError: "invalid version constraint \"^0.100\" specified. Please specify the version constraint in the format (>=) 0.0.0(, <= 1.0.0)", }, + { + currentVersion: "0.0.0-dev+06b169284737", + constraint: ">= 0.100.0", + expectedError: "Ignoring Databricks CLI version constraint for development build. Required: >= 0.100.0", + }, } t.Cleanup(func() { @@ -130,7 +135,7 @@ func TestVerifyCliVersion(t *testing.T) { diags := bundle.Apply(context.Background(), b, VerifyCliVersion()) if tc.expectedError != "" { require.NotEmpty(t, diags) - require.Equal(t, tc.expectedError, diags.Error().Error()) + require.Contains(t, diags[0].Summary, tc.expectedError) } else { require.Empty(t, diags) } From 783e05c939a694fe722e52ddea9c48f0ea077181 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 26 Aug 2024 12:03:56 +0200 Subject: [PATCH 02/78] Do not treat empty path as a local path (#1717) ## Changes Fixes issue introduced here https://github.com/databricks/cli/pull/1699 where PyPi packages were treated as local library. The reason is that `libraryPath` returns an empty string as a path for PyPi packages and then `IsLibraryLocal` treated empty string as local path. Both of these functions are fixed in this PR. ## Tests Added regression test --- bundle/libraries/helpers.go | 19 +++++++---- bundle/libraries/helpers_test.go | 28 +++++++++++++--- bundle/libraries/libraries.go | 7 +++- bundle/libraries/local_path.go | 4 +++ bundle/libraries/local_path_test.go | 1 + bundle/libraries/workspace_path.go | 4 +-- bundle/python/warning_test.go | 51 +++++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 15 deletions(-) diff --git a/bundle/libraries/helpers.go b/bundle/libraries/helpers.go index b7e707cc..2149e588 100644 --- a/bundle/libraries/helpers.go +++ b/bundle/libraries/helpers.go @@ -1,19 +1,24 @@ package libraries -import "github.com/databricks/databricks-sdk-go/service/compute" +import ( + "fmt" -func libraryPath(library *compute.Library) string { + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func libraryPath(library *compute.Library) (string, error) { if library.Whl != "" { - return library.Whl + return library.Whl, nil } if library.Jar != "" { - return library.Jar + return library.Jar, nil } if library.Egg != "" { - return library.Egg + return library.Egg, nil } if library.Requirements != "" { - return library.Requirements + return library.Requirements, nil } - return "" + + return "", fmt.Errorf("not supported library type") } diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go index e4bd3277..9d7e12ee 100644 --- a/bundle/libraries/helpers_test.go +++ b/bundle/libraries/helpers_test.go @@ -10,9 +10,27 @@ import ( func TestLibraryPath(t *testing.T) { path := "/some/path" - assert.Equal(t, path, libraryPath(&compute.Library{Whl: path})) - assert.Equal(t, path, libraryPath(&compute.Library{Jar: path})) - assert.Equal(t, path, libraryPath(&compute.Library{Egg: path})) - assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path})) - assert.Equal(t, "", libraryPath(&compute.Library{})) + p, err := libraryPath(&compute.Library{Whl: path}) + assert.Equal(t, path, p) + assert.Nil(t, err) + + p, err = libraryPath(&compute.Library{Jar: path}) + assert.Equal(t, path, p) + assert.Nil(t, err) + + p, err = libraryPath(&compute.Library{Egg: path}) + assert.Equal(t, path, p) + assert.Nil(t, err) + + p, err = libraryPath(&compute.Library{Requirements: path}) + assert.Equal(t, path, p) + assert.Nil(t, err) + + p, err = libraryPath(&compute.Library{}) + assert.Equal(t, "", p) + assert.NotNil(t, err) + + p, err = libraryPath(&compute.Library{Pypi: &compute.PythonPyPiLibrary{Package: "pypipackage"}}) + assert.Equal(t, "", p) + assert.NotNil(t, err) } diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 33b848dd..f75e23a8 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -67,7 +67,12 @@ func FindTasksWithLocalLibraries(b *bundle.Bundle) []jobs.Task { func isTaskWithLocalLibraries(task jobs.Task) bool { for _, l := range task.Libraries { - if IsLibraryLocal(libraryPath(&l)) { + p, err := libraryPath(&l) + // If there's an error, skip the library because it's not of supported type + if err != nil { + continue + } + if IsLibraryLocal(p) { return true } } diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 417bce10..e4956240 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -43,6 +43,10 @@ func IsLocalPath(p string) bool { // We can't use IsLocalPath beacuse environment dependencies can be // a pypi package name which can be misinterpreted as a local path by IsLocalPath. func IsLibraryLocal(dep string) bool { + if dep == "" { + return false + } + possiblePrefixes := []string{ ".", } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index 7f84b324..667d64ec 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -48,6 +48,7 @@ func TestIsLibraryLocal(t *testing.T) { {path: "../../local/*.whl", expected: true}, {path: "..\\..\\local\\*.whl", expected: true}, {path: "file://path/to/package/whl.whl", expected: true}, + {path: "", expected: false}, {path: "pypipackage", expected: false}, {path: "/Volumes/catalog/schema/volume/path.whl", expected: false}, {path: "/Workspace/my_project/dist.whl", expected: false}, diff --git a/bundle/libraries/workspace_path.go b/bundle/libraries/workspace_path.go index b08ca161..126ad3f1 100644 --- a/bundle/libraries/workspace_path.go +++ b/bundle/libraries/workspace_path.go @@ -29,8 +29,8 @@ func IsWorkspacePath(path string) bool { // IsWorkspaceLibrary returns true if the specified library refers to a workspace path. func IsWorkspaceLibrary(library *compute.Library) bool { - path := libraryPath(library) - if path == "" { + path, err := libraryPath(library) + if err != nil { return false } diff --git a/bundle/python/warning_test.go b/bundle/python/warning_test.go index dd6397f7..b2296392 100644 --- a/bundle/python/warning_test.go +++ b/bundle/python/warning_test.go @@ -223,6 +223,17 @@ func TestNoIncompatibleWheelTasks(t *testing.T) { {Whl: "./dist/test.whl"}, }, }, + { + TaskKey: "key7", + PythonWheelTask: &jobs.PythonWheelTask{}, + ExistingClusterId: "test-key-2", + Libraries: []compute.Library{ + {Whl: "signol_lib-0.4.4-20240822+prod-py3-none-any.whl"}, + {Pypi: &compute.PythonPyPiLibrary{ + Package: "requests==2.25.1", + }}, + }, + }, }, }, }, @@ -241,6 +252,46 @@ func TestNoIncompatibleWheelTasks(t *testing.T) { require.False(t, hasIncompatibleWheelTasks(context.Background(), b)) } +func TestTasksWithPyPiPackageAreCompatible(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + JobSettings: &jobs.JobSettings{ + JobClusters: []jobs.JobCluster{ + { + JobClusterKey: "cluster1", + NewCluster: compute.ClusterSpec{ + SparkVersion: "12.2.x-scala2.12", + }, + }, + }, + Tasks: []jobs.Task{ + { + TaskKey: "key1", + PythonWheelTask: &jobs.PythonWheelTask{}, + ExistingClusterId: "test-key-2", + Libraries: []compute.Library{ + {Pypi: &compute.PythonPyPiLibrary{ + Package: "requests==2.25.1", + }}, + }, + }, + }, + }, + }, + }, + }, + }, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + require.False(t, hasIncompatibleWheelTasks(context.Background(), b)) +} + func TestNoWarningWhenPythonWheelWrapperIsOn(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ From 056d2032368ead1e3a7e65f9304508498bc53403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:54:05 +0200 Subject: [PATCH 03/78] Bump github.com/databricks/databricks-sdk-go from 0.44.0 to 0.45.0 (#1719) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.44.0 to 0.45.0.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.45.0

0.45.0

Bug Fixes

  • Add INVALID_STATE to error code mapping (#1014).
  • Do not specify --tenant flag when fetching managed identity access token from the CLI (#1021).

Internal Changes

  • Add terraform aliases to Entity (#1017).
  • Added Service.NamedIdMap (#1016).
  • Fix billing test for budget configuration update (#1019).

API Changes:

OpenAPI SHA: 3eae49b444cac5a0118a3503e5b7ecef7f96527a, Date: 2024-08-21

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.45.0

Bug Fixes

  • Add INVALID_STATE to error code mapping (#1014).
  • Do not specify --tenant flag when fetching managed identity access token from the CLI (#1021).

Internal Changes

  • Add terraform aliases to Entity (#1017).
  • Added Service.NamedIdMap (#1016).
  • Fix billing test for budget configuration update (#1019).

API Changes:

OpenAPI SHA: 3eae49b444cac5a0118a3503e5b7ecef7f96527a, Date: 2024-08-21

Commits
  • 6d86788 [Release] Release v0.45.0 (#1023)
  • ba4489b [Fix] Do not specify --tenant flag when fetching managed identity access to...
  • f624809 [Internal] Fix billing test for budget configuration update (#1019)
  • 27a5055 [Internal] Add terraform aliases to Entity (#1017)
  • 382a38d [Internal] Added Service.NamedIdMap (#1016)
  • 1ef9931 [Fix] Add INVALID_STATE to error code mapping (#1014)
  • See full diff in compare view

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.44.0&new-version=0.45.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- .gitattributes | 3 + bundle/schema/docs/bundle_descriptions.json | 64 +++++ cmd/workspace/cmd.go | 6 + .../external-locations/external-locations.go | 2 + .../policy-compliance-for-clusters.go | 260 +++++++++++++++++ .../policy-compliance-for-jobs.go | 262 ++++++++++++++++++ cmd/workspace/query-history/query-history.go | 8 +- .../resource-quotas/resource-quotas.go | 168 +++++++++++ go.mod | 2 +- go.sum | 4 +- 11 files changed, 773 insertions(+), 8 deletions(-) create mode 100755 cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go create mode 100755 cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go create mode 100755 cmd/workspace/resource-quotas/resource-quotas.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index fef6f268..8b01a242 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -f98c07f9c71f579de65d2587bb0292f83d10e55d \ No newline at end of file +3eae49b444cac5a0118a3503e5b7ecef7f96527a \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index bdb3f398..d82ab769 100755 --- a/.gitattributes +++ b/.gitattributes @@ -75,6 +75,8 @@ cmd/workspace/online-tables/online-tables.go linguist-generated=true cmd/workspace/permission-migration/permission-migration.go linguist-generated=true cmd/workspace/permissions/permissions.go linguist-generated=true cmd/workspace/pipelines/pipelines.go linguist-generated=true +cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go linguist-generated=true +cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go linguist-generated=true cmd/workspace/policy-families/policy-families.go linguist-generated=true cmd/workspace/provider-exchange-filters/provider-exchange-filters.go linguist-generated=true cmd/workspace/provider-exchanges/provider-exchanges.go linguist-generated=true @@ -94,6 +96,7 @@ cmd/workspace/recipient-activation/recipient-activation.go linguist-generated=tr cmd/workspace/recipients/recipients.go linguist-generated=true cmd/workspace/registered-models/registered-models.go linguist-generated=true cmd/workspace/repos/repos.go linguist-generated=true +cmd/workspace/resource-quotas/resource-quotas.go linguist-generated=true cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go linguist-generated=true cmd/workspace/schemas/schemas.go linguist-generated=true cmd/workspace/secrets/secrets.go linguist-generated=true diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json index d888b366..908a1c2b 100644 --- a/bundle/schema/docs/bundle_descriptions.json +++ b/bundle/schema/docs/bundle_descriptions.json @@ -85,6 +85,12 @@ "enabled": { "description": "" }, + "import": { + "description": "", + "items": { + "description": "" + } + }, "venv_path": { "description": "" } @@ -130,6 +136,29 @@ } } }, + "presets": { + "description": "", + "properties": { + "jobs_max_concurrent_runs": { + "description": "" + }, + "name_prefix": { + "description": "" + }, + "pipelines_development": { + "description": "" + }, + "tags": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "trigger_pause_status": { + "description": "" + } + } + }, "resources": { "description": "Collection of Databricks resources to deploy.", "properties": { @@ -3079,6 +3108,12 @@ "items": { "description": "" } + }, + "paths": { + "description": "", + "items": { + "description": "" + } } } }, @@ -3202,6 +3237,29 @@ } } }, + "presets": { + "description": "", + "properties": { + "jobs_max_concurrent_runs": { + "description": "" + }, + "name_prefix": { + "description": "" + }, + "pipelines_development": { + "description": "" + }, + "tags": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "trigger_pause_status": { + "description": "" + } + } + }, "resources": { "description": "Collection of Databricks resources to deploy.", "properties": { @@ -6151,6 +6209,12 @@ "items": { "description": "" } + }, + "paths": { + "description": "", + "items": { + "description": "" + } } } }, diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 75664c79..11be8077 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -44,6 +44,8 @@ import ( permission_migration "github.com/databricks/cli/cmd/workspace/permission-migration" permissions "github.com/databricks/cli/cmd/workspace/permissions" pipelines "github.com/databricks/cli/cmd/workspace/pipelines" + policy_compliance_for_clusters "github.com/databricks/cli/cmd/workspace/policy-compliance-for-clusters" + policy_compliance_for_jobs "github.com/databricks/cli/cmd/workspace/policy-compliance-for-jobs" policy_families "github.com/databricks/cli/cmd/workspace/policy-families" provider_exchange_filters "github.com/databricks/cli/cmd/workspace/provider-exchange-filters" provider_exchanges "github.com/databricks/cli/cmd/workspace/provider-exchanges" @@ -63,6 +65,7 @@ import ( recipients "github.com/databricks/cli/cmd/workspace/recipients" registered_models "github.com/databricks/cli/cmd/workspace/registered-models" repos "github.com/databricks/cli/cmd/workspace/repos" + resource_quotas "github.com/databricks/cli/cmd/workspace/resource-quotas" schemas "github.com/databricks/cli/cmd/workspace/schemas" secrets "github.com/databricks/cli/cmd/workspace/secrets" service_principals "github.com/databricks/cli/cmd/workspace/service-principals" @@ -130,6 +133,8 @@ func All() []*cobra.Command { out = append(out, permission_migration.New()) out = append(out, permissions.New()) out = append(out, pipelines.New()) + out = append(out, policy_compliance_for_clusters.New()) + out = append(out, policy_compliance_for_jobs.New()) out = append(out, policy_families.New()) out = append(out, provider_exchange_filters.New()) out = append(out, provider_exchanges.New()) @@ -149,6 +154,7 @@ func All() []*cobra.Command { out = append(out, recipients.New()) out = append(out, registered_models.New()) out = append(out, repos.New()) + out = append(out, resource_quotas.New()) out = append(out, schemas.New()) out = append(out, secrets.New()) out = append(out, service_principals.New()) diff --git a/cmd/workspace/external-locations/external-locations.go b/cmd/workspace/external-locations/external-locations.go index 8f0dd346..42493fc4 100755 --- a/cmd/workspace/external-locations/external-locations.go +++ b/cmd/workspace/external-locations/external-locations.go @@ -75,6 +75,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.AccessPoint, "access-point", createReq.AccessPoint, `The AWS access point to use when accesing s3 for this external location.`) cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `User-provided free-form text description.`) // TODO: complex arg: encryption_details + cmd.Flags().BoolVar(&createReq.Fallback, "fallback", createReq.Fallback, `Indicates whether fallback mode is enabled for this external location.`) cmd.Flags().BoolVar(&createReq.ReadOnly, "read-only", createReq.ReadOnly, `Indicates whether the external location is read-only.`) cmd.Flags().BoolVar(&createReq.SkipValidation, "skip-validation", createReq.SkipValidation, `Skips validation of the storage credential associated with the external location.`) @@ -347,6 +348,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `User-provided free-form text description.`) cmd.Flags().StringVar(&updateReq.CredentialName, "credential-name", updateReq.CredentialName, `Name of the storage credential used with this location.`) // TODO: complex arg: encryption_details + cmd.Flags().BoolVar(&updateReq.Fallback, "fallback", updateReq.Fallback, `Indicates whether fallback mode is enabled for this external location.`) cmd.Flags().BoolVar(&updateReq.Force, "force", updateReq.Force, `Force update even if changing url invalidates dependent external tables or mounts.`) cmd.Flags().Var(&updateReq.IsolationMode, "isolation-mode", `Whether the current securable is accessible from all workspaces or a specific set of workspaces. Supported values: [ISOLATION_MODE_ISOLATED, ISOLATION_MODE_OPEN]`) cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the external location.`) diff --git a/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go new file mode 100755 index 00000000..1274c879 --- /dev/null +++ b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go @@ -0,0 +1,260 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package policy_compliance_for_clusters + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "policy-compliance-for-clusters", + Short: `The policy compliance APIs allow you to view and manage the policy compliance status of clusters in your workspace.`, + Long: `The policy compliance APIs allow you to view and manage the policy compliance + status of clusters in your workspace. + + A cluster is compliant with its policy if its configuration satisfies all its + policy rules. Clusters could be out of compliance if their policy was updated + after the cluster was last edited. + + The get and list compliance APIs allow you to view the policy compliance + status of a cluster. The enforce compliance API allows you to update a cluster + to be compliant with the current version of its policy.`, + GroupID: "compute", + Annotations: map[string]string{ + "package": "compute", + }, + } + + // Add methods + cmd.AddCommand(newEnforceCompliance()) + cmd.AddCommand(newGetCompliance()) + cmd.AddCommand(newListCompliance()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start enforce-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var enforceComplianceOverrides []func( + *cobra.Command, + *compute.EnforceClusterComplianceRequest, +) + +func newEnforceCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var enforceComplianceReq compute.EnforceClusterComplianceRequest + var enforceComplianceJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&enforceComplianceJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().BoolVar(&enforceComplianceReq.ValidateOnly, "validate-only", enforceComplianceReq.ValidateOnly, `If set, previews the changes that would be made to a cluster to enforce compliance but does not update the cluster.`) + + cmd.Use = "enforce-compliance CLUSTER_ID" + cmd.Short = `Enforce cluster policy compliance.` + cmd.Long = `Enforce cluster policy compliance. + + Updates a cluster to be compliant with the current version of its policy. A + cluster can be updated if it is in a RUNNING or TERMINATED state. + + If a cluster is updated while in a RUNNING state, it will be restarted so + that the new attributes can take effect. + + If a cluster is updated while in a TERMINATED state, it will remain + TERMINATED. The next time the cluster is started, the new attributes will + take effect. + + Clusters created by the Databricks Jobs, DLT, or Models services cannot be + enforced by this API. Instead, use the "Enforce job policy compliance" API to + enforce policy compliance on jobs. + + Arguments: + CLUSTER_ID: The ID of the cluster you want to enforce policy compliance on.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'cluster_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if err != nil { + return err + } + } + if !cmd.Flags().Changed("json") { + enforceComplianceReq.ClusterId = args[0] + } + + response, err := w.PolicyComplianceForClusters.EnforceCompliance(ctx, enforceComplianceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range enforceComplianceOverrides { + fn(cmd, &enforceComplianceReq) + } + + return cmd +} + +// start get-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getComplianceOverrides []func( + *cobra.Command, + *compute.GetClusterComplianceRequest, +) + +func newGetCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var getComplianceReq compute.GetClusterComplianceRequest + + // TODO: short flags + + cmd.Use = "get-compliance CLUSTER_ID" + cmd.Short = `Get cluster policy compliance.` + cmd.Long = `Get cluster policy compliance. + + Returns the policy compliance status of a cluster. Clusters could be out of + compliance if their policy was updated after the cluster was last edited. + + Arguments: + CLUSTER_ID: The ID of the cluster to get the compliance status` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getComplianceReq.ClusterId = args[0] + + response, err := w.PolicyComplianceForClusters.GetCompliance(ctx, getComplianceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getComplianceOverrides { + fn(cmd, &getComplianceReq) + } + + return cmd +} + +// start list-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listComplianceOverrides []func( + *cobra.Command, + *compute.ListClusterCompliancesRequest, +) + +func newListCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var listComplianceReq compute.ListClusterCompliancesRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listComplianceReq.PageSize, "page-size", listComplianceReq.PageSize, `Use this field to specify the maximum number of results to be returned by the server.`) + cmd.Flags().StringVar(&listComplianceReq.PageToken, "page-token", listComplianceReq.PageToken, `A page token that can be used to navigate to the next page or previous page as returned by next_page_token or prev_page_token.`) + + cmd.Use = "list-compliance POLICY_ID" + cmd.Short = `List cluster policy compliance.` + cmd.Long = `List cluster policy compliance. + + Returns the policy compliance status of all clusters that use a given policy. + Clusters could be out of compliance if their policy was updated after the + cluster was last edited. + + Arguments: + POLICY_ID: Canonical unique identifier for the cluster policy.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + listComplianceReq.PolicyId = args[0] + + response := w.PolicyComplianceForClusters.ListCompliance(ctx, listComplianceReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listComplianceOverrides { + fn(cmd, &listComplianceReq) + } + + return cmd +} + +// end service PolicyComplianceForClusters diff --git a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go new file mode 100755 index 00000000..d74caa57 --- /dev/null +++ b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go @@ -0,0 +1,262 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package policy_compliance_for_jobs + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "policy-compliance-for-jobs", + Short: `The compliance APIs allow you to view and manage the policy compliance status of jobs in your workspace.`, + Long: `The compliance APIs allow you to view and manage the policy compliance status + of jobs in your workspace. This API currently only supports compliance + controls for cluster policies. + + A job is in compliance if its cluster configurations satisfy the rules of all + their respective cluster policies. A job could be out of compliance if a + cluster policy it uses was updated after the job was last edited. The job is + considered out of compliance if any of its clusters no longer comply with + their updated policies. + + The get and list compliance APIs allow you to view the policy compliance + status of a job. The enforce compliance API allows you to update a job so that + it becomes compliant with all of its policies.`, + GroupID: "jobs", + Annotations: map[string]string{ + "package": "jobs", + }, + } + + // Add methods + cmd.AddCommand(newEnforceCompliance()) + cmd.AddCommand(newGetCompliance()) + cmd.AddCommand(newListCompliance()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start enforce-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var enforceComplianceOverrides []func( + *cobra.Command, + *jobs.EnforcePolicyComplianceRequest, +) + +func newEnforceCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var enforceComplianceReq jobs.EnforcePolicyComplianceRequest + var enforceComplianceJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&enforceComplianceJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().BoolVar(&enforceComplianceReq.ValidateOnly, "validate-only", enforceComplianceReq.ValidateOnly, `If set, previews changes made to the job to comply with its policy, but does not update the job.`) + + cmd.Use = "enforce-compliance JOB_ID" + cmd.Short = `Enforce job policy compliance.` + cmd.Long = `Enforce job policy compliance. + + Updates a job so the job clusters that are created when running the job + (specified in new_cluster) are compliant with the current versions of their + respective cluster policies. All-purpose clusters used in the job will not be + updated. + + Arguments: + JOB_ID: The ID of the job you want to enforce policy compliance on.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'job_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if err != nil { + return err + } + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[0], &enforceComplianceReq.JobId) + if err != nil { + return fmt.Errorf("invalid JOB_ID: %s", args[0]) + } + } + + response, err := w.PolicyComplianceForJobs.EnforceCompliance(ctx, enforceComplianceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range enforceComplianceOverrides { + fn(cmd, &enforceComplianceReq) + } + + return cmd +} + +// start get-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getComplianceOverrides []func( + *cobra.Command, + *jobs.GetPolicyComplianceRequest, +) + +func newGetCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var getComplianceReq jobs.GetPolicyComplianceRequest + + // TODO: short flags + + cmd.Use = "get-compliance JOB_ID" + cmd.Short = `Get job policy compliance.` + cmd.Long = `Get job policy compliance. + + Returns the policy compliance status of a job. Jobs could be out of compliance + if a cluster policy they use was updated after the job was last edited and + some of its job clusters no longer comply with their updated policies. + + Arguments: + JOB_ID: The ID of the job whose compliance status you are requesting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + _, err = fmt.Sscan(args[0], &getComplianceReq.JobId) + if err != nil { + return fmt.Errorf("invalid JOB_ID: %s", args[0]) + } + + response, err := w.PolicyComplianceForJobs.GetCompliance(ctx, getComplianceReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getComplianceOverrides { + fn(cmd, &getComplianceReq) + } + + return cmd +} + +// start list-compliance command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listComplianceOverrides []func( + *cobra.Command, + *jobs.ListJobComplianceRequest, +) + +func newListCompliance() *cobra.Command { + cmd := &cobra.Command{} + + var listComplianceReq jobs.ListJobComplianceRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listComplianceReq.PageSize, "page-size", listComplianceReq.PageSize, `Use this field to specify the maximum number of results to be returned by the server.`) + cmd.Flags().StringVar(&listComplianceReq.PageToken, "page-token", listComplianceReq.PageToken, `A page token that can be used to navigate to the next page or previous page as returned by next_page_token or prev_page_token.`) + + cmd.Use = "list-compliance POLICY_ID" + cmd.Short = `List job policy compliance.` + cmd.Long = `List job policy compliance. + + Returns the policy compliance status of all jobs that use a given policy. Jobs + could be out of compliance if a cluster policy they use was updated after the + job was last edited and its job clusters no longer comply with the updated + policy. + + Arguments: + POLICY_ID: Canonical unique identifier for the cluster policy.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + listComplianceReq.PolicyId = args[0] + + response := w.PolicyComplianceForJobs.ListCompliance(ctx, listComplianceReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listComplianceOverrides { + fn(cmd, &listComplianceReq) + } + + return cmd +} + +// end service PolicyComplianceForJobs diff --git a/cmd/workspace/query-history/query-history.go b/cmd/workspace/query-history/query-history.go index 5155b5cc..bfa013f2 100755 --- a/cmd/workspace/query-history/query-history.go +++ b/cmd/workspace/query-history/query-history.go @@ -16,9 +16,9 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "query-history", - Short: `A service responsible for storing and retrieving the list of queries run against SQL endpoints, serverless compute, and DLT.`, + Short: `A service responsible for storing and retrieving the list of queries run against SQL endpoints and serverless compute.`, Long: `A service responsible for storing and retrieving the list of queries run - against SQL endpoints, serverless compute, and DLT.`, + against SQL endpoints and serverless compute.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -53,6 +53,7 @@ func newList() *cobra.Command { // TODO: short flags // TODO: complex arg: filter_by + cmd.Flags().BoolVar(&listReq.IncludeMetrics, "include-metrics", listReq.IncludeMetrics, `Whether to include the query metrics with each query.`) cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Limit the number of results returned in one page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) @@ -60,8 +61,7 @@ func newList() *cobra.Command { cmd.Short = `List Queries.` cmd.Long = `List Queries. - List the history of queries through SQL warehouses, serverless compute, and - DLT. + List the history of queries through SQL warehouses, and serverless compute. You can filter by user ID, warehouse ID, status, and time range. Most recently started queries are returned first (up to max_results in request). The diff --git a/cmd/workspace/resource-quotas/resource-quotas.go b/cmd/workspace/resource-quotas/resource-quotas.go new file mode 100755 index 00000000..9a0c3068 --- /dev/null +++ b/cmd/workspace/resource-quotas/resource-quotas.go @@ -0,0 +1,168 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package resource_quotas + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "resource-quotas", + Short: `Unity Catalog enforces resource quotas on all securable objects, which limits the number of resources that can be created.`, + Long: `Unity Catalog enforces resource quotas on all securable objects, which limits + the number of resources that can be created. Quotas are expressed in terms of + a resource type and a parent (for example, tables per metastore or schemas per + catalog). The resource quota APIs enable you to monitor your current usage and + limits. For more information on resource quotas see the [Unity Catalog + documentation]. + + [Unity Catalog documentation]: https://docs.databricks.com/en/data-governance/unity-catalog/index.html#resource-quotas`, + GroupID: "catalog", + Annotations: map[string]string{ + "package": "catalog", + }, + } + + // Add methods + cmd.AddCommand(newGetQuota()) + cmd.AddCommand(newListQuotas()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start get-quota command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getQuotaOverrides []func( + *cobra.Command, + *catalog.GetQuotaRequest, +) + +func newGetQuota() *cobra.Command { + cmd := &cobra.Command{} + + var getQuotaReq catalog.GetQuotaRequest + + // TODO: short flags + + cmd.Use = "get-quota PARENT_SECURABLE_TYPE PARENT_FULL_NAME QUOTA_NAME" + cmd.Short = `Get information for a single resource quota.` + cmd.Long = `Get information for a single resource quota. + + The GetQuota API returns usage information for a single resource quota, + defined as a child-parent pair. This API also refreshes the quota count if it + is out of date. Refreshes are triggered asynchronously. The updated count + might not be returned in the first call. + + Arguments: + PARENT_SECURABLE_TYPE: Securable type of the quota parent. + PARENT_FULL_NAME: Full name of the parent resource. Provide the metastore ID if the parent + is a metastore. + QUOTA_NAME: Name of the quota. Follows the pattern of the quota type, with "-quota" + added as a suffix.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getQuotaReq.ParentSecurableType = args[0] + getQuotaReq.ParentFullName = args[1] + getQuotaReq.QuotaName = args[2] + + response, err := w.ResourceQuotas.GetQuota(ctx, getQuotaReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getQuotaOverrides { + fn(cmd, &getQuotaReq) + } + + return cmd +} + +// start list-quotas command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listQuotasOverrides []func( + *cobra.Command, + *catalog.ListQuotasRequest, +) + +func newListQuotas() *cobra.Command { + cmd := &cobra.Command{} + + var listQuotasReq catalog.ListQuotasRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listQuotasReq.MaxResults, "max-results", listQuotasReq.MaxResults, `The number of quotas to return.`) + cmd.Flags().StringVar(&listQuotasReq.PageToken, "page-token", listQuotasReq.PageToken, `Opaque token for the next page of results.`) + + cmd.Use = "list-quotas" + cmd.Short = `List all resource quotas under a metastore.` + cmd.Long = `List all resource quotas under a metastore. + + ListQuotas returns all quota values under the metastore. There are no SLAs on + the freshness of the counts returned. This API does not trigger a refresh of + quota counts.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response := w.ResourceQuotas.ListQuotas(ctx, listQuotasReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listQuotasOverrides { + fn(cmd, &listQuotasReq) + } + + return cmd +} + +// end service ResourceQuotas diff --git a/go.mod b/go.mod index 838a45f3..4aa27992 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.44.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.45.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index f55f329f..2e58948a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.44.0 h1:9/FZACv4EFQIOYxfwYVKnY7v46xio9FKCw9tpKB2O/s= -github.com/databricks/databricks-sdk-go v0.44.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.45.0 h1:wdx5Wm/ESrahdHeq62WrjLeGjV4r722LLanD8ahI0Mo= +github.com/databricks/databricks-sdk-go v0.45.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From edc08149d3d6a4943072f23492fce494082edf90 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 27 Aug 2024 14:51:10 +0200 Subject: [PATCH 04/78] Disable prompt for storage-credentials get command (#1723) ## Changes Fixes #1079 --- .codegen/service.go.tmpl | 1 + .../storage-credentials.go | 22 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index 111745e4..281dfd6e 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -154,6 +154,7 @@ func new{{.PascalName}}() *cobra.Command { "provider-exchanges delete-listing-from-exchange" "provider-exchanges list-exchanges-for-listing" "provider-exchanges list-listings-for-exchange" + "storage-credentials get" -}} {{- $fullCommandName := (print $serviceName " " .KebabName) -}} {{- $noPrompt := or .IsCrudCreate (in $excludeFromPrompts $fullCommandName) }} diff --git a/cmd/workspace/storage-credentials/storage-credentials.go b/cmd/workspace/storage-credentials/storage-credentials.go index 18656a61..f4ec5eb4 100755 --- a/cmd/workspace/storage-credentials/storage-credentials.go +++ b/cmd/workspace/storage-credentials/storage-credentials.go @@ -241,28 +241,16 @@ func newGet() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No NAME argument specified. Loading names for Storage Credentials drop-down." - names, err := w.StorageCredentials.StorageCredentialInfoNameToIdMap(ctx, catalog.ListStorageCredentialsRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Storage Credentials drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Name of the storage credential") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have name of the storage credential") - } getReq.Name = args[0] response, err := w.StorageCredentials.Get(ctx, getReq) From 70363836d5a462851d8d2196eb288d7bdb1ad01d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 28 Aug 2024 13:39:06 +0200 Subject: [PATCH 05/78] Correctly mark PyPI package name specs with multiple specifiers as remote libraries (#1725) Correctly mark pypi package name specs with multiple specifiers as remote libraries Fixes this https://github.com/databricks/cli/issues/1728 --- bundle/libraries/local_path.go | 6 ++++-- bundle/libraries/local_path_test.go | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index e4956240..6d60d56b 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -72,9 +72,11 @@ func IsLibraryLocal(dep string) bool { // ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_). // \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security]. -// ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?)?: Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1). +// ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?): Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1). +// ,?: Optionally matches a comma (,) at the end of the specifier which is used to separate multiple specifiers. +// There can be multiple version specifiers separated by commas or no specifiers. // Spec for package name and version specifier: https://pip.pypa.io/en/stable/reference/requirement-specifiers/ -var packageRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+\s?(\[.*\])?\s?((==|!=|<=|>=|~=|==|>|<)\s?\d+(\.\d+){0,2}(\.\*)?)?$`) +var packageRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+\s?(\[.*\])?\s?((==|!=|<=|>=|~=|==|>|<)\s?\d+(\.\d+){0,2}(\.\*)?,?)*$`) func isPackage(name string) bool { if packageRegex.MatchString(name) { diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index 667d64ec..dc157cad 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -62,6 +62,8 @@ func TestIsLibraryLocal(t *testing.T) { {path: "beautifulsoup4 ~= 4.12.3", expected: false}, {path: "beautifulsoup4[security, tests]", expected: false}, {path: "beautifulsoup4[security, tests] ~= 4.12.3", expected: false}, + {path: "beautifulsoup4>=1.0.0,<2.0.0", expected: false}, + {path: "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0", expected: false}, {path: "https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, {path: "pip @ https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, {path: "requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip", expected: false}, From 85459c1963e6e86f3ece1029caae51156f6b87f6 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Wed, 28 Aug 2024 14:14:19 +0200 Subject: [PATCH 06/78] Improve error handling for /Volumes paths in mode: development (#1716) ## Changes * Provide a more helpful error when using an artifact_path based on /Volumes * Allow the use of short_names in /Volumes paths ## Example cases Example of a valid /Volumes artifact_path: * `artifact_path: /Volumes/catalog/schema/${workspace.current_user.short_name}/libs` Example of an invalid /Volumes path (when using `mode: development`): * `artifact_path: /Volumes/catalog/schema/libs` * Resulting error: `artifact_path should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'` --- bundle/config/mutator/process_target_mode.go | 35 +++++++++++++------ .../mutator/process_target_mode_test.go | 12 ++++++- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 92ed2868..70382f05 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -64,6 +64,7 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { } func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics p := b.Config.Presets u := b.Config.Workspace.CurrentUser @@ -74,44 +75,56 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { // status to UNPAUSED at the level of an individual object, whic hwas // historically allowed.) if p.TriggerPauseStatus == config.Unpaused { - return diag.Diagnostics{{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default", Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, - }} + }) } // Make sure this development copy has unique names and paths to avoid conflicts if path := findNonUserPath(b); path != "" { - return diag.Errorf("%s must start with '~/' or contain the current username when using 'mode: development'", path) + if path == "artifact_path" && strings.HasPrefix(b.Config.Workspace.ArtifactPath, "/Volumes") { + // For Volumes paths we recommend including the current username as a substring + diags = diags.Extend(diag.Errorf("%s should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", path)) + } else { + // For non-Volumes paths recommend simply putting things in the home folder + diags = diags.Extend(diag.Errorf("%s must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'", path)) + } } if p.NamePrefix != "" && !strings.Contains(p.NamePrefix, u.ShortName) && !strings.Contains(p.NamePrefix, u.UserName) { // Resources such as pipelines require a unique name, e.g. '[dev steve] my_pipeline'. // For this reason we require the name prefix to contain the current username; // it's a pitfall for users if they don't include it and later find out that // only a single user can do development deployments. - return diag.Diagnostics{{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", Locations: []dyn.Location{b.Config.GetLocation("presets.name_prefix")}, - }} + }) } - return nil + return diags } +// findNonUserPath finds the first workspace path such as root_path that doesn't +// contain the current username or current user's shortname. func findNonUserPath(b *bundle.Bundle) string { - username := b.Config.Workspace.CurrentUser.UserName + containsName := func(path string) bool { + username := b.Config.Workspace.CurrentUser.UserName + shortname := b.Config.Workspace.CurrentUser.ShortName + return strings.Contains(path, username) || strings.Contains(path, shortname) + } - if b.Config.Workspace.RootPath != "" && !strings.Contains(b.Config.Workspace.RootPath, username) { + if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) { return "root_path" } - if b.Config.Workspace.StatePath != "" && !strings.Contains(b.Config.Workspace.StatePath, username) { + if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { return "state_path" } - if b.Config.Workspace.FilePath != "" && !strings.Contains(b.Config.Workspace.FilePath, username) { + if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) { return "file_path" } - if b.Config.Workspace.ArtifactPath != "" && !strings.Contains(b.Config.Workspace.ArtifactPath, username) { + if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) { return "artifact_path" } return "" diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 1c8671b4..42f1929c 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -230,10 +230,20 @@ func TestValidateDevelopmentMode(t *testing.T) { diags := validateDevelopmentMode(b) require.NoError(t, diags.Error()) + // Test with /Volumes path + b = mockBundle(config.Development) + b.Config.Workspace.ArtifactPath = "/Volumes/catalog/schema/lennart/libs" + diags = validateDevelopmentMode(b) + require.NoError(t, diags.Error()) + b.Config.Workspace.ArtifactPath = "/Volumes/catalog/schema/libs" + diags = validateDevelopmentMode(b) + require.ErrorContains(t, diags.Error(), "artifact_path should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'") + // Test with a bundle that has a non-user path + b = mockBundle(config.Development) b.Config.Workspace.RootPath = "/Shared/.bundle/x/y/state" diags = validateDevelopmentMode(b) - require.ErrorContains(t, diags.Error(), "root_path") + require.ErrorContains(t, diags.Error(), "root_path must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'") // Test with a bundle that has an unpaused trigger pause status b = mockBundle(config.Development) From 7dcc791b05905a0ac78345651320969212f0123f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 28 Aug 2024 15:09:07 +0200 Subject: [PATCH 07/78] [Release] Release v0.227.1 (#1729) CLI: * Disable prompt for storage-credentials get command ([#1723](https://github.com/databricks/cli/pull/1723)). Bundles: * Do not treat empty path as a local path ([#1717](https://github.com/databricks/cli/pull/1717)). * Correctly mark PyPI package name specs with multiple specifiers as remote libraries ([#1725](https://github.com/databricks/cli/pull/1725)). * Improve error handling for /Volumes paths in mode: development ([#1716](https://github.com/databricks/cli/pull/1716)). Internal: * Ignore CLI version check on development builds of the CLI ([#1714](https://github.com/databricks/cli/pull/1714)). API Changes: * Added `databricks resource-quotas` command group. * Added `databricks policy-compliance-for-clusters` command group. * Added `databricks policy-compliance-for-jobs` command group. OpenAPI commit 3eae49b444cac5a0118a3503e5b7ecef7f96527a (2024-08-21) Dependency updates: * Bump github.com/databricks/databricks-sdk-go from 0.44.0 to 0.45.0 ([#1719](https://github.com/databricks/cli/pull/1719)). * Revert hc-install version to 0.7.0 ([#1711](https://github.com/databricks/cli/pull/1711)). --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a62d09..fac7d597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Version changelog +## [Release] Release v0.227.1 + +CLI: + * Disable prompt for storage-credentials get command ([#1723](https://github.com/databricks/cli/pull/1723)). + +Bundles: + * Do not treat empty path as a local path ([#1717](https://github.com/databricks/cli/pull/1717)). + * Correctly mark PyPI package name specs with multiple specifiers as remote libraries ([#1725](https://github.com/databricks/cli/pull/1725)). + * Improve error handling for /Volumes paths in mode: development ([#1716](https://github.com/databricks/cli/pull/1716)). + +Internal: + * Ignore CLI version check on development builds of the CLI ([#1714](https://github.com/databricks/cli/pull/1714)). + +API Changes: + * Added `databricks resource-quotas` command group. + * Added `databricks policy-compliance-for-clusters` command group. + * Added `databricks policy-compliance-for-jobs` command group. + +OpenAPI commit 3eae49b444cac5a0118a3503e5b7ecef7f96527a (2024-08-21) +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.44.0 to 0.45.0 ([#1719](https://github.com/databricks/cli/pull/1719)). + * Revert hc-install version to 0.7.0 ([#1711](https://github.com/databricks/cli/pull/1711)). + ## [Release] Release v0.227.0 CLI: From 43ace69bb955a445885d4477a55de88c237a05ba Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 29 Aug 2024 14:47:44 +0200 Subject: [PATCH 08/78] Consider serverless clusters as compatible for Python wheel tasks (#1733) ## Changes Consider serverless clusters as compatible for Python wheel tasks. Fixes a `Python wheel tasks require compute with DBR 13.3+ to include local libraries` warning shown for serverless clusters --- bundle/python/warning.go | 20 ++++++++++++++++---- bundle/python/warning_test.go | 4 +++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bundle/python/warning.go b/bundle/python/warning.go index d53796d7..0e9d8bef 100644 --- a/bundle/python/warning.go +++ b/bundle/python/warning.go @@ -2,6 +2,7 @@ package python import ( "context" + "strconv" "strings" "github.com/databricks/cli/bundle" @@ -38,7 +39,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { if task.NewCluster != nil { - if lowerThanExpectedVersion(ctx, task.NewCluster.SparkVersion) { + if lowerThanExpectedVersion(task.NewCluster.SparkVersion) { return true } } @@ -47,7 +48,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { for _, job := range b.Config.Resources.Jobs { for _, cluster := range job.JobClusters { if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" { - if lowerThanExpectedVersion(ctx, cluster.NewCluster.SparkVersion) { + if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) { return true } } @@ -64,7 +65,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { return false } - if lowerThanExpectedVersion(ctx, version) { + if lowerThanExpectedVersion(version) { return true } } @@ -73,7 +74,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { return false } -func lowerThanExpectedVersion(ctx context.Context, sparkVersion string) bool { +func lowerThanExpectedVersion(sparkVersion string) bool { parts := strings.Split(sparkVersion, ".") if len(parts) < 2 { return false @@ -82,6 +83,17 @@ func lowerThanExpectedVersion(ctx context.Context, sparkVersion string) bool { if parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99) parts[1] = "99" } + + // if any of the version parts are not numbers, we can't compare + // so consider it as compatible version + if _, err := strconv.Atoi(parts[0]); err != nil { + return false + } + + if _, err := strconv.Atoi(parts[1]); err != nil { + return false + } + v := "v" + parts[0] + "." + parts[1] return semver.Compare(v, "v13.1") < 0 } diff --git a/bundle/python/warning_test.go b/bundle/python/warning_test.go index b2296392..a5ab7563 100644 --- a/bundle/python/warning_test.go +++ b/bundle/python/warning_test.go @@ -344,6 +344,8 @@ func TestSparkVersionLowerThanExpected(t *testing.T) { "14.1.x-scala2.12": false, "13.x-snapshot-scala-2.12": false, "13.x-rc-scala-2.12": false, + "client.1.10-scala2.12": false, + "latest-stable-gpu-scala2.11": false, "10.4.x-aarch64-photon-scala2.12": true, "10.4.x-scala2.12": true, "13.0.x-scala2.12": true, @@ -351,7 +353,7 @@ func TestSparkVersionLowerThanExpected(t *testing.T) { } for k, v := range testCases { - result := lowerThanExpectedVersion(context.Background(), k) + result := lowerThanExpectedVersion(k) require.Equal(t, v, result, k) } } From 0f4891f0fef96fab6bffce7572caa2b97615911a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 29 Aug 2024 15:02:34 +0200 Subject: [PATCH 09/78] Add `dyn.Time` to box a timestamp with its original string value (#1732) ## Changes If not explicitly quoted, the YAML loader interprets a value like `2024-08-29` as a timestamp. Such a value is usually intended to be a string instead. Our normalization logic was not able to turn a time value back into the original string. This change boxes the time value to include its original string representation. Normalization of one of these values into a string can now use the original input value. ## Tests Unit tests in `libs/dyn/convert`. --- libs/dyn/convert/normalize.go | 2 + libs/dyn/convert/normalize_test.go | 8 ++++ libs/dyn/kind.go | 3 +- libs/dyn/merge/override_test.go | 12 +++--- libs/dyn/time.go | 62 ++++++++++++++++++++++++++++++ libs/dyn/time_test.go | 41 ++++++++++++++++++++ libs/dyn/value.go | 3 +- libs/dyn/value_underlying.go | 7 ++-- libs/dyn/value_underlying_test.go | 2 +- libs/dyn/yamlloader/loader.go | 15 ++------ libs/dyn/yamlsaver/saver.go | 2 +- libs/dyn/yamlsaver/saver_test.go | 9 +++-- 12 files changed, 136 insertions(+), 30 deletions(-) create mode 100644 libs/dyn/time.go create mode 100644 libs/dyn/time_test.go diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index c80a914f..bc80a150 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -267,6 +267,8 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path out = strconv.FormatInt(src.MustInt(), 10) case dyn.KindFloat: out = strconv.FormatFloat(src.MustFloat(), 'f', -1, 64) + case dyn.KindTime: + out = src.MustTime().String() case dyn.KindNil: // Return a warning if the field is present but has a null value. return dyn.InvalidValue, diags.Append(nullWarning(dyn.KindString, src, path)) diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index c2256615..4b2a3c18 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -569,6 +569,14 @@ func TestNormalizeStringFromFloat(t *testing.T) { assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) } +func TestNormalizeStringFromTime(t *testing.T) { + var typ string + vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vout, err := Normalize(&typ, vin) + assert.Empty(t, err) + assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) +} + func TestNormalizeStringError(t *testing.T) { var typ string vin := dyn.V(map[string]dyn.Value{"an": dyn.V("error")}) diff --git a/libs/dyn/kind.go b/libs/dyn/kind.go index 9d507fbc..9c0c1442 100644 --- a/libs/dyn/kind.go +++ b/libs/dyn/kind.go @@ -2,7 +2,6 @@ package dyn import ( "fmt" - "time" ) type Kind int @@ -34,7 +33,7 @@ func kindOf(v any) Kind { return KindInt case float32, float64: return KindFloat - case time.Time: + case Time: return KindTime case nil: return KindNil diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go index 9d41a526..264c32e5 100644 --- a/libs/dyn/merge/override_test.go +++ b/libs/dyn/merge/override_test.go @@ -83,16 +83,16 @@ func TestOverride_Primitive(t *testing.T) { { name: "time (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), - right: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), - expected: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), + left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), + right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}), + expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10001)), []dyn.Location{rightLocation}), }, { name: "time (not updated)", state: visitorState{}, - left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), - right: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{rightLocation}), - expected: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), + left: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), + right: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{rightLocation}), + expected: dyn.NewValue(dyn.FromTime(time.UnixMilli(10000)), []dyn.Location{leftLocation}), }, { name: "different types (updated)", diff --git a/libs/dyn/time.go b/libs/dyn/time.go new file mode 100644 index 00000000..b3b3de12 --- /dev/null +++ b/libs/dyn/time.go @@ -0,0 +1,62 @@ +package dyn + +import ( + "fmt" + "time" +) + +// Time represents a time-like primitive value. +// +// It represents a timestamp and includes the original string value +// that was parsed to create the timestamp. This makes it possible +// to coalesce a value that YAML interprets as a timestamp back into +// a string without losing information. +type Time struct { + t time.Time + s string +} + +// NewTime creates a new Time from the given string. +func NewTime(str string) (Time, error) { + // Try a couple of layouts + for _, layout := range []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + } { + t, terr := time.Parse(layout, str) + if terr == nil { + return Time{t: t, s: str}, nil + } + } + + return Time{}, fmt.Errorf("invalid time value: %q", str) +} + +// MustTime creates a new Time from the given string. +// It panics if the string cannot be parsed. +func MustTime(str string) Time { + t, err := NewTime(str) + if err != nil { + panic(err) + } + return t +} + +// FromTime creates a new Time from the given time.Time. +// It uses the RFC3339Nano format for its string representation. +// This guarantees that it can roundtrip into a string without losing information. +func FromTime(t time.Time) Time { + return Time{t: t, s: t.Format(time.RFC3339Nano)} +} + +// Time returns the time.Time value. +func (t Time) Time() time.Time { + return t.t +} + +// String returns the original string value that was parsed to create the timestamp. +func (t Time) String() string { + return t.s +} diff --git a/libs/dyn/time_test.go b/libs/dyn/time_test.go new file mode 100644 index 00000000..653d6855 --- /dev/null +++ b/libs/dyn/time_test.go @@ -0,0 +1,41 @@ +package dyn_test + +import ( + "testing" + "time" + + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestTimeValid(t *testing.T) { + for _, tc := range []string{ + "2024-08-29", + "2024-01-15T12:34:56.789012345Z", + } { + tm, err := dyn.NewTime(tc) + if assert.NoError(t, err) { + assert.NotEqual(t, time.Time{}, tm.Time()) + assert.Equal(t, tc, tm.String()) + } + } +} + +func TestTimeInvalid(t *testing.T) { + tm, err := dyn.NewTime("invalid") + assert.Error(t, err) + assert.Equal(t, dyn.Time{}, tm) +} + +func TestTimeFromTime(t *testing.T) { + tref := time.Now() + t1 := dyn.FromTime(tref) + + // Verify that the underlying value is the same. + assert.Equal(t, tref, t1.Time()) + + // Verify that the string representation can be used to construct the same. + t2, err := dyn.NewTime(t1.String()) + assert.NoError(t, err) + assert.True(t, t1.Time().Equal(t2.Time())) +} diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 2aed2f6c..81524db7 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -127,7 +127,8 @@ func (v Value) AsAny() any { case KindFloat: return v.v case KindTime: - return v.v + t := v.v.(Time) + return t.Time() default: // Panic because we only want to deal with known types. panic(fmt.Sprintf("invalid kind: %d", v.k)) diff --git a/libs/dyn/value_underlying.go b/libs/dyn/value_underlying.go index 2f0f26a1..0a867375 100644 --- a/libs/dyn/value_underlying.go +++ b/libs/dyn/value_underlying.go @@ -2,7 +2,6 @@ package dyn import ( "fmt" - "time" ) // AsMap returns the underlying mapping if this value is a map, @@ -123,14 +122,14 @@ func (v Value) MustFloat() float64 { // AsTime returns the underlying time if this value is a time, // the zero value and false otherwise. -func (v Value) AsTime() (time.Time, bool) { - vv, ok := v.v.(time.Time) +func (v Value) AsTime() (Time, bool) { + vv, ok := v.v.(Time) return vv, ok } // MustTime returns the underlying time if this value is a time, // panics otherwise. -func (v Value) MustTime() time.Time { +func (v Value) MustTime() Time { vv, ok := v.AsTime() if !ok || v.k != KindTime { panic(fmt.Sprintf("expected kind %s, got %s", KindTime, v.k)) diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go index e35cde58..73baeeeb 100644 --- a/libs/dyn/value_underlying_test.go +++ b/libs/dyn/value_underlying_test.go @@ -143,7 +143,7 @@ func TestValueUnderlyingFloat(t *testing.T) { } func TestValueUnderlyingTime(t *testing.T) { - v := dyn.V(time.Now()) + v := dyn.V(dyn.FromTime(time.Now())) vv1, ok := v.AsTime() assert.True(t, ok) diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index fbb52b50..c3e8d081 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -5,7 +5,6 @@ import ( "math" "strconv" "strings" - "time" "github.com/databricks/cli/libs/dyn" "gopkg.in/yaml.v3" @@ -207,17 +206,9 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error case "!!null": return dyn.NewValue(nil, []dyn.Location{loc}), nil case "!!timestamp": - // Try a couple of layouts - for _, layout := range []string{ - "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. - "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". - "2006-1-2 15:4:5.999999999", // space separated with no time zone - "2006-1-2", // date only - } { - t, terr := time.Parse(layout, node.Value) - if terr == nil { - return dyn.NewValue(t, []dyn.Location{loc}), nil - } + t, err := dyn.NewTime(node.Value) + if err == nil { + return dyn.NewValue(t, []dyn.Location{loc}), nil } return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) default: diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go index fe4cfb85..f4c7157f 100644 --- a/libs/dyn/yamlsaver/saver.go +++ b/libs/dyn/yamlsaver/saver.go @@ -129,7 +129,7 @@ func (s *saver) toYamlNodeWithStyle(v dyn.Value, style yaml.Style) (*yaml.Node, case dyn.KindFloat: return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprint(v.MustFloat()), Style: style}, nil case dyn.KindTime: - return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().UTC().String(), Style: style}, nil + return &yaml.Node{Kind: yaml.ScalarNode, Value: v.MustTime().String(), Style: style}, nil default: // Panic because we only want to deal with known types. panic(fmt.Sprintf("invalid kind: %d", v.Kind())) diff --git a/libs/dyn/yamlsaver/saver_test.go b/libs/dyn/yamlsaver/saver_test.go index 38709010..aa481c20 100644 --- a/libs/dyn/yamlsaver/saver_test.go +++ b/libs/dyn/yamlsaver/saver_test.go @@ -2,10 +2,10 @@ package yamlsaver import ( "testing" - "time" "github.com/databricks/cli/libs/dyn" assert "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -45,11 +45,14 @@ func TestMarshalBoolValue(t *testing.T) { } func TestMarshalTimeValue(t *testing.T) { + tm, err := dyn.NewTime("1970-01-01") + require.NoError(t, err) + s := NewSaver() - var timeValue = dyn.V(time.Unix(0, 0)) + var timeValue = dyn.V(tm) v, err := s.toYamlNode(timeValue) assert.NoError(t, err) - assert.Equal(t, "1970-01-01 00:00:00 +0000 UTC", v.Value) + assert.Equal(t, "1970-01-01", v.Value) assert.Equal(t, yaml.ScalarNode, v.Kind) } From 5fac7edcdf5f3e371920c2907a311cd1564bebf8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 29 Aug 2024 16:41:12 +0200 Subject: [PATCH 10/78] Pass along $AZURE_CONFIG_FILE to Terraform process (#1734) ## Changes This ensures that the CLI and Terraform can both use an Azure CLI session configured under a non-standard path. This is the default behavior on Azure DevOps when using the AzureCLI@2 task. Fixes #1722. ## Tests Unit test. --- bundle/deploy/terraform/init.go | 7 +++++++ bundle/deploy/terraform/init_test.go | 19 ++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index e7f720d0..7d75ee8a 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -111,6 +111,13 @@ func inheritEnvVars(ctx context.Context, environ map[string]string) error { environ["PATH"] = path } + // Include $AZURE_CONFIG_FILE in set of environment variables to pass along. + // This is set in Azure DevOps by the AzureCLI@2 task. + azureConfigFile, ok := env.Lookup(ctx, "AZURE_CONFIG_FILE") + if ok { + environ["AZURE_CONFIG_FILE"] = azureConfigFile + } + // Include $TF_CLI_CONFIG_FILE to override terraform provider in development. // See: https://developer.hashicorp.com/terraform/cli/config/config-file#explicit-installation-method-configuration devConfigFile, ok := env.Lookup(ctx, "TF_CLI_CONFIG_FILE") diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index 94e47dbc..450e7eb6 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -269,19 +269,20 @@ func TestSetUserAgentExtraEnvVar(t *testing.T) { } func TestInheritEnvVars(t *testing.T) { - env := map[string]string{} - t.Setenv("HOME", "/home/testuser") t.Setenv("PATH", "/foo:/bar") t.Setenv("TF_CLI_CONFIG_FILE", "/tmp/config.tfrc") + t.Setenv("AZURE_CONFIG_FILE", "/tmp/foo/bar") - err := inheritEnvVars(context.Background(), env) - - require.NoError(t, err) - - require.Equal(t, env["HOME"], "/home/testuser") - require.Equal(t, env["PATH"], "/foo:/bar") - require.Equal(t, env["TF_CLI_CONFIG_FILE"], "/tmp/config.tfrc") + ctx := context.Background() + env := map[string]string{} + err := inheritEnvVars(ctx, env) + if assert.NoError(t, err) { + assert.Equal(t, "/home/testuser", env["HOME"]) + assert.Equal(t, "/foo:/bar", env["PATH"]) + assert.Equal(t, "/tmp/config.tfrc", env["TF_CLI_CONFIG_FILE"]) + assert.Equal(t, "/tmp/foo/bar", env["AZURE_CONFIG_FILE"]) + } } func TestSetUserProfileFromInheritEnvVars(t *testing.T) { From 2e000f1ebd0f22933bb2fd64ee7a9f44cb53b5d8 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 29 Aug 2024 21:07:21 +0200 Subject: [PATCH 11/78] Use materialized views in the default-sql template (#1709) ## Changes Materialized views now support `CREATE OR REPLACE` ([docs](https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-create-materialized-view.html))! This makes it possible to use them with Workflows in DABs.This PR updates the template to use a materialized view rather than a regular view. ## Tests Manually validated in production. --- .../template/{{.project_name}}/src/orders_daily.sql.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index 8a9d12ea..e5ceb77a 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -1,10 +1,9 @@ -- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) -{{- /* We can't use a materialized view here since they don't support 'create or refresh' yet.*/}} USE CATALOG {{"{{"}}catalog{{"}}"}}; USE IDENTIFIER({{"{{"}}schema{{"}}"}}); -CREATE OR REPLACE VIEW +CREATE OR REPLACE MATERIALIZED VIEW orders_daily AS SELECT order_date, count(*) AS number_of_orders From ed4a4585c07602e84a8ce97336688463d9833527 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Fri, 30 Aug 2024 09:32:10 +0200 Subject: [PATCH 12/78] Update templates to latest LTS DBR (#1715) ## Changes This updates the templates to use the latest DBR 15 LTS version. - [x] DB Connect 15.4 must be released + validated before this can go out --- libs/template/templates/dbt-sql/library/versions.tmpl | 4 ++-- libs/template/templates/default-python/library/versions.tmpl | 4 ++-- libs/template/templates/default-sql/library/versions.tmpl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/template/templates/dbt-sql/library/versions.tmpl b/libs/template/templates/dbt-sql/library/versions.tmpl index f9a879d2..7d0c88e7 100644 --- a/libs/template/templates/dbt-sql/library/versions.tmpl +++ b/libs/template/templates/dbt-sql/library/versions.tmpl @@ -1,7 +1,7 @@ {{define "latest_lts_dbr_version" -}} - 13.3.x-scala2.12 + 15.4.x-scala2.12 {{- end}} {{define "latest_lts_db_connect_version_spec" -}} - >=13.3,<13.4 + >=15.4,<15.5 {{- end}} diff --git a/libs/template/templates/default-python/library/versions.tmpl b/libs/template/templates/default-python/library/versions.tmpl index f9a879d2..7d0c88e7 100644 --- a/libs/template/templates/default-python/library/versions.tmpl +++ b/libs/template/templates/default-python/library/versions.tmpl @@ -1,7 +1,7 @@ {{define "latest_lts_dbr_version" -}} - 13.3.x-scala2.12 + 15.4.x-scala2.12 {{- end}} {{define "latest_lts_db_connect_version_spec" -}} - >=13.3,<13.4 + >=15.4,<15.5 {{- end}} diff --git a/libs/template/templates/default-sql/library/versions.tmpl b/libs/template/templates/default-sql/library/versions.tmpl index f9a879d2..7d0c88e7 100644 --- a/libs/template/templates/default-sql/library/versions.tmpl +++ b/libs/template/templates/default-sql/library/versions.tmpl @@ -1,7 +1,7 @@ {{define "latest_lts_dbr_version" -}} - 13.3.x-scala2.12 + 15.4.x-scala2.12 {{- end}} {{define "latest_lts_db_connect_version_spec" -}} - >=13.3,<13.4 + >=15.4,<15.5 {{- end}} From 70ce80251853ae6443ea280ba6b2bc4614c7267a Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Fri, 30 Aug 2024 15:29:00 +0200 Subject: [PATCH 13/78] PythonMutator: preserve normalize diagnostics (#1735) ## Changes Preserve diagnostics if there are any errors or warnings when PythonMutator normalizes output. If anything goes wrong during conversion, diagnostics contain the relevant location and path. ## Tests Unit tests --- .../config/mutator/python/python_mutator.go | 35 +++++++++++-------- .../mutator/python/python_mutator_test.go | 20 ++++++++++- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 4f44df0a..bf30f070 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -205,10 +205,8 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) } - output, err := loadOutputFile(rootPath, outputPath) - if err != nil { - return dyn.InvalidValue, diag.Errorf("failed to load Python mutator output: %s", err) - } + output, outputDiags := loadOutputFile(rootPath, outputPath) + pythonDiagnostics = pythonDiagnostics.Extend(outputDiags) // we pass through pythonDiagnostic because it contains warnings return output, pythonDiagnostics @@ -225,10 +223,10 @@ func writeInputFile(inputPath string, input dyn.Value) error { return os.WriteFile(inputPath, rootConfigJson, 0600) } -func loadOutputFile(rootPath string, outputPath string) (dyn.Value, error) { +func loadOutputFile(rootPath string, outputPath string) (dyn.Value, diag.Diagnostics) { outputFile, err := os.Open(outputPath) if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to open output file: %w", err) + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) } defer outputFile.Close() @@ -243,27 +241,34 @@ func loadOutputFile(rootPath string, outputPath string) (dyn.Value, error) { // for that, we pass virtualPath instead of outputPath as file location virtualPath, err := filepath.Abs(filepath.Join(rootPath, "__generated_by_pydabs__.yml")) if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to get absolute path: %w", err) + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) } generated, err := yamlloader.LoadYAML(virtualPath, outputFile) if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to parse output file: %w", err) + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) } - normalized, diagnostic := convert.Normalize(config.Root{}, generated) - if diagnostic.Error() != nil { - return dyn.InvalidValue, fmt.Errorf("failed to normalize output: %w", diagnostic.Error()) - } + return strictNormalize(config.Root{}, generated) +} + +func strictNormalize(dst any, generated dyn.Value) (dyn.Value, diag.Diagnostics) { + normalized, diags := convert.Normalize(dst, generated) // warnings shouldn't happen because output should be already normalized // when it happens, it's a bug in the mutator, and should be treated as an error - for _, d := range diagnostic.Filter(diag.Warning) { - return dyn.InvalidValue, fmt.Errorf("failed to normalize output: %s", d.Summary) + strictDiags := diag.Diagnostics{} + + for _, d := range diags { + if d.Severity == diag.Warning { + d.Severity = diag.Error + } + + strictDiags = strictDiags.Append(d) } - return normalized, nil + return normalized, strictDiags } // loadDiagnosticsFile loads diagnostics from a file. diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index ea02d1ce..26350d88 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -10,6 +10,8 @@ import ( "runtime" "testing" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/bundle/env" @@ -255,7 +257,7 @@ func TestPythonMutator_badOutput(t *testing.T) { mutator := PythonMutator(PythonMutatorPhaseLoad) diag := bundle.Apply(ctx, b, mutator) - assert.EqualError(t, diag.Error(), "failed to load Python mutator output: failed to normalize output: unknown field: unknown_property") + assert.EqualError(t, diag.Error(), "unknown field: unknown_property") } func TestPythonMutator_disabled(t *testing.T) { @@ -546,6 +548,22 @@ func TestInterpreterPath(t *testing.T) { } } +func TestStrictNormalize(t *testing.T) { + // NB: there is no way to trigger diag.Error, so we don't test it + + type TestStruct struct { + A int `json:"a"` + } + + value := dyn.NewValue(map[string]dyn.Value{"A": dyn.NewValue("abc", nil)}, nil) + + _, diags := convert.Normalize(TestStruct{}, value) + _, strictDiags := strictNormalize(TestStruct{}, value) + + assert.False(t, diags.HasError()) + assert.True(t, strictDiags.HasError()) +} + func withProcessStub(t *testing.T, args []string, output string, diagnostics string) context.Context { ctx := context.Background() ctx, stub := process.WithStub(ctx) From 5d9910c8e0f3119276b6ee4afd534ac82d4fb6c2 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:09:08 +0530 Subject: [PATCH 14/78] Make lock optional in the JSON schema (#1738) Fixes https://github.com/databricks/cli/issues/1561 --- bundle/config/deployment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index 7f0f57a8..b7efb445 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -6,5 +6,5 @@ type Deployment struct { FailOnActiveRuns bool `json:"fail_on_active_runs,omitempty"` // Lock configures locking behavior on deployment. - Lock Lock `json:"lock"` + Lock Lock `json:"lock,omitempty"` } From 582558cac22017c2265b384c5380179975dc6f3d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 2 Sep 2024 11:17:18 +0200 Subject: [PATCH 15/78] Do not suppress normalisation diagnostics for resolving variables (#1740) ## Changes Tested on the following bundle configuration ``` bundle: name: clusters mode: development variables: webhook_notifications: description: Webhook URL for notifications type: complex default: on_failure: id: 6a6c04c1-389c-4534-95af-b68b62a9dbe6 resources: jobs: test_job: name: "Andrew Nester Test Job" tasks: - task_key: test_task notebook_task: notebook_path: "./src/test.py" new_cluster: num_workers: 2 node_type_id: "i3.xlarge" autoscale: min_workers: 2 max_workers: 7 spark_version: "12.2.x-scala2.12" spark_conf: "spark.executor.memory": "2g" webhook_notifications: ${var.webhook_notifications} ``` bundle validate output is below ``` andrew.nester@HFW9Y94129 wheel % databricks bundle validate Warning: expected sequence, found map at resources.jobs.test_job.webhook_notifications.on_failure in bundle.yml:11:9 Name: clusters Target: default Workspace: User: andrew.nester@databricks.com Path: /Users/andrew.nester@databricks.com/.bundle/clusters/default ``` **Note** that error correctly points to the variable --- .../config/mutator/resolve_variable_references.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 61940be5..5e5b7610 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/log" ) type resolveVariableReferences struct { @@ -124,6 +123,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) // We rewrite it here to make the resolution logic simpler. varPath := dyn.NewPath(dyn.Key("var")) + var diags diag.Diagnostics err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { // Synthesize a copy of the root that has all fields that are present in the type // but not set in the dynamic value set to their corresponding empty value. @@ -180,14 +180,13 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) // Normalize the result because variable resolution may have been applied to non-string fields. // For example, a variable reference may have been resolved to a integer. - root, diags := convert.Normalize(b.Config, root) - for _, diag := range diags { - // This occurs when a variable's resolved value is incompatible with the field's type. - // Log a warning until we have a better way to surface these diagnostics to the user. - log.Warnf(ctx, "normalization diagnostic: %s", diag.Summary) - } + root, normaliseDiags := convert.Normalize(b.Config, root) + diags = diags.Extend(normaliseDiags) return root, nil }) - return diag.FromErr(err) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + return diags } From ed448815b4e9881494d3ef561ec84ef694ea4b11 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Mon, 2 Sep 2024 11:49:30 +0200 Subject: [PATCH 16/78] PythonMutator: explain missing package error (#1736) ## Changes Explain the error when the `databricks-pydabs` package is not installed or the Python environment isn't correctly activated. Example output: ``` Error: python mutator process failed: ".venv/bin/python3 -m databricks.bundles.build --phase load --input .../input.json --output .../output.json --diagnostics .../diagnostics.json: exit status 1", use --debug to enable logging .../.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks') Explanation: 'databricks-pydabs' library is not installed in the Python environment. If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, and that the wheel is installed in the Python environment: $ .venv/bin/pip install -e . If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: pydabs: venv_path: .venv ``` ## Tests Unit tests --- .../config/mutator/python/python_mutator.go | 49 +++++++++++++++++-- .../mutator/python/python_mutator_test.go | 24 +++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index bf30f070..fbf3b7e0 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -1,15 +1,21 @@ package python import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" - "github.com/databricks/cli/libs/python" "github.com/databricks/databricks-sdk-go/logger" + "github.com/fatih/color" + + "strings" + + "github.com/databricks/cli/libs/python" "github.com/databricks/cli/bundle/env" @@ -169,7 +175,11 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r return dyn.InvalidValue, diag.Errorf("failed to write input file: %s", err) } - stderrWriter := newLogWriter(ctx, "stderr: ") + stderrBuf := bytes.Buffer{} + stderrWriter := io.MultiWriter( + newLogWriter(ctx, "stderr: "), + &stderrBuf, + ) stdoutWriter := newLogWriter(ctx, "stdout: ") _, processErr := process.Background( @@ -197,7 +207,13 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r // process can fail without reporting errors in diagnostics file or creating it, for instance, // venv doesn't have PyDABs library installed if processErr != nil { - return dyn.InvalidValue, diag.Errorf("python mutator process failed: %sw, use --debug to enable logging", processErr) + diagnostic := diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("python mutator process failed: %q, use --debug to enable logging", processErr), + Detail: explainProcessErr(stderrBuf.String()), + } + + return dyn.InvalidValue, diag.Diagnostics{diagnostic} } // or we can fail to read diagnostics file, that should always be created @@ -212,6 +228,33 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r return output, pythonDiagnostics } +const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +and that the wheel is installed in the Python environment: + + $ .venv/bin/pip install -e . + +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +or activate the environment before running CLI commands: + + experimental: + pydabs: + venv_path: .venv +` + +// explainProcessErr provides additional explanation for common errors. +// It's meant to be the best effort, and not all errors are covered. +// Output should be used only used for error reporting. +func explainProcessErr(stderr string) string { + // implemented in cpython/Lib/runpy.py and portable across Python 3.x, including pypy + if strings.Contains(stderr, "Error while finding module specification for 'databricks.bundles.build'") { + summary := color.CyanString("Explanation: ") + "'databricks-pydabs' library is not installed in the Python environment.\n" + + return stderr + "\n" + summary + "\n" + installExplanation + } + + return stderr +} + func writeInputFile(inputPath string, input dyn.Value) error { // we need to marshal dyn.Value instead of bundle.Config to JSON to support // non-string fields assigned with bundle variables diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 26350d88..bf12b249 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -564,6 +564,30 @@ func TestStrictNormalize(t *testing.T) { assert.True(t, strictDiags.HasError()) } +func TestExplainProcessErr(t *testing.T) { + stderr := "/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks')\n" + expected := `/home/test/.venv/bin/python3: Error while finding module specification for 'databricks.bundles.build' (ModuleNotFoundError: No module named 'databricks') + +Explanation: 'databricks-pydabs' library is not installed in the Python environment. + +If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +and that the wheel is installed in the Python environment: + + $ .venv/bin/pip install -e . + +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +or activate the environment before running CLI commands: + + experimental: + pydabs: + venv_path: .venv +` + + out := explainProcessErr(stderr) + + assert.Equal(t, expected, out) +} + func withProcessStub(t *testing.T, args []string, output string, diagnostics string) context.Context { ctx := context.Background() ctx, stub := process.WithStub(ctx) From 0ce7be8ff07dcf1f57d41b620b209f73e20df118 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:39:16 +0200 Subject: [PATCH 17/78] Bump github.com/Masterminds/semver/v3 from 3.2.1 to 3.3.0 (#1741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/Masterminds/semver/v3](https://github.com/Masterminds/semver) from 3.2.1 to 3.3.0.
Release notes

Sourced from github.com/Masterminds/semver/v3's releases.

v3.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/Masterminds/semver/compare/v3.2.1...v3.3.0

Changelog

Sourced from github.com/Masterminds/semver/v3's changelog.

3.3.0 (2024-08-27)

Added

Changed

  • #241: Simplify StrictNewVersion parsing (thanks @​grosser)
  • Testing support up through Go 1.23
  • Minimum version set to 1.21 as this is what's tested now
  • Fuzz testing now supports caching
Commits
  • e6e3d4d Merge pull request #249 from mattfarina/update-changelog-3.3.0
  • e80c4ea Updating changelog for 3.3.0
  • 80427ad Merge pull request #248 from mattfarina/bump-min-version
  • b610837 bumping min version in go.mod based on what's tested
  • a4cccd8 Merge pull request #246 from mattfarina/bump-go-1.23
  • 7c178cf Updating the testing version of Go used
  • 29f94c1 Merge pull request #241 from grosser/grosser/validate
  • 2cf1b16 Merge pull request #245 from mattfarina/remove-vert
  • b55476a Removing reference to vert
  • d07450b simplify StrictNewVersion
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/Masterminds/semver/v3&package-manager=go_modules&previous-version=3.2.1&new-version=3.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4aa27992..9777106c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/databricks/cli go 1.22 require ( - github.com/Masterminds/semver/v3 v3.2.1 // MIT + github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 github.com/databricks/databricks-sdk-go v0.45.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT diff --git a/go.sum b/go.sum index 2e58948a..b232e8e4 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1h dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg= From 096123674afb6c593f2c242c3210fa5735ed98cb Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 2 Sep 2024 19:13:17 +0530 Subject: [PATCH 18/78] Fix streaming of stdout, stdin, stderr in cobra test runner (#1742) ## Changes We were not using the readers and writers set in the test fixtures in the progress logger. This PR fixes that. It also modifies `TestAccAbortBind`, which was implicitly relying on the bug. I encountered this bug while working on https://github.com/databricks/cli/pull/1672. ## Tests Manually. From non-tty: ``` Error: failed to bind the resource, err: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. ``` From tty, bind works as expected. ``` Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'. [y/n]: y Updating deployment state... Successfully bound databricks_pipeline with an id '9d2dedbb-f522-4503-96ba-4bc4d5bfa77d'. Run 'bundle deploy' to deploy changes to your workspace ``` --- bundle/deploy/terraform/import.go | 5 +++++ cmd/root/progress_logger.go | 6 ++++++ internal/bundle/bind_resource_test.go | 8 ++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/bundle/deploy/terraform/import.go b/bundle/deploy/terraform/import.go index 7c1a6815..dfe60a58 100644 --- a/bundle/deploy/terraform/import.go +++ b/bundle/deploy/terraform/import.go @@ -69,6 +69,11 @@ func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn // Remove output starting from Warning until end of output output = output[:bytes.Index([]byte(output), []byte("Warning:"))] cmdio.LogString(ctx, output) + + if !cmdio.IsPromptSupported(ctx) { + return diag.Errorf("This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed.") + } + ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") if err != nil { return diag.FromErr(err) diff --git a/cmd/root/progress_logger.go b/cmd/root/progress_logger.go index c05ecb04..7d6a1fa4 100644 --- a/cmd/root/progress_logger.go +++ b/cmd/root/progress_logger.go @@ -29,6 +29,12 @@ func (f *progressLoggerFlag) resolveModeDefault(format flags.ProgressLogFormat) } func (f *progressLoggerFlag) initializeContext(ctx context.Context) (context.Context, error) { + // No need to initialize the logger if it's already set in the context. This + // happens in unit tests where the logger is setup as a fixture. + if _, ok := cmdio.FromContext(ctx); ok { + return ctx, nil + } + if f.log.level.String() != "disabled" && f.log.file.String() == "stderr" && f.ProgressLogFormat == flags.ModeInplace { return nil, fmt.Errorf("inplace progress logging cannot be used when log-file is stderr") diff --git a/internal/bundle/bind_resource_test.go b/internal/bundle/bind_resource_test.go index d44ad2c3..2449c31f 100644 --- a/internal/bundle/bind_resource_test.go +++ b/internal/bundle/bind_resource_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -101,12 +102,15 @@ func TestAccAbortBind(t *testing.T) { destroyBundle(t, ctx, bundleRoot) }) + // Bind should fail because prompting is not possible. t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") c := internal.NewCobraTestRunner(t, "bundle", "deployment", "bind", "foo", fmt.Sprint(jobId)) - // Simulate user aborting the bind. This is done by not providing any input to the prompt in non-interactive mode. + // Expect error suggesting to use --auto-approve _, _, err = c.Run() - require.ErrorContains(t, err, "failed to bind the resource") + assert.ErrorContains(t, err, "failed to bind the resource") + assert.ErrorContains(t, err, "This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") err = deployBundle(t, ctx, bundleRoot) require.NoError(t, err) From 072fa812e28c32d374afdb5409e7320f1c5894c7 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Tue, 3 Sep 2024 09:51:54 +0200 Subject: [PATCH 19/78] Include a permissions section in all templates (#1713) ## Changes This updates the templates to include a `permissions` section. Having a permissions section is a best practice, is helpful to understand the notion of permissions, and helps diagnose permission errors (https://github.com/databricks/cli/pull/1386). This is a cherry-pick from https://github.com/databricks/cli/pull/1387. This change was verified to work both in dev and prod. Existing unit tests validate the validity of the templates in these modes. --- .../{{.project_name}}/databricks.yml.tmpl | 17 ++++---- .../{{.project_name}}/databricks.yml.tmpl | 36 ++++------------- .../{{.project_name}}/databricks.yml.tmpl | 39 ++++++------------- 3 files changed, 28 insertions(+), 64 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index fdda03c0..f96ce4fe 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -12,8 +12,10 @@ include: targets: dev: default: true - # We use 'mode: development' to indicate this is a personal development copy. - # Any job schedules and triggers are paused by default. + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. mode: development workspace: host: {{workspace_host}} @@ -22,11 +24,10 @@ targets: mode: production workspace: host: {{workspace_host}} - # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy. + # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} - {{- if not is_service_principal}} + permissions: + - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} + level: CAN_MANAGE run_as: - # This runs as {{user_name}} in production. We could also use a service principal here - # using service_principal_name (see the Databricks documentation). - user_name: {{user_name}} - {{- end}} + {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl index e3572326..8544dc83 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl @@ -7,44 +7,24 @@ include: - resources/*.yml targets: - # The 'dev' target, for development purposes. This target is the default. dev: - # We use 'mode: development' to indicate this is a personal development copy: + # The default target uses 'mode: development' to create a development copy. # - Deployed resources get prefixed with '[dev my_user_name]' - # - Any job schedules and triggers are paused by default - # - The 'development' mode is used for Delta Live Tables pipelines + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. mode: development default: true workspace: host: {{workspace_host}} - ## Optionally, there could be a 'staging' target here. - ## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/ci-cd.html.) - # - # staging: - # workspace: - # host: {{workspace_host}} - - # The 'prod' target, used for production deployment. prod: - # We use 'mode: production' to indicate this is a production deployment. - # Doing so enables strict verification of the settings below. mode: production workspace: host: {{workspace_host}} - # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy. - {{- /* - Internal note 2023-12: CLI versions v0.211.0 and before would show an error when using `mode: production` - with a path that doesn't say "/Shared". For now, we'll include an extra comment in the template - to explain that customers should update if they see this. - */}} - # If this path results in an error, please make sure you have a recent version of the CLI installed. + # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + permissions: + - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} + level: CAN_MANAGE run_as: - {{- if is_service_principal}} - service_principal_name: {{user_name}} - {{- else}} - # This runs as {{user_name}} in production. We could also use a service principal here, - # see https://docs.databricks.com/dev-tools/bundles/permissions.html. - user_name: {{user_name}} - {{- end}} + {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index a47fb7c1..55c1aae4 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -18,16 +18,16 @@ variables: {{- $dev_schema := .shared_schema }} {{- $prod_schema := .shared_schema }} {{- if (regexp "^yes").MatchString .personal_schemas}} -{{- $dev_schema = "${workspace.current_user.short_name}"}} -{{- $prod_schema = "default"}} + {{- $dev_schema = "${workspace.current_user.short_name}"}} + {{- $prod_schema = "default"}} {{- end}} -# Deployment targets. targets: - # The 'dev' target, for development purposes. This target is the default. dev: - # We use 'mode: development' to indicate this is a personal development copy. - # Any job schedules and triggers are paused by default + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. mode: development default: true workspace: @@ -37,35 +37,18 @@ targets: catalog: {{.default_catalog}} schema: {{$dev_schema}} - ## Optionally, there could be a 'staging' target here. - ## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/ci-cd.html.) - # - # staging: - # workspace: - # host: {{workspace_host}} - - # The 'prod' target, used for production deployment. prod: - # We use 'mode: production' to indicate this is a production deployment. - # Doing so enables strict verification of the settings below. mode: production workspace: host: {{workspace_host}} - # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy. - {{- /* - Internal note 2023-12: CLI versions v0.211.0 and before would show an error when using `mode: production` - with a path that doesn't say "/Shared". For now, we'll include an extra comment in the template - to explain that customers should update if they see this. - */}} - # If this path results in an error, please make sure you have a recent version of the CLI installed. + # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}} catalog: {{.default_catalog}} schema: {{$prod_schema}} - {{- if not is_service_principal}} + permissions: + - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} + level: CAN_MANAGE run_as: - # This runs as {{user_name}} in production. We could also use a service principal here - # using service_principal_name (see https://docs.databricks.com/en/dev-tools/bundles/permissions.html). - user_name: {{user_name}} - {{end -}} + {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} From 35cdf4010dad156902d8d8b20b5fc0fe01de6acc Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:44:21 +0530 Subject: [PATCH 20/78] Do not error if we cannot prompt for a profile in `auth login` (#1745) ## Changes With https://github.com/databricks/cli/pull/1370 we started to error if a profile name was not provided in a non-tty setting. The Databricks VSCode extension, however, uses the `auth login` command to simply refresh the tokens. Thus, this PR is a regression fix for that use case. ## Tests Manually, `databricks auth login --host https://e2-dogfood.staging.cloud.databricks.com` no longer errors. Instead it successfully refreshes the credentials. --- cmd/auth/login.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/auth/login.go b/cmd/auth/login.go index f87a2a02..79b79546 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -19,7 +19,7 @@ import ( func promptForProfile(ctx context.Context, defaultValue string) (string, error) { if !cmdio.IsInTTY(ctx) { - return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a profile using --profile") + return "", nil } prompt := cmdio.Prompt(ctx) From a27c24a397f91048677c38e48ee31213f278f640 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:41:47 +0530 Subject: [PATCH 21/78] Add prompt when a pipeline recreation happens (#1672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes DLT pipeline recreations are destructive. They can lead to lost history of previous updates, outage of the tables temporarily and are potentially computationally expensive. Thus we make a breaking change where a prompt is shown to the user if there configuration changes will lead to a DLT recreation. Users can skip the prompt by specifying the `--auto-approve` flag. This PR also fixes an issue with our test runner where logs from the cmdio.Logger would not get propagated to the reader returned by our cobra test runner. ## Tests Manually, and new unit and integration tests. ``` ➜ bundle-playground-3 cli bundle deploy Uploading bundle files to /Users/63ec021d-b0c6-49c0-93a0-5123953a1cb2/.bundle/test/development/files... The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation: recreate pipeline foo Would you like to proceed? [y/n]: n Deployment cancelled! ``` --- bundle/phases/deploy.go | 96 +++++++++++++------ bundle/phases/deploy_test.go | 67 +++++++++++++ .../databricks_template_schema.json | 8 ++ .../template/databricks.yml.tmpl | 25 +++++ .../bundles/recreate_pipeline/template/nb.sql | 2 + internal/bundle/deploy_test.go | 91 +++++++++++++++++- 6 files changed, 260 insertions(+), 29 deletions(-) create mode 100644 bundle/phases/deploy_test.go create mode 100644 internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json create mode 100644 internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/recreate_pipeline/template/nb.sql diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index ca967c32..49544227 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -19,9 +19,38 @@ import ( "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/libs/cmdio" terraformlib "github.com/databricks/cli/libs/terraform" + tfjson "github.com/hashicorp/terraform-json" ) -func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { +func parseTerraformActions(changes []*tfjson.ResourceChange, toInclude func(typ string, actions tfjson.Actions) bool) []terraformlib.Action { + res := make([]terraformlib.Action, 0) + for _, rc := range changes { + if !toInclude(rc.Type, rc.Change.Actions) { + continue + } + + var actionType terraformlib.ActionType + switch { + case rc.Change.Actions.Delete(): + actionType = terraformlib.ActionTypeDelete + case rc.Change.Actions.Replace(): + actionType = terraformlib.ActionTypeRecreate + default: + // No use case for other action types yet. + continue + } + + res = append(res, terraformlib.Action{ + Action: actionType, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + + return res +} + +func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { tf := b.Terraform if tf == nil { return false, fmt.Errorf("terraform not initialized") @@ -33,41 +62,52 @@ func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, err return false, err } - actions := make([]terraformlib.Action, 0) - for _, rc := range plan.ResourceChanges { - // We only care about destructive actions on UC schema resources. - if rc.Type != "databricks_schema" { - continue + schemaActions := parseTerraformActions(plan.ResourceChanges, func(typ string, actions tfjson.Actions) bool { + // Filter in only UC schema resources. + if typ != "databricks_schema" { + return false } - var actionType terraformlib.ActionType + // We only display prompts for destructive actions like deleting or + // recreating a schema. + return actions.Delete() || actions.Replace() + }) - switch { - case rc.Change.Actions.Delete(): - actionType = terraformlib.ActionTypeDelete - case rc.Change.Actions.Replace(): - actionType = terraformlib.ActionTypeRecreate - default: - // We don't need a prompt for non-destructive actions like creating - // or updating a schema. - continue + dltActions := parseTerraformActions(plan.ResourceChanges, func(typ string, actions tfjson.Actions) bool { + // Filter in only DLT pipeline resources. + if typ != "databricks_pipeline" { + return false } - actions = append(actions, terraformlib.Action{ - Action: actionType, - ResourceType: rc.Type, - ResourceName: rc.Name, - }) - } + // Recreating DLT pipeline leads to metadata loss and for a transient period + // the underling tables will be unavailable. + return actions.Replace() || actions.Delete() + }) - // No restricted actions planned. No need for approval. - if len(actions) == 0 { + // We don't need to display any prompts in this case. + if len(dltActions) == 0 && len(schemaActions) == 0 { return true, nil } - cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") - for _, action := range actions { - cmdio.Log(ctx, action) + // One or more UC schema resources will be deleted or recreated. + if len(schemaActions) != 0 { + cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") + for _, action := range schemaActions { + cmdio.Log(ctx, action) + } + } + + // One or more DLT pipelines is being recreated. + if len(dltActions) != 0 { + msg := ` +This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed:` + cmdio.LogString(ctx, msg) + for _, action := range dltActions { + cmdio.Log(ctx, action) + } } if b.AutoApprove { @@ -126,7 +166,7 @@ func Deploy() bundle.Mutator { terraform.CheckRunningResource(), terraform.Plan(terraform.PlanGoal("deploy")), bundle.If( - approvalForUcSchemaDelete, + approvalForDeploy, deployCore, bundle.LogString("Deployment cancelled!"), ), diff --git a/bundle/phases/deploy_test.go b/bundle/phases/deploy_test.go new file mode 100644 index 00000000..e00370b3 --- /dev/null +++ b/bundle/phases/deploy_test.go @@ -0,0 +1,67 @@ +package phases + +import ( + "testing" + + terraformlib "github.com/databricks/cli/libs/terraform" + tfjson "github.com/hashicorp/terraform-json" + "github.com/stretchr/testify/assert" +) + +func TestParseTerraformActions(t *testing.T) { + changes := []*tfjson.ResourceChange{ + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionCreate}, + }, + Name: "create pipeline", + }, + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete}, + }, + Name: "delete pipeline", + }, + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}, + }, + Name: "recreate pipeline", + }, + { + Type: "databricks_whatever", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}, + }, + Name: "recreate whatever", + }, + } + + res := parseTerraformActions(changes, func(typ string, actions tfjson.Actions) bool { + if typ != "databricks_pipeline" { + return false + } + + if actions.Delete() || actions.Replace() { + return true + } + + return false + }) + + assert.Equal(t, []terraformlib.Action{ + { + Action: terraformlib.ActionTypeDelete, + ResourceType: "databricks_pipeline", + ResourceName: "delete pipeline", + }, + { + Action: terraformlib.ActionTypeRecreate, + ResourceType: "databricks_pipeline", + ResourceName: "recreate pipeline", + }, + }, res) +} diff --git a/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json b/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json new file mode 100644 index 00000000..762f4470 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema and pipeline names" + } + } +} diff --git a/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl b/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl new file mode 100644 index 00000000..10350f13 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: "bundle-playground" + +variables: + catalog: + description: The catalog the DLT pipeline should use. + default: main + + +resources: + pipelines: + foo: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: ${var.catalog} + +include: + - "*.yml" + +targets: + development: + default: true diff --git a/internal/bundle/bundles/recreate_pipeline/template/nb.sql b/internal/bundle/bundles/recreate_pipeline/template/nb.sql new file mode 100644 index 00000000..199ff507 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/template/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 269b7c80..736c880d 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -120,8 +120,97 @@ func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { t.Setenv("BUNDLE_ROOT", bundleRoot) t.Setenv("TERM", "dumb") c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") - stdout, _, err := c.Run() + stdout, stderr, err := c.Run() + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), "The following UC schemas will be deleted or recreated. Any underlying data may be lost:\n delete schema bar") + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") +} + +func TestAccBundlePipelineDeleteWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + // deploy pipeline + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + // assert pipeline is created + pipelineName := "test-bundle-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + assert.Equal(t, pipeline.Name, pipelineName) + + // assert job is created + jobName := "test-bundle-job-" + uniqueId + job, err := w.Jobs.GetBySettingsName(ctx, jobName) + require.NoError(t, err) + assert.Equal(t, job.Settings.Name, jobName) + + // delete resources.yml + err = os.Remove(filepath.Join(bundleRoot, "resources.yml")) + require.NoError(t, err) + + // Redeploy the bundle. Expect it to fail because deleting the pipeline requires --auto-approve. + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + stdout, stderr, err := c.Run() + + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + delete pipeline bar`) + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") + +} + +func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + uniqueId := uuid.New().String() + + bundleRoot, err := initTestTemplate(t, ctx, "recreate_pipeline", map[string]any{ + "unique_id": uniqueId, + }) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + // Assert the pipeline is created + pipelineName := "test-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + require.Equal(t, pipelineName, pipeline.Name) + + // Redeploy the bundle, pointing the DLT pipeline to a different UC catalog. + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--var=\"catalog=whatever\"") + stdout, stderr, err := c.Run() + + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate pipeline foo`) assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } From ca6332a5a4325aff1be848536f45d13bd74d93b3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 4 Sep 2024 13:24:55 +0200 Subject: [PATCH 22/78] Fixed complex variables are not being correctly merged from include files (#1746) ## Changes Fixes an `Error: no value assigned to required variable .` when the main complex variable definition is defined in one file but target override is defined in separate file which is included in the main one. ## Tests Added regression test --- bundle/config/root.go | 12 +++++++++++- bundle/tests/complex_variables_test.go | 19 +++++++++++++++++++ .../complex_multiple_files/databricks.yml | 17 +++++++++++++++++ .../variables/clusters.yml | 11 +++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 bundle/tests/variables/complex_multiple_files/databricks.yml create mode 100644 bundle/tests/variables/complex_multiple_files/variables/clusters.yml diff --git a/bundle/config/root.go b/bundle/config/root.go index 86dc3392..281c5c2a 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -433,10 +433,20 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { }, variable.Locations()), nil case dyn.KindMap, dyn.KindSequence: + lookup, err := dyn.Get(variable, "lookup") + // If lookup is set, we don't want to rewrite the variable and return it as is. + if err == nil && lookup.Kind() != dyn.KindInvalid { + return variable, nil + } + // Check if the original definition of variable has a type field. + // Type might not be found if the variable overriden in a separate file + // and configuration is not merged yet. typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type"))) if err != nil { - return variable, nil + return dyn.NewValue(map[string]dyn.Value{ + "default": variable, + }, variable.Locations()), nil } if typeV.MustString() == "complex" { diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index 1badea6d..d46d8d8c 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -68,3 +68,22 @@ func TestComplexVariablesOverride(t *testing.T) { require.Equal(t, "", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.random"]) require.Equal(t, "", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.PolicyId) } + +func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) { + b, diags := loadTargetWithDiags("variables/complex_multiple_files", "dev") + require.Empty(t, diags) + + diags = bundle.Apply(context.Background(), b, bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferencesInComplexVariables(), + mutator.ResolveVariableReferences( + "variables", + ), + )) + require.NoError(t, diags.Error()) + + require.Equal(t, "14.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion) + require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId) + require.Equal(t, 4, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers) + require.Equal(t, "false", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"]) +} diff --git a/bundle/tests/variables/complex_multiple_files/databricks.yml b/bundle/tests/variables/complex_multiple_files/databricks.yml new file mode 100644 index 00000000..cb5d2439 --- /dev/null +++ b/bundle/tests/variables/complex_multiple_files/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: complex-variables-multiple-files + +resources: + jobs: + my_job: + job_clusters: + - job_cluster_key: key + new_cluster: ${var.cluster} + +variables: + cluster: + type: complex + description: "A cluster definition" + +include: + - ./variables/*.yml diff --git a/bundle/tests/variables/complex_multiple_files/variables/clusters.yml b/bundle/tests/variables/complex_multiple_files/variables/clusters.yml new file mode 100644 index 00000000..badd4516 --- /dev/null +++ b/bundle/tests/variables/complex_multiple_files/variables/clusters.yml @@ -0,0 +1,11 @@ +targets: + default: + dev: + variables: + cluster: + spark_version: "14.2.x-scala2.11" + node_type_id: "Standard_DS3_v2" + num_workers: 4 + spark_conf: + spark.speculation: false + spark.databricks.delta.retentionDurationCheck.enabled: false From 72030844c52f9499b10e7158544a80fde56451f3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 4 Sep 2024 19:16:40 +0200 Subject: [PATCH 23/78] Fixed variable override in target with full variable syntax (#1749) ## Changes This PR makes sure that both of this override syntax for variables work correctly ``` targets: dev: variables: cluster1: spark_version: "14.2.x-scala2.11" node_type_id: "Standard_DS3_v2" num_workers: 4 spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false cluster2: default: spark_version: "14.2.x-scala2.11" node_type_id: "Standard_DS3_v2" num_workers: 4 spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false ``` ## Tests Added regression test --------- Co-authored-by: Pieter Noordhuis --- bundle/config/root.go | 43 ++++++++++++++----- bundle/tests/complex_variables_test.go | 11 ++--- .../complex_multiple_files/databricks.yml | 43 +++++++++++++++++-- .../variables/clusters.yml | 10 ++++- 4 files changed, 86 insertions(+), 21 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 281c5c2a..46578769 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -406,6 +406,30 @@ func (r *Root) MergeTargetOverrides(name string) error { return r.updateWithDynamicValue(root) } +var variableKeywords = []string{"default", "lookup"} + +// isFullVariableOverrideDef checks if the given value is a full syntax varaible override. +// A full syntax variable override is a map with only one of the following +// keys: "default", "lookup". +func isFullVariableOverrideDef(v dyn.Value) bool { + mv, ok := v.AsMap() + if !ok { + return false + } + + if mv.Len() != 1 { + return false + } + + for _, keyword := range variableKeywords { + if _, ok := mv.GetByString(keyword); ok { + return true + } + } + + return false +} + // rewriteShorthands performs lightweight rewriting of the configuration // tree where we allow users to write a shorthand and must rewrite to the full form. func rewriteShorthands(v dyn.Value) (dyn.Value, error) { @@ -433,30 +457,27 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { }, variable.Locations()), nil case dyn.KindMap, dyn.KindSequence: - lookup, err := dyn.Get(variable, "lookup") - // If lookup is set, we don't want to rewrite the variable and return it as is. - if err == nil && lookup.Kind() != dyn.KindInvalid { + // If it's a full variable definition, leave it as is. + if isFullVariableOverrideDef(variable) { return variable, nil } // Check if the original definition of variable has a type field. + // If it has a type field, it means the shorthand is a value of a complex type. // Type might not be found if the variable overriden in a separate file // and configuration is not merged yet. typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type"))) - if err != nil { - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Locations()), nil - } - - if typeV.MustString() == "complex" { + if err == nil && typeV.MustString() == "complex" { return dyn.NewValue(map[string]dyn.Value{ "type": typeV, "default": variable, }, variable.Locations()), nil } - return variable, nil + // If it's a shorthand, rewrite it to a full variable definition. + return dyn.NewValue(map[string]dyn.Value{ + "default": variable, + }, variable.Locations()), nil default: return variable, nil diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index d46d8d8c..6371071c 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -81,9 +81,10 @@ func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) { ), )) require.NoError(t, diags.Error()) - - require.Equal(t, "14.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion) - require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId) - require.Equal(t, 4, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers) - require.Equal(t, "false", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"]) + for _, cluster := range b.Config.Resources.Jobs["my_job"].JobClusters { + require.Equalf(t, "14.2.x-scala2.11", cluster.NewCluster.SparkVersion, "cluster: %v", cluster.JobClusterKey) + require.Equalf(t, "Standard_DS3_v2", cluster.NewCluster.NodeTypeId, "cluster: %v", cluster.JobClusterKey) + require.Equalf(t, 4, cluster.NewCluster.NumWorkers, "cluster: %v", cluster.JobClusterKey) + require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", cluster.JobClusterKey) + } } diff --git a/bundle/tests/variables/complex_multiple_files/databricks.yml b/bundle/tests/variables/complex_multiple_files/databricks.yml index cb5d2439..42a82c61 100644 --- a/bundle/tests/variables/complex_multiple_files/databricks.yml +++ b/bundle/tests/variables/complex_multiple_files/databricks.yml @@ -5,13 +5,48 @@ resources: jobs: my_job: job_clusters: - - job_cluster_key: key - new_cluster: ${var.cluster} - + - job_cluster_key: key1 + new_cluster: ${var.cluster1} + - job_cluster_key: key2 + new_cluster: ${var.cluster2} + - job_cluster_key: key3 + new_cluster: ${var.cluster3} + - job_cluster_key: key4 + new_cluster: ${var.cluster4} variables: - cluster: + cluster1: + type: complex + description: "A cluster definition" + cluster2: + type: complex + description: "A cluster definition" + cluster3: + type: complex + description: "A cluster definition" + cluster4: type: complex description: "A cluster definition" include: - ./variables/*.yml + + +targets: + default: + dev: + variables: + cluster3: + spark_version: "14.2.x-scala2.11" + node_type_id: "Standard_DS3_v2" + num_workers: 4 + spark_conf: + spark.speculation: false + spark.databricks.delta.retentionDurationCheck.enabled: false + cluster4: + default: + spark_version: "14.2.x-scala2.11" + node_type_id: "Standard_DS3_v2" + num_workers: 4 + spark_conf: + spark.speculation: false + spark.databricks.delta.retentionDurationCheck.enabled: false diff --git a/bundle/tests/variables/complex_multiple_files/variables/clusters.yml b/bundle/tests/variables/complex_multiple_files/variables/clusters.yml index badd4516..0186c437 100644 --- a/bundle/tests/variables/complex_multiple_files/variables/clusters.yml +++ b/bundle/tests/variables/complex_multiple_files/variables/clusters.yml @@ -2,10 +2,18 @@ targets: default: dev: variables: - cluster: + cluster1: spark_version: "14.2.x-scala2.11" node_type_id: "Standard_DS3_v2" num_workers: 4 spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false + cluster2: + default: + spark_version: "14.2.x-scala2.11" + node_type_id: "Standard_DS3_v2" + num_workers: 4 + spark_conf: + spark.speculation: false + spark.databricks.delta.retentionDurationCheck.enabled: false From f71d9e76499f9296daad620f6efed651b9864679 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 5 Sep 2024 10:56:52 +0200 Subject: [PATCH 24/78] [Release] Release v0.228.0 (#1752) CLI: * Do not error if we cannot prompt for a profile in `auth login` ([#1745](https://github.com/databricks/cli/pull/1745)). Bundles: As of this release CLI will show a prompt is if there are configuration changes which will lead to a DLT recreation. Users can skip the prompt by specifying the `--auto-approve` flag * Pass along $AZURE_CONFIG_FILE to Terraform process ([#1734](https://github.com/databricks/cli/pull/1734)). * Add prompt when a pipeline recreation happens ([#1672](https://github.com/databricks/cli/pull/1672)). * Use materialized views in the default-sql template ([#1709](https://github.com/databricks/cli/pull/1709)). * Update templates to latest LTS DBR ([#1715](https://github.com/databricks/cli/pull/1715)). * Make lock optional in the JSON schema ([#1738](https://github.com/databricks/cli/pull/1738)). * Do not suppress normalisation diagnostics for resolving variables ([#1740](https://github.com/databricks/cli/pull/1740)). * Include a permissions section in all templates ([#1713](https://github.com/databricks/cli/pull/1713)). * Fixed complex variables are not being correctly merged from include files ([#1746](https://github.com/databricks/cli/pull/1746)). * Fixed variable override in target with full variable syntax ([#1749](https://github.com/databricks/cli/pull/1749)). Internal: * Consider serverless clusters as compatible for Python wheel tasks ([#1733](https://github.com/databricks/cli/pull/1733)). * PythonMutator: explain missing package error ([#1736](https://github.com/databricks/cli/pull/1736)). * Add `dyn.Time` to box a timestamp with its original string value ([#1732](https://github.com/databricks/cli/pull/1732)). * Fix streaming of stdout, stdin, stderr in cobra test runner ([#1742](https://github.com/databricks/cli/pull/1742)). Dependency updates: * Bump github.com/Masterminds/semver/v3 from 3.2.1 to 3.3.0 ([#1741](https://github.com/databricks/cli/pull/1741)). --------- Co-authored-by: Pieter Noordhuis --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fac7d597..d6383125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Version changelog +## [Release] Release v0.228.0 + +CLI: + * Do not error if we cannot prompt for a profile in `auth login` ([#1745](https://github.com/databricks/cli/pull/1745)). + +Bundles: + +As of this release, the CLI will show a prompt if there are configuration changes that lead to DLT pipeline recreation. +Users can skip the prompt by specifying the `--auto-approve` flag. + + * Pass along to Terraform process ([#1734](https://github.com/databricks/cli/pull/1734)). + * Add prompt when a pipeline recreation happens ([#1672](https://github.com/databricks/cli/pull/1672)). + * Use materialized views in the default-sql template ([#1709](https://github.com/databricks/cli/pull/1709)). + * Update templates to latest LTS DBR ([#1715](https://github.com/databricks/cli/pull/1715)). + * Make lock optional in the JSON schema ([#1738](https://github.com/databricks/cli/pull/1738)). + * Do not suppress normalisation diagnostics for resolving variables ([#1740](https://github.com/databricks/cli/pull/1740)). + * Include a permissions section in all templates ([#1713](https://github.com/databricks/cli/pull/1713)). + * Fixed complex variables are not being correctly merged from include files ([#1746](https://github.com/databricks/cli/pull/1746)). + * Fixed variable override in target with full variable syntax ([#1749](https://github.com/databricks/cli/pull/1749)). + +Internal: + * Consider serverless clusters as compatible for Python wheel tasks ([#1733](https://github.com/databricks/cli/pull/1733)). + * PythonMutator: explain missing package error ([#1736](https://github.com/databricks/cli/pull/1736)). + * Add `dyn.Time` to box a timestamp with its original string value ([#1732](https://github.com/databricks/cli/pull/1732)). + * Fix streaming of stdout, stdin, stderr in cobra test runner ([#1742](https://github.com/databricks/cli/pull/1742)). + +Dependency updates: + * Bump github.com/Masterminds/semver/v3 from 3.2.1 to 3.3.0 ([#1741](https://github.com/databricks/cli/pull/1741)). + ## [Release] Release v0.227.1 CLI: From ceefa80d7278eeef9cce500b2c69453da2040f40 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 5 Sep 2024 13:05:16 +0200 Subject: [PATCH 25/78] Pass copy of `dyn.Path` to callback function (#1747) ## Changes Some call sites hold on to the `dyn.Path` provided to them by the callback. It must therefore never be mutated after the callback returns, or these mutations leak out into unknown scope. This change means it is no longer possible for this failure mode to happen. ## Tests Unit test. --- bundle/artifacts/expand_globs.go | 7 +--- .../config/validate/unique_resource_keys.go | 6 +--- bundle/libraries/expand_glob_references.go | 8 ++--- bundle/libraries/upload.go | 2 +- libs/dyn/visit.go | 2 +- libs/dyn/visit_map.go | 4 +-- libs/dyn/visit_test.go | 36 +++++++++++++++++++ 7 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 libs/dyn/visit_test.go diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 61744405..cdf3d459 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -33,12 +33,7 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { Severity: diag.Error, Summary: fmt.Sprintf("%s: %s", source, message), Locations: []dyn.Location{v.Location()}, - - Paths: []dyn.Path{ - // Hack to clone the path. This path copy is mutable. - // To be addressed in a later PR. - p.Append(), - }, + Paths: []dyn.Path{p}, } } diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index d6212b0a..50295375 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -3,7 +3,6 @@ package validate import ( "context" "fmt" - "slices" "sort" "github.com/databricks/cli/bundle" @@ -66,10 +65,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - // dyn.Path under the hood is a slice. The code that walks the configuration - // tree uses the same underlying slice to track the path as it walks - // the tree. So, we need to clone it here. - m.paths = append(m.paths, slices.Clone(p)) + m.paths = append(m.paths, p) m.locations = append(m.locations, v.Locations()...) resourceMetadata[k] = m diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 9e90a2a1..9322a06b 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -16,12 +16,10 @@ type expand struct { func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Error, - Summary: message, - Paths: []dyn.Path{ - p.Append(), - }, + Severity: diag.Error, + Summary: message, Locations: l, + Paths: []dyn.Path{p}, } } diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index be7cc41d..224e7ab2 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -76,7 +76,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error source = filepath.Join(b.RootPath, source) libs[source] = append(libs[source], configLocation{ - configPath: p.Append(), // Hack to get the copy of path + configPath: p, location: v.Location(), }) diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go index 4d3cf501..38adec24 100644 --- a/libs/dyn/visit.go +++ b/libs/dyn/visit.go @@ -70,7 +70,7 @@ type visitOptions struct { func visit(v Value, prefix Path, suffix Pattern, opts visitOptions) (Value, error) { if len(suffix) == 0 { - return opts.fn(prefix, v) + return opts.fn(slices.Clone(prefix), v) } // Initialize prefix if it is empty. diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go index cd2cd483..3f0cded0 100644 --- a/libs/dyn/visit_map.go +++ b/libs/dyn/visit_map.go @@ -21,7 +21,7 @@ func Foreach(fn MapFunc) MapFunc { for _, pair := range m.Pairs() { pk := pair.Key pv := pair.Value - nv, err := fn(append(p, Key(pk.MustString())), pv) + nv, err := fn(p.Append(Key(pk.MustString())), pv) if err != nil { return InvalidValue, err } @@ -32,7 +32,7 @@ func Foreach(fn MapFunc) MapFunc { s := slices.Clone(v.MustSequence()) for i, value := range s { var err error - s[i], err = fn(append(p, Index(i)), value) + s[i], err = fn(p.Append(Index(i)), value) if err != nil { return InvalidValue, err } diff --git a/libs/dyn/visit_test.go b/libs/dyn/visit_test.go new file mode 100644 index 00000000..5b61399b --- /dev/null +++ b/libs/dyn/visit_test.go @@ -0,0 +1,36 @@ +package dyn_test + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestVisitCallbackPathCopy(t *testing.T) { + vin := dyn.V(map[string]dyn.Value{ + "foo": dyn.V(42), + "bar": dyn.V(43), + }) + + var paths []dyn.Path + + // The callback should receive a copy of the path. + // If the same underlying value is used, all collected paths will be the same. + // This test uses `MapByPattern` to collect all paths in the map. + // Visit itself doesn't have public functions and we exclusively use black-box testing for this package. + _, _ = dyn.MapByPattern(vin, dyn.NewPattern(dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + paths = append(paths, p) + return v, nil + }) + + // Verify that the paths retained their original values. + var strings []string + for _, p := range paths { + strings = append(strings, p.String()) + } + assert.ElementsMatch(t, strings, []string{ + "foo", + "bar", + }) +} From 02e83877f4284589942bb439e849e01fc81a4314 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Sep 2024 13:34:57 +0200 Subject: [PATCH 26/78] Added listing cluster filtering for cluster lookups (#1754) ## Changes We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters. Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts. ## Tests Existing unit tests passing --- .codegen/lookup.go.tmpl | 4 ++ .../resolve_resource_references_test.go | 33 +++++++++----- bundle/config/variable/lookup.go | 44 +++++++++++++++++++ bundle/config/variable/lookup_overrides.go | 41 +++++++++++++++++ bundle/tests/variables_test.go | 9 +++- 5 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 bundle/config/variable/lookup_overrides.go diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 431709f9..124b629d 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -116,6 +116,10 @@ func allResolvers() *resolvers { {{range .Services -}} {{- if in $allowlist .KebabName -}} r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["{{.Singular.PascalName}}"] + if ok { + return fn(ctx, w, name) + } entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) if err != nil { return "", err diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go index 86a03b23..ee2f0e2e 100644 --- a/bundle/config/mutator/resolve_resource_references_test.go +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -2,7 +2,6 @@ package mutator import ( "context" - "fmt" "testing" "github.com/databricks/cli/bundle" @@ -44,11 +43,13 @@ func TestResolveClusterReference(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef1).Return(&compute.ClusterDetails{ - ClusterId: "1234-5678-abcd", - }, nil) - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef2).Return(&compute.ClusterDetails{ - ClusterId: "9876-5432-xywz", + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: clusterRef1}, + {ClusterId: "9876-5432-xywz", ClusterName: clusterRef2}, }, nil) diags := bundle.Apply(context.Background(), b, ResolveResourceReferences()) @@ -78,10 +79,16 @@ func TestResolveNonExistentClusterReference(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, clusterRef).Return(nil, fmt.Errorf("ClusterDetails named '%s' does not exist", clusterRef)) + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: "some other cluster"}, + }, nil) diags := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: ClusterDetails named 'Random' does not exist") + require.ErrorContains(t, diags.Error(), "failed to resolve cluster: Random, err: cluster named 'Random' does not exist") } func TestNoLookupIfVariableIsSet(t *testing.T) { @@ -158,8 +165,14 @@ func TestResolveVariableReferencesInVariableLookups(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) clusterApi := m.GetMockClustersAPI() - clusterApi.EXPECT().GetByClusterName(mock.Anything, "cluster-bar-dev").Return(&compute.ClusterDetails{ - ClusterId: "1234-5678-abcd", + + clusterApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: "cluster-bar-dev"}, + {ClusterId: "9876-5432-xywz", ClusterName: "some other cluster"}, }, nil) diags := bundle.Apply(context.Background(), b, bundle.Seq(ResolveVariableReferencesInLookup(), ResolveResourceReferences())) diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 9c85e2a7..e40b0ef7 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -220,6 +220,10 @@ type resolvers struct { func allResolvers() *resolvers { r := &resolvers{} r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Alert"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Alerts.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -228,6 +232,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.ClusterPolicy = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["ClusterPolicy"] + if ok { + return fn(ctx, w, name) + } entity, err := w.ClusterPolicies.GetByName(ctx, name) if err != nil { return "", err @@ -236,6 +244,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PolicyId), nil } r.Cluster = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Cluster"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Clusters.GetByClusterName(ctx, name) if err != nil { return "", err @@ -244,6 +256,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.ClusterId), nil } r.Dashboard = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Dashboard"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Dashboards.GetByName(ctx, name) if err != nil { return "", err @@ -252,6 +268,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.InstancePool = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["InstancePool"] + if ok { + return fn(ctx, w, name) + } entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) if err != nil { return "", err @@ -260,6 +280,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.InstancePoolId), nil } r.Job = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Job"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Jobs.GetBySettingsName(ctx, name) if err != nil { return "", err @@ -268,6 +292,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.JobId), nil } r.Metastore = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Metastore"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Metastores.GetByName(ctx, name) if err != nil { return "", err @@ -276,6 +304,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.MetastoreId), nil } r.Pipeline = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Pipeline"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Pipelines.GetByName(ctx, name) if err != nil { return "", err @@ -284,6 +316,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PipelineId), nil } r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Query"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Queries.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -292,6 +328,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.Id), nil } r.ServicePrincipal = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["ServicePrincipal"] + if ok { + return fn(ctx, w, name) + } entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name) if err != nil { return "", err @@ -300,6 +340,10 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.ApplicationId), nil } r.Warehouse = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + fn, ok := lookupOverrides["Warehouse"] + if ok { + return fn(ctx, w, name) + } entity, err := w.Warehouses.GetByName(ctx, name) if err != nil { return "", err diff --git a/bundle/config/variable/lookup_overrides.go b/bundle/config/variable/lookup_overrides.go new file mode 100644 index 00000000..1be373dc --- /dev/null +++ b/bundle/config/variable/lookup_overrides.go @@ -0,0 +1,41 @@ +package variable + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +var lookupOverrides = map[string]resolverFunc{ + "Cluster": resolveCluster, +} + +// We added a custom resolver for the cluster to add filtering for the cluster source when we list all clusters. +// Without the filtering listing could take a very long time (5-10 mins) which leads to lookup timeouts. +func resolveCluster(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + result, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }) + + if err != nil { + return "", err + } + + tmp := map[string][]compute.ClusterDetails{} + for _, v := range result { + key := v.ClusterName + tmp[key] = append(tmp[key], v) + } + alternatives, ok := tmp[name] + if !ok || len(alternatives) == 0 { + return "", fmt.Errorf("cluster named '%s' does not exist", name) + } + if len(alternatives) > 1 { + return "", fmt.Errorf("there are %d instances of clusters named '%s'", len(alternatives), name) + } + return alternatives[0].ClusterId, nil +} diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index 51a23e5d..9451c5a0 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -124,8 +124,13 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) { }, nil) clustersApi := mockWorkspaceClient.GetMockClustersAPI() - clustersApi.EXPECT().GetByClusterName(mock.Anything, "some-test-cluster").Return(&compute.ClusterDetails{ - ClusterId: "4321", + clustersApi.EXPECT().ListAll(mock.Anything, compute.ListClustersRequest{ + FilterBy: &compute.ListClustersFilterBy{ + ClusterSources: []compute.ClusterSource{compute.ClusterSourceApi, compute.ClusterSourceUi}, + }, + }).Return([]compute.ClusterDetails{ + {ClusterId: "4321", ClusterName: "some-test-cluster"}, + {ClusterId: "9876", ClusterName: "some-other-cluster"}, }, nil) clusterPoliciesApi := mockWorkspaceClient.GetMockClusterPoliciesAPI() From b451905b6ea3091fac34775e3a14a5af9f649d53 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 9 Sep 2024 11:56:16 +0200 Subject: [PATCH 27/78] Expand library globs relative to the sync root (#1756) ## Changes Library glob expansion happens during deployment. Before that, all entries that refer to local paths in resource definitions are made relative to the _sync root_. Before #1694, they were made relative to the _bundle root_. This PR didn't update the library glob expansion code to use the sync root path. If you were using the sync paths setting with library globs, the CLI would fail to expand the globs because the code was using the wrong path to anchor those globs. This change fixes the issue. ## Tests Manually confirmed that this fixes the issue reported in #1755. --- bundle/libraries/expand_glob_references.go | 10 +-- .../libraries/expand_glob_references_test.go | 6 +- bundle/libraries/match_test.go | 8 +-- bundle/libraries/upload.go | 2 +- bundle/libraries/upload_test.go | 8 +-- .../bundle.yml | 3 + bundle/tests/python_wheel_test.go | 64 +++++++++---------- 7 files changed, 49 insertions(+), 52 deletions(-) diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 9322a06b..c71615e0 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -39,7 +39,7 @@ func getLibDetails(v dyn.Value) (string, string, bool) { } func findMatches(b *bundle.Bundle, path string) ([]string, error) { - matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, path)) if err != nil { return nil, err } @@ -52,10 +52,10 @@ func findMatches(b *bundle.Bundle, path string) ([]string, error) { } } - // We make the matched path relative to the root path before storing it + // We make the matched path relative to the sync root path before storing it // to allow upload mutator to distinguish between local and remote paths for i, match := range matches { - matches[i], err = filepath.Rel(b.RootPath, match) + matches[i], err = filepath.Rel(b.SyncRootPath, match) if err != nil { return nil, err } @@ -211,8 +211,8 @@ func (e *expand) Name() string { // ExpandGlobReferences expands any glob references in the libraries or environments section // to corresponding local paths. -// We only expand local paths (i.e. paths that are relative to the root path). -// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to +// We only expand local paths (i.e. paths that are relative to the sync root path). +// After expanding we make the paths relative to the sync root path to allow upload mutator later in the chain to // distinguish between local and remote paths. func ExpandGlobReferences() bundle.Mutator { return &expand{} diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index 34855b53..e7f2e169 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -23,7 +23,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -104,7 +104,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -189,7 +189,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { testutil.Touch(t, dir, "jar", "my2.jar") b := &bundle.Bundle{ - RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/libraries/match_test.go b/bundle/libraries/match_test.go index e60504c8..78765cbd 100644 --- a/bundle/libraries/match_test.go +++ b/bundle/libraries/match_test.go @@ -18,7 +18,7 @@ func TestValidateEnvironments(t *testing.T) { testutil.Touch(t, tmpDir, "wheel.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -50,7 +50,7 @@ func TestValidateEnvironmentsNoFile(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -84,7 +84,7 @@ func TestValidateTaskLibraries(t *testing.T) { testutil.Touch(t, tmpDir, "wheel.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -117,7 +117,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 224e7ab2..90a1a21f 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -74,7 +74,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return v, nil } - source = filepath.Join(b.RootPath, source) + source = filepath.Join(b.SyncRootPath, source) libs[source] = append(libs[source], configLocation{ configPath: p, location: v.Location(), diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go index 82fe6e7c..44b194c5 100644 --- a/bundle/libraries/upload_test.go +++ b/bundle/libraries/upload_test.go @@ -24,7 +24,7 @@ func TestArtifactUploadForWorkspace(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/foo/bar/artifacts", @@ -112,7 +112,7 @@ func TestArtifactUploadForVolumes(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/Volumes/foo/bar/artifacts", @@ -200,7 +200,7 @@ func TestArtifactUploadWithNoLibraryReference(t *testing.T) { whlLocalPath := filepath.Join(whlFolder, "source.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/Workspace/foo/bar/artifacts", @@ -240,7 +240,7 @@ func TestUploadMultipleLibraries(t *testing.T) { testutil.Touch(t, whlFolder, "source4.whl") b := &bundle.Bundle{ - RootPath: tmpDir, + SyncRootPath: tmpDir, Config: config.Root{ Workspace: config.Workspace{ ArtifactPath: "/foo/bar/artifacts", diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml index 49286196..d0308430 100644 --- a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml @@ -1,6 +1,9 @@ bundle: name: python-wheel-local +workspace: + artifact_path: /foo/bar + resources: jobs: test_job: diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index c4d85703..c982c09d 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -15,11 +15,10 @@ import ( ) func TestPythonWheelBuild(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") @@ -32,11 +31,10 @@ func TestPythonWheelBuild(t *testing.T) { } func TestPythonWheelBuildAutoDetect(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") @@ -49,11 +47,10 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { } func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") @@ -66,11 +63,10 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { } func TestPythonWheelWithDBFSLib(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_dbfs_lib") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) match := libraries.ExpandGlobReferences() @@ -79,11 +75,11 @@ func TestPythonWheelWithDBFSLib(t *testing.T) { } func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") - b.Config.Workspace.ArtifactPath = "/foo/bar" + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) + require.NoError(t, diags.Error()) mockFiler := mockfiler.NewMockFiler(t) mockFiler.EXPECT().Write( @@ -94,20 +90,20 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { filer.CreateParentDirectories, ).Return(nil) - u := libraries.UploadWithClient(mockFiler) - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u)) + diags = bundle.Apply(ctx, b, bundle.Seq( + libraries.ExpandGlobReferences(), + libraries.UploadWithClient(mockFiler), + )) require.NoError(t, diags.Error()) require.Empty(t, diags) - require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/environment_key") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/environment_key", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") @@ -120,11 +116,10 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { } func TestPythonWheelBuildMultiple(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") @@ -137,11 +132,10 @@ func TestPythonWheelBuildMultiple(t *testing.T) { } func TestPythonWheelNoBuild(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build") - require.NoError(t, err) + b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default") - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + ctx := context.Background() + diags := bundle.Apply(ctx, b, phases.Build()) require.NoError(t, diags.Error()) match := libraries.ExpandGlobReferences() From 90244f9c169b8caac7f9c7ee0040c8e4bf38ee36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:16:51 +0200 Subject: [PATCH 28/78] Bump golang.org/x/term from 0.23.0 to 0.24.0 (#1757) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.23.0 to 0.24.0.
Commits
  • 2f7b0dd go.mod: update golang.org/x dependencies
  • f867b76 x/term: set missing VIRTUAL_TERMINAL_INPUT flag on Windows
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.23.0&new-version=0.24.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9777106c..e14e3b78 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.23.0 + golang.org/x/term v0.24.0 golang.org/x/text v0.17.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -61,7 +61,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index b232e8e4..808a4803 100644 --- a/go.sum +++ b/go.sum @@ -208,10 +208,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= From f19f7fa130e3aff75721e4b7e9d8091dc0d3a949 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:17:02 +0200 Subject: [PATCH 29/78] Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 (#1761) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.22.0 to 0.23.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.22.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e14e3b78..4e435fea 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.20.0 - golang.org/x/oauth2 v0.22.0 + golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.24.0 golang.org/x/text v0.17.0 diff --git a/go.sum b/go.sum index 808a4803..ff7e394b 100644 --- a/go.sum +++ b/go.sum @@ -191,8 +191,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 0ef1ada14bab40b7a03ec8dc418bd4e948c90d50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:40:02 +0200 Subject: [PATCH 30/78] Bump golang.org/x/text from 0.17.0 to 0.18.0 (#1759) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.17.0 to 0.18.0.
Commits
  • 1e3e9fd all: rename Example test functions to prevent vet errors
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.17.0&new-version=0.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4e435fea..e1f8e806 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.24.0 - golang.org/x/text v0.17.0 + golang.org/x/text v0.18.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index ff7e394b..71efdc55 100644 --- a/go.sum +++ b/go.sum @@ -214,8 +214,8 @@ golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From d3e221a116bfadfb525a82f1e31ad5dd7224b2cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:49:39 +0200 Subject: [PATCH 31/78] Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 (#1760) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.45.0 to 0.46.0.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.46.0

Bug Fixes

  • Fail fast when authenticating if host is not configured (#1033).
  • Improve non-JSON error handling (#1031).

Internal Changes

  • Add TestAccCreateOboTokenOnAws to flaky test list (#1029).
  • Add workflows manage integration tests checks (#1032).
  • Fix TestMwsAccWorkspaces cleanup (#1028).
  • Improve integration test comment (#1035).
  • Temporary ignore Metastore test failures (#1027).
  • Update test to support new accounts (#1026).
  • Use statuses instead of checks (#1036).

API Changes:

OpenAPI SHA: d05898328669a3f8ab0c2ecee37db2673d3ea3f7, Date: 2024-09-04

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.46.0

Bug Fixes

  • Fail fast when authenticating if host is not configured (#1033).
  • Improve non-JSON error handling (#1031).

Internal Changes

  • Add TestAccCreateOboTokenOnAws to flaky test list (#1029).
  • Add workflows manage integration tests checks (#1032).
  • Fix TestMwsAccWorkspaces cleanup (#1028).
  • Improve integration test comment (#1035).
  • Temporary ignore Metastore test failures (#1027).
  • Update test to support new accounts (#1026).
  • Use statuses instead of checks (#1036).

API Changes:

OpenAPI SHA: d05898328669a3f8ab0c2ecee37db2673d3ea3f7, Date: 2024-09-04

Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.45.0&new-version=0.46.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- bundle/schema/docs/bundle_descriptions.json | 12 +++ cmd/workspace/experiments/experiments.go | 7 +- cmd/workspace/metastores/metastores.go | 6 +- cmd/workspace/permissions/permissions.go | 21 +++-- .../quality-monitors/quality-monitors.go | 82 +++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 8 files changed, 120 insertions(+), 16 deletions(-) diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 8b01a242..4ceeab3d 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -3eae49b444cac5a0118a3503e5b7ecef7f96527a \ No newline at end of file +d05898328669a3f8ab0c2ecee37db2673d3ea3f7 \ No newline at end of file diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json index 908a1c2b..f03a4480 100644 --- a/bundle/schema/docs/bundle_descriptions.json +++ b/bundle/schema/docs/bundle_descriptions.json @@ -2046,6 +2046,12 @@ "instance_profile_arn": { "description": "ARN of the instance profile that the served model will use to access AWS resources." }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to." + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to." + }, "model_name": { "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" }, @@ -5147,6 +5153,12 @@ "instance_profile_arn": { "description": "ARN of the instance profile that the served model will use to access AWS resources." }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to." + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to." + }, "model_name": { "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" }, diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index e1e00380..b1af2f86 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -941,7 +941,12 @@ func newListArtifacts() *cobra.Command { cmd.Long = `Get all artifacts. List artifacts for a run. Takes an optional artifact_path prefix. If it is - specified, the response contains only artifacts with the specified prefix.",` + specified, the response contains only artifacts with the specified prefix. + This API does not support pagination when listing artifacts in UC Volumes. A + maximum of 1000 artifacts will be retrieved for UC Volumes. Please call + /api/2.0/fs/directories{directory_path} for listing artifacts in UC Volumes, + which supports pagination. See [List directory contents | Files + API](/api/workspace/files/listdirectorycontents).` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/metastores/metastores.go b/cmd/workspace/metastores/metastores.go index dd40bf92..22bcd3dc 100755 --- a/cmd/workspace/metastores/metastores.go +++ b/cmd/workspace/metastores/metastores.go @@ -88,7 +88,9 @@ func newAssign() *cobra.Command { Arguments: WORKSPACE_ID: A workspace ID. METASTORE_ID: The unique ID of the metastore. - DEFAULT_CATALOG_NAME: The name of the default catalog in the metastore.` + DEFAULT_CATALOG_NAME: The name of the default catalog in the metastore. This field is depracted. + Please use "Default Namespace API" to configure the default catalog for a + Databricks workspace.` cmd.Annotations = make(map[string]string) @@ -665,7 +667,7 @@ func newUpdateAssignment() *cobra.Command { // TODO: short flags cmd.Flags().Var(&updateAssignmentJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateAssignmentReq.DefaultCatalogName, "default-catalog-name", updateAssignmentReq.DefaultCatalogName, `The name of the default catalog for the metastore.`) + cmd.Flags().StringVar(&updateAssignmentReq.DefaultCatalogName, "default-catalog-name", updateAssignmentReq.DefaultCatalogName, `The name of the default catalog in the metastore.`) cmd.Flags().StringVar(&updateAssignmentReq.MetastoreId, "metastore-id", updateAssignmentReq.MetastoreId, `The unique ID of the metastore.`) cmd.Use = "update-assignment WORKSPACE_ID" diff --git a/cmd/workspace/permissions/permissions.go b/cmd/workspace/permissions/permissions.go index fd9c1a46..c6033e4a 100755 --- a/cmd/workspace/permissions/permissions.go +++ b/cmd/workspace/permissions/permissions.go @@ -117,9 +117,10 @@ func newGet() *cobra.Command { Arguments: REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, - authorization, clusters, cluster-policies, dbsql-dashboards, directories, - experiments, files, instance-pools, jobs, notebooks, pipelines, queries, - registered-models, repos, serving-endpoints, or warehouses. + authorization, clusters, cluster-policies, dashboards, dbsql-dashboards, + directories, experiments, files, instance-pools, jobs, notebooks, + pipelines, queries, registered-models, repos, serving-endpoints, or + warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -245,9 +246,10 @@ func newSet() *cobra.Command { Arguments: REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, - authorization, clusters, cluster-policies, dbsql-dashboards, directories, - experiments, files, instance-pools, jobs, notebooks, pipelines, queries, - registered-models, repos, serving-endpoints, or warehouses. + authorization, clusters, cluster-policies, dashboards, dbsql-dashboards, + directories, experiments, files, instance-pools, jobs, notebooks, + pipelines, queries, registered-models, repos, serving-endpoints, or + warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -319,9 +321,10 @@ func newUpdate() *cobra.Command { Arguments: REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, - authorization, clusters, cluster-policies, dbsql-dashboards, directories, - experiments, files, instance-pools, jobs, notebooks, pipelines, queries, - registered-models, repos, serving-endpoints, or warehouses. + authorization, clusters, cluster-policies, dashboards, dbsql-dashboards, + directories, experiments, files, instance-pools, jobs, notebooks, + pipelines, queries, registered-models, repos, serving-endpoints, or + warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/quality-monitors/quality-monitors.go b/cmd/workspace/quality-monitors/quality-monitors.go index 95d99216..1ff9b017 100755 --- a/cmd/workspace/quality-monitors/quality-monitors.go +++ b/cmd/workspace/quality-monitors/quality-monitors.go @@ -41,6 +41,7 @@ func New() *cobra.Command { cmd.AddCommand(newGet()) cmd.AddCommand(newGetRefresh()) cmd.AddCommand(newListRefreshes()) + cmd.AddCommand(newRegenerateDashboard()) cmd.AddCommand(newRunRefresh()) cmd.AddCommand(newUpdate()) @@ -503,6 +504,87 @@ func newListRefreshes() *cobra.Command { return cmd } +// start regenerate-dashboard command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var regenerateDashboardOverrides []func( + *cobra.Command, + *catalog.RegenerateDashboardRequest, +) + +func newRegenerateDashboard() *cobra.Command { + cmd := &cobra.Command{} + + var regenerateDashboardReq catalog.RegenerateDashboardRequest + var regenerateDashboardJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(®enerateDashboardJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(®enerateDashboardReq.WarehouseId, "warehouse-id", regenerateDashboardReq.WarehouseId, `Optional argument to specify the warehouse for dashboard regeneration.`) + + cmd.Use = "regenerate-dashboard TABLE_NAME" + cmd.Short = `Regenerate a monitoring dashboard.` + cmd.Long = `Regenerate a monitoring dashboard. + + Regenerates the monitoring dashboard for the specified table. + + The caller must either: 1. be an owner of the table's parent catalog 2. have + **USE_CATALOG** on the table's parent catalog and be an owner of the table's + parent schema 3. have the following permissions: - **USE_CATALOG** on the + table's parent catalog - **USE_SCHEMA** on the table's parent schema - be an + owner of the table + + The call must be made from the workspace where the monitor was created. The + dashboard will be regenerated in the assets directory that was specified when + the monitor was created. + + Arguments: + TABLE_NAME: Full name of the table.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = regenerateDashboardJson.Unmarshal(®enerateDashboardReq) + if err != nil { + return err + } + } + regenerateDashboardReq.TableName = args[0] + + response, err := w.QualityMonitors.RegenerateDashboard(ctx, regenerateDashboardReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range regenerateDashboardOverrides { + fn(cmd, ®enerateDashboardReq) + } + + return cmd +} + // start run-refresh command // Slice with functions to override default command behavior. diff --git a/go.mod b/go.mod index e1f8e806..ba41ef3a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.45.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.46.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index 71efdc55..3d4a2cdc 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.45.0 h1:wdx5Wm/ESrahdHeq62WrjLeGjV4r722LLanD8ahI0Mo= -github.com/databricks/databricks-sdk-go v0.45.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.46.0 h1:D0TxmtSVAOsdnfzH4OGtAmcq+8TyA7Z6fA6JEYhupeY= +github.com/databricks/databricks-sdk-go v0.46.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 28b39cd3f7c7c1af809e4cc7d9a5bd5f3f44daa0 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 10 Sep 2024 19:25:18 +0530 Subject: [PATCH 32/78] Make bundle JSON schema modular with `$defs` (#1700) ## Changes This PR makes sweeping changes to the way we generate and test the bundle JSON schema. The main benefits are: 1. More modular JSON schema. Every definition in the schema now is one level deep and points to references instead of inlining the entire schema for a field. This unblocks PyDABs from taking a dependency on the JSON schema. 2. Generate the JSON schema during CLI code generation. Directly stream it instead of computing it at runtime whenever a user calls `databricks bundle schema`. This is nice because we no longer need to embed a partial OpenAPI spec in the CLI. Down the line, we can add a `Schema()` method to every struct in the Databricks Go SDK and remove the dependency on the OpenAPI spec altogether. It'll become more important once we decouple Go SDK structs and methods from the underlying APIs. 3. Add enum values for Go SDK fields in the JSON schema. Better autocompletion and validation for these fields. As a follow-up, we can add enum values for non-Go SDK enums as well (created internal ticket to track). 4. Use "packageName.structName" as a key to read JSON schemas from the OpenAPI spec for Go SDK structs. Before, we would use an unrolled presentation of the JSON schema (stored in `bundle_descriptions.json`), which was complex to parse and include in the final JSON schema output. This also means loading values from the OpenAPI spec for `target` schema works automatically and no longer needs custom code. 5. Support recursive types (eg: `for_each_task`). With us now using $refs everywhere it's trivial to support. 6. Using complex variables would be invalid according to the schema generated before this PR. Now that bug is fixed. In the future adding more custom rules will be easier as well due to the single level nature of the JSON schema. Since this is a complete change of approach in how we generate the JSON schema, there are a few (very minor) regressions worth calling out. 1. We'll lose a few custom descriptions for non Go SDK structs that were a part of `bundle_descriptions.json`. Support for those can be added in the future as a followup. 2. Since now the final JSON schema is a static artefact, we lose some lead time for the signal that JSON schema integration tests are failing. It's okay though since we have a lot of coverage via the existing unit tests. ## Tests Unit tests. End to end tests are being added in this PR: https://github.com/databricks/cli/pull/1726 Previous unit tests were all deleted because they were bloated. Effort was made to make the new unit tests provide (almost) equivalent coverage. --- .codegen.json | 4 +- .gitattributes | 2 +- bundle/internal/bundle/schema/main.go | 42 - bundle/internal/schema/main.go | 93 + bundle/internal/schema/parser.go | 123 + bundle/schema/README.md | 18 - bundle/schema/docs.go | 109 - bundle/schema/docs/bundle_descriptions.json | 6447 ------------------- bundle/schema/docs_test.go | 62 - bundle/schema/embed.go | 6 + bundle/schema/embed_test.go | 71 + bundle/schema/jsonschema.json | 5524 ++++++++++++++++ bundle/schema/openapi.go | 293 - bundle/schema/openapi_test.go | 493 -- bundle/schema/schema.go | 287 - bundle/schema/schema_test.go | 1900 ------ bundle/schema/spec.go | 11 - bundle/schema/tracker.go | 53 - cmd/bundle/schema.go | 33 +- libs/dyn/dynvar/ref.go | 4 +- libs/jsonschema/from_type.go | 356 + libs/jsonschema/from_type_test.go | 521 ++ libs/jsonschema/schema.go | 42 +- libs/jsonschema/schema_test.go | 90 - libs/jsonschema/test_types/test_types.go | 25 + libs/template/config_test.go | 2 +- 26 files changed, 6731 insertions(+), 9880 deletions(-) delete mode 100644 bundle/internal/bundle/schema/main.go create mode 100644 bundle/internal/schema/main.go create mode 100644 bundle/internal/schema/parser.go delete mode 100644 bundle/schema/README.md delete mode 100644 bundle/schema/docs.go delete mode 100644 bundle/schema/docs/bundle_descriptions.json delete mode 100644 bundle/schema/docs_test.go create mode 100644 bundle/schema/embed.go create mode 100644 bundle/schema/embed_test.go create mode 100644 bundle/schema/jsonschema.json delete mode 100644 bundle/schema/openapi.go delete mode 100644 bundle/schema/openapi_test.go delete mode 100644 bundle/schema/schema.go delete mode 100644 bundle/schema/schema_test.go delete mode 100644 bundle/schema/spec.go delete mode 100644 bundle/schema/tracker.go create mode 100644 libs/jsonschema/from_type.go create mode 100644 libs/jsonschema/from_type_test.go create mode 100644 libs/jsonschema/test_types/test_types.go diff --git a/.codegen.json b/.codegen.json index 8cb42b41..077d072b 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json", + "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes" + "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/.gitattributes b/.gitattributes index d82ab769..f35c4f81 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -bundle/schema/docs/bundle_descriptions.json linguist-generated=true +bundle/schema/jsonschema.json linguist-generated=true diff --git a/bundle/internal/bundle/schema/main.go b/bundle/internal/bundle/schema/main.go deleted file mode 100644 index c9cc7cd4..00000000 --- a/bundle/internal/bundle/schema/main.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - "os" - - "github.com/databricks/cli/bundle/schema" -) - -func main() { - if len(os.Args) != 2 { - fmt.Println("Usage: go run main.go ") - os.Exit(1) - } - - // Output file, to write the generated schema descriptions to. - outputFile := os.Args[1] - - // Input file, the databricks openapi spec. - inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC") - if inputFile == "" { - log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set") - } - - // Generate the schema descriptions. - docs, err := schema.UpdateBundleDescriptions(inputFile) - if err != nil { - log.Fatal(err) - } - result, err := json.MarshalIndent(docs, "", " ") - if err != nil { - log.Fatal(err) - } - - // Write the schema descriptions to the output file. - err = os.WriteFile(outputFile, result, 0644) - if err != nil { - log.Fatal(err) - } -} diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go new file mode 100644 index 00000000..3c1fb5da --- /dev/null +++ b/bundle/internal/schema/main.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "reflect" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/libs/jsonschema" +) + +func interpolationPattern(s string) string { + return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)+)\}`, s) +} + +func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + if typ == reflect.TypeOf(config.Root{}) || typ == reflect.TypeOf(variable.Variable{}) { + return s + } + + switch s.Type { + case jsonschema.ArrayType, jsonschema.ObjectType: + // arrays and objects can have complex variable values specified. + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ + s, + { + Type: jsonschema.StringType, + Pattern: interpolationPattern("var"), + }}, + } + case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: + // primitives can have variable values, or references like ${bundle.xyz} + // or ${workspace.xyz} + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ + s, + {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("artifacts")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("var")}, + }, + } + default: + return s + } +} + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: go run main.go ") + os.Exit(1) + } + + // Output file, where the generated JSON schema will be written to. + outputFile := os.Args[1] + + // Input file, the databricks openapi spec. + inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC") + if inputFile == "" { + log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set") + } + + p, err := newParser(inputFile) + if err != nil { + log.Fatal(err) + } + + // Generate the JSON schema from the bundle Go struct. + s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ + p.addDescriptions, + p.addEnums, + addInterpolationPatterns, + }) + if err != nil { + log.Fatal(err) + } + + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + log.Fatal(err) + } + + // Write the schema descriptions to the output file. + err = os.WriteFile(outputFile, b, 0644) + if err != nil { + log.Fatal(err) + } +} diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go new file mode 100644 index 00000000..ef3d6e71 --- /dev/null +++ b/bundle/internal/schema/parser.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path" + "reflect" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +type Components struct { + Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"` +} + +type Specification struct { + Components Components `json:"components"` +} + +type openapiParser struct { + ref map[string]jsonschema.Schema +} + +func newParser(path string) (*openapiParser, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + spec := Specification{} + err = json.Unmarshal(b, &spec) + if err != nil { + return nil, err + } + + p := &openapiParser{} + p.ref = spec.Components.Schemas + return p, nil +} + +// This function checks if the input type: +// 1. Is a Databricks Go SDK type. +// 2. Has a Databricks Go SDK type embedded in it. +// +// If the above conditions are met, the function returns the JSON schema +// corresponding to the Databricks Go SDK type from the OpenAPI spec. +func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { + typs := []reflect.Type{typ} + + // Check for embedded Databricks Go SDK types. + if typ.Kind() == reflect.Struct { + for i := 0; i < typ.NumField(); i++ { + if !typ.Field(i).Anonymous { + continue + } + + // Deference current type if it's a pointer. + ctyp := typ.Field(i).Type + for ctyp.Kind() == reflect.Ptr { + ctyp = ctyp.Elem() + } + + typs = append(typs, ctyp) + } + } + + for _, ctyp := range typs { + // Skip if it's not a Go SDK type. + if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") { + continue + } + + pkgName := path.Base(ctyp.PkgPath()) + k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name()) + + // Skip if the type is not in the openapi spec. + _, ok := p.ref[k] + if !ok { + continue + } + + // Return the first Go SDK type found in the openapi spec. + return p.ref[k], true + } + + return jsonschema.Schema{}, false +} + +// Use the OpenAPI spec to load descriptions for the given type. +func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + ref, ok := p.findRef(typ) + if !ok { + return s + } + + s.Description = ref.Description + for k, v := range s.Properties { + if refProp, ok := ref.Properties[k]; ok { + v.Description = refProp.Description + } + } + + return s +} + +// Use the OpenAPI spec add enum values for the given type. +func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + ref, ok := p.findRef(typ) + if !ok { + return s + } + + s.Enum = append(s.Enum, ref.Enum...) + for k, v := range s.Properties { + if refProp, ok := ref.Properties[k]; ok { + v.Enum = append(v.Enum, refProp.Enum...) + } + } + + return s +} diff --git a/bundle/schema/README.md b/bundle/schema/README.md deleted file mode 100644 index bf6b87df..00000000 --- a/bundle/schema/README.md +++ /dev/null @@ -1,18 +0,0 @@ -### Overview - -`docs/bundle_descriptions.json` contains both autogenerated as well as manually written -descriptions for the json schema. Specifically -1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec -2. `targets` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`) -3. `bundle` : manually editted -4. `include` : manually editted -5. `workspace` : manually editted -6. `artifacts` : manually editted - -These descriptions are rendered in the inline documentation in an IDE - -### SOP: Add schema descriptions for new fields in bundle config - -Manually edit bundle_descriptions.json to add your descriptions. Note that the -descriptions in `resources` block is generated from the OpenAPI spec, and thus -any changes there will be overwritten. diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go deleted file mode 100644 index 6e9289f9..00000000 --- a/bundle/schema/docs.go +++ /dev/null @@ -1,109 +0,0 @@ -package schema - -import ( - _ "embed" - "encoding/json" - "fmt" - "os" - "reflect" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/jsonschema" -) - -// A subset of Schema struct -type Docs struct { - Description string `json:"description"` - Properties map[string]*Docs `json:"properties,omitempty"` - Items *Docs `json:"items,omitempty"` - AdditionalProperties *Docs `json:"additionalproperties,omitempty"` -} - -//go:embed docs/bundle_descriptions.json -var bundleDocs []byte - -func (docs *Docs) refreshTargetsDocs() error { - targetsDocs, ok := docs.Properties["targets"] - if !ok || targetsDocs.AdditionalProperties == nil || - targetsDocs.AdditionalProperties.Properties == nil { - return fmt.Errorf("invalid targets descriptions") - } - targetProperties := targetsDocs.AdditionalProperties.Properties - propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"} - for _, p := range propertiesToCopy { - targetProperties[p] = docs.Properties[p] - } - return nil -} - -func LoadBundleDescriptions() (*Docs, error) { - embedded := Docs{} - err := json.Unmarshal(bundleDocs, &embedded) - return &embedded, err -} - -func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) { - embedded, err := LoadBundleDescriptions() - if err != nil { - return nil, err - } - - // Generate schema from the embedded descriptions, and convert it back to docs. - // This creates empty descriptions for any properties that were missing in the - // embedded descriptions. - schema, err := New(reflect.TypeOf(config.Root{}), embedded) - if err != nil { - return nil, err - } - docs := schemaToDocs(schema) - - // Load the Databricks OpenAPI spec - openapiSpec, err := os.ReadFile(openapiSpecPath) - if err != nil { - return nil, err - } - spec := &Specification{} - err = json.Unmarshal(openapiSpec, spec) - if err != nil { - return nil, err - } - openapiReader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - - // Generate descriptions for the "resources" field - resourcesDocs, err := openapiReader.ResourcesDocs() - if err != nil { - return nil, err - } - resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs) - if err != nil { - return nil, err - } - docs.Properties["resources"] = schemaToDocs(resourceSchema) - docs.refreshTargetsDocs() - return docs, nil -} - -// *Docs are a subset of *Schema, this function selects that subset -func schemaToDocs(jsonSchema *jsonschema.Schema) *Docs { - // terminate recursion if schema is nil - if jsonSchema == nil { - return nil - } - docs := &Docs{ - Description: jsonSchema.Description, - } - if len(jsonSchema.Properties) > 0 { - docs.Properties = make(map[string]*Docs) - } - for k, v := range jsonSchema.Properties { - docs.Properties[k] = schemaToDocs(v) - } - docs.Items = schemaToDocs(jsonSchema.Items) - if additionalProperties, ok := jsonSchema.AdditionalProperties.(*jsonschema.Schema); ok { - docs.AdditionalProperties = schemaToDocs(additionalProperties) - } - return docs -} diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json deleted file mode 100644 index f03a4480..00000000 --- a/bundle/schema/docs/bundle_descriptions.json +++ /dev/null @@ -1,6447 +0,0 @@ -{ - "description": "", - "properties": { - "artifacts": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "build": { - "description": "" - }, - "executable": { - "description": "" - }, - "files": { - "description": "", - "items": { - "description": "", - "properties": { - "source": { - "description": "" - } - } - } - }, - "path": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "bundle": { - "description": "", - "properties": { - "compute_id": { - "description": "" - }, - "databricks_cli_version": { - "description": "" - }, - "deployment": { - "description": "", - "properties": { - "fail_on_active_runs": { - "description": "" - }, - "lock": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "force": { - "description": "" - } - } - } - } - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "name": { - "description": "" - } - } - }, - "experimental": { - "description": "", - "properties": { - "pydabs": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "import": { - "description": "", - "items": { - "description": "" - } - }, - "venv_path": { - "description": "" - } - } - }, - "python_wheel_wrapper": { - "description": "" - }, - "scripts": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "use_legacy_run_as": { - "description": "" - } - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "presets": { - "description": "", - "properties": { - "jobs_max_concurrent_runs": { - "description": "" - }, - "name_prefix": { - "description": "" - }, - "pipelines_development": { - "description": "" - }, - "tags": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "trigger_pause_status": { - "description": "" - } - } - }, - "resources": { - "description": "Collection of Databricks resources to deploy.", - "properties": { - "experiments": { - "description": "List of MLflow experiments", - "additionalproperties": { - "description": "", - "properties": { - "artifact_location": { - "description": "Location where artifacts for the experiment are stored." - }, - "creation_time": { - "description": "Creation time" - }, - "experiment_id": { - "description": "Unique identifier for the experiment." - }, - "last_update_time": { - "description": "Last update time" - }, - "lifecycle_stage": { - "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs." - }, - "name": { - "description": "Human readable name that identifies the experiment." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - } - } - } - }, - "jobs": { - "description": "List of Databricks jobs", - "additionalproperties": { - "description": "", - "properties": { - "continuous": { - "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", - "properties": { - "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." - } - } - }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "properties": { - "kind": { - "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle." - }, - "metadata_file_path": { - "description": "Path of the file that contains deployment metadata." - } - } - }, - "description": { - "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." - }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", - "items": { - "description": "", - "properties": { - "environment_key": { - "description": "The key of an environment. It has to be unique within a job." - }, - "spec": { - "description": "", - "properties": { - "client": { - "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." - }, - "dependencies": { - "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", - "items": { - "description": "" - } - } - } - } - } - } - }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`." - }, - "git_source": { - "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", - "properties": { - "git_branch": { - "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit." - }, - "git_commit": { - "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag." - }, - "git_provider": { - "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive." - }, - "git_snapshot": { - "description": "", - "properties": { - "used_commit": { - "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to." - } - } - }, - "git_tag": { - "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit." - }, - "git_url": { - "description": "URL of the repository to be cloned by this job." - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "properties": { - "dirty_state": { - "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced." - }, - "import_from_git_branch": { - "description": "Name of the branch which the job is imported from." - }, - "job_config_path": { - "description": "Path of the job YAML file that contains the job specification." - } - } - } - } - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_clusters": { - "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", - "items": { - "description": "", - "properties": { - "job_cluster_key": { - "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution." - }, - "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - } - } - } - }, - "max_concurrent_runs": { - "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped." - }, - "name": { - "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding." - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", - "properties": { - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "parameters": { - "description": "Job-level parameter definitions", - "items": { - "description": "", - "properties": { - "default": { - "description": "Default value of the parameter." - }, - "name": { - "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`" - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "queue": { - "description": "The queue settings of the job.", - "properties": { - "enabled": { - "description": "If true, enable queueing for the job. This is a required field." - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role." - }, - "user_name": { - "description": "The email of an active workspace user. Non-admin users can only set this field to their own email." - } - } - }, - "schedule": { - "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "pause_status": { - "description": "Indicate whether this schedule is paused or not." - }, - "quartz_cron_expression": { - "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." - }, - "timezone_id": { - "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required." - } - } - }, - "tags": { - "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", - "additionalproperties": { - "description": "" - } - }, - "tasks": { - "description": "A list of task specifications to be executed by this job.", - "items": { - "description": "", - "properties": { - "condition_task": { - "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", - "properties": { - "left": { - "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference." - }, - "op": { - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison." - }, - "right": { - "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference." - } - } - }, - "dbt_task": { - "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", - "properties": { - "catalog": { - "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1." - }, - "commands": { - "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", - "items": { - "description": "" - } - }, - "profiles_directory": { - "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used." - }, - "project_directory": { - "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used." - }, - "schema": { - "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." - }, - "source": { - "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." - }, - "warehouse_id": { - "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." - } - } - }, - "depends_on": { - "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", - "items": { - "description": "", - "properties": { - "outcome": { - "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run." - }, - "task_key": { - "description": "The name of the task this task depends on." - } - } - } - }, - "description": { - "description": "An optional description for this task." - }, - "disable_auto_optimization": { - "description": "An option to disable auto optimization in serverless" - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environment_key": { - "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute." - }, - "existing_cluster_id": { - "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability" - }, - "for_each_task": { - "description": "" - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_cluster_key": { - "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`." - }, - "libraries": { - "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", - "items": { - "description": "", - "properties": { - "cran": { - "description": "Specification of a CRAN library to be installed as part of the library", - "properties": { - "package": { - "description": "The name of the CRAN package to install." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default CRAN repo is used." - } - } - }, - "egg": { - "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." - }, - "jar": { - "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - }, - "maven": { - "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "pypi": { - "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", - "properties": { - "package": { - "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\"." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default pip index is\nused." - } - } - }, - "requirements": { - "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" - }, - "whl": { - "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - } - } - } - }, - "max_retries": { - "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry." - }, - "min_retry_interval_millis": { - "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." - }, - "new_cluster": { - "description": "If new_cluster, a description of a new cluster that is created for each run.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - }, - "notebook_task": { - "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", - "properties": { - "base_parameters": { - "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", - "additionalproperties": { - "description": "" - } - }, - "notebook_path": { - "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." - }, - "warehouse_id": { - "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." - } - } - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", - "properties": { - "alert_on_last_attempt": { - "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run." - }, - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "pipeline_task": { - "description": "If pipeline_task, indicates that this task must execute a Pipeline.", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - }, - "pipeline_id": { - "description": "The full name of the pipeline task to execute." - } - } - }, - "python_wheel_task": { - "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", - "properties": { - "entry_point": { - "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`" - }, - "named_parameters": { - "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", - "additionalproperties": { - "description": "" - } - }, - "package_name": { - "description": "Name of the package to execute" - }, - "parameters": { - "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", - "items": { - "description": "" - } - } - } - }, - "retry_on_timeout": { - "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout." - }, - "run_if": { - "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed" - }, - "run_job_task": { - "description": "If run_job_task, indicates that this task must execute another job.", - "properties": { - "dbt_commands": { - "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", - "items": { - "description": "" - } - }, - "jar_params": { - "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "job_id": { - "description": "ID of the job to trigger." - }, - "job_parameters": { - "description": "Job-level parameters used to trigger the job.", - "additionalproperties": { - "description": "" - } - }, - "notebook_params": { - "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", - "additionalproperties": { - "description": "" - } - }, - "pipeline_params": { - "description": "", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - } - } - }, - "python_named_params": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "python_params": { - "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "spark_submit_params": { - "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "sql_params": { - "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - } - } - }, - "spark_jar_task": { - "description": "If spark_jar_task, indicates that this task must run a JAR.", - "properties": { - "jar_uri": { - "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create." - }, - "main_class_name": { - "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail." - }, - "parameters": { - "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "spark_python_task": { - "description": "If spark_python_task, indicates that this task must run a Python file.", - "properties": { - "parameters": { - "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "python_file": { - "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." - } - } - }, - "spark_submit_task": { - "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", - "properties": { - "parameters": { - "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "sql_task": { - "description": "If sql_task, indicates that this job must execute a SQL task.", - "properties": { - "alert": { - "description": "If alert, indicates that this job must refresh a SQL alert.", - "properties": { - "alert_id": { - "description": "The canonical identifier of the SQL alert." - }, - "pause_subscriptions": { - "description": "If true, the alert notifications are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, alert notifications are sent to subscribers.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "dashboard": { - "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", - "properties": { - "custom_subject": { - "description": "Subject of the email sent to subscribers of this task." - }, - "dashboard_id": { - "description": "The canonical identifier of the SQL dashboard." - }, - "pause_subscriptions": { - "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, dashboard snapshots are sent to subscriptions.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", - "properties": { - "path": { - "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." - }, - "source": { - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." - } - } - }, - "parameters": { - "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - }, - "query": { - "description": "If query, indicates that this job must execute a SQL query.", - "properties": { - "query_id": { - "description": "The canonical identifier of the SQL query." - } - } - }, - "warehouse_id": { - "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs." - } - } - }, - "task_key": { - "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset." - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout." - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout." - }, - "trigger": { - "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "file_arrival": { - "description": "File arrival trigger settings.", - "properties": { - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds" - }, - "url": { - "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location." - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "pause_status": { - "description": "Whether this trigger is paused or not." - }, - "periodic": { - "description": "Periodic trigger settings.", - "properties": { - "interval": { - "description": "The interval at which the trigger should run." - }, - "unit": { - "description": "The unit of time for the interval." - } - } - }, - "table": { - "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "table_update": { - "description": "", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - } - } - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "model_serving_endpoints": { - "description": "List of Model Serving Endpoints", - "additionalproperties": { - "description": "", - "properties": { - "config": { - "description": "The core config of the serving endpoint.", - "properties": { - "auto_capture_config": { - "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", - "properties": { - "catalog_name": { - "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled." - }, - "enabled": { - "description": "Indicates whether the inference table is enabled." - }, - "schema_name": { - "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled." - }, - "table_name_prefix": { - "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled." - } - } - }, - "served_entities": { - "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", - "items": { - "description": "", - "properties": { - "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n" - }, - "entity_version": { - "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC." - }, - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", - "properties": { - "ai21labs_config": { - "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", - "properties": { - "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - }, - "ai21labs_api_key_plaintext": { - "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - } - } - }, - "amazon_bedrock_config": { - "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", - "properties": { - "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_access_key_id_plaintext": { - "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_region": { - "description": "The AWS region to use. Bedrock has to be enabled there." - }, - "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "aws_secret_access_key_plaintext": { - "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "bedrock_provider": { - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." - } - } - }, - "anthropic_config": { - "description": "Anthropic Config. Only required if the provider is 'anthropic'.", - "properties": { - "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - }, - "anthropic_api_key_plaintext": { - "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - } - } - }, - "cohere_config": { - "description": "Cohere Config. Only required if the provider is 'cohere'.", - "properties": { - "cohere_api_base": { - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" - }, - "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - }, - "cohere_api_key_plaintext": { - "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - } - } - }, - "databricks_model_serving_config": { - "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", - "properties": { - "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_api_token_plaintext": { - "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_workspace_url": { - "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" - } - } - }, - "google_cloud_vertex_ai_config": { - "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", - "properties": { - "private_key": { - "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" - }, - "private_key_plaintext": { - "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." - }, - "project_id": { - "description": "This is the Google Cloud project id that the service account is associated with." - }, - "region": { - "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." - } - } - }, - "name": { - "description": "The name of the external model." - }, - "openai_config": { - "description": "OpenAI Config. Only required if the provider is 'openai'.", - "properties": { - "microsoft_entra_client_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" - }, - "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_client_secret_plaintext": { - "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_tenant_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" - }, - "openai_api_base": { - "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" - }, - "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_key_plaintext": { - "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_type": { - "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" - }, - "openai_api_version": { - "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n" - }, - "openai_deployment_name": { - "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n" - }, - "openai_organization": { - "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n" - } - } - }, - "palm_config": { - "description": "PaLM Config. Only required if the provider is 'palm'.", - "properties": { - "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - }, - "palm_api_key_plaintext": { - "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - } - } - }, - "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" - }, - "task": { - "description": "The task type of the external model." - } - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served entity uses to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served entity should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n" - }, - "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "served_models": { - "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", - "items": { - "description": "", - "properties": { - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served model will use to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "model_name": { - "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" - }, - "model_version": { - "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served." - }, - "name": { - "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served model should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n" - }, - "workload_type": { - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "traffic_config": { - "description": "The traffic config defining how invocations to the serving endpoint should be routed.", - "properties": { - "routes": { - "description": "The list of routes that define traffic to each served entity.", - "items": { - "description": "", - "properties": { - "served_model_name": { - "description": "The name of the served model this route configures traffic for." - }, - "traffic_percentage": { - "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive." - } - } - } - } - } - } - } - }, - "name": { - "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", - "items": { - "description": "", - "properties": { - "calls": { - "description": "Used to specify how many calls are allowed for a key within the renewal_period." - }, - "key": { - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified." - }, - "renewal_period": { - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported." - } - } - } - }, - "route_optimized": { - "description": "Enable route optimization for the serving endpoint." - }, - "tags": { - "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "Key field for a serving endpoint tag." - }, - "value": { - "description": "Optional value field for a serving endpoint tag." - } - } - } - } - } - } - }, - "models": { - "description": "List of MLflow models", - "additionalproperties": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `registered_model` was created." - }, - "description": { - "description": "Description of this `registered_model`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `registered_model` was last updated." - }, - "latest_versions": { - "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", - "items": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `model_version` was created." - }, - "current_stage": { - "description": "Current stage for this `model_version`." - }, - "description": { - "description": "Description of this `model_version`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `model_version` was last updated." - }, - "name": { - "description": "Unique name of the model" - }, - "run_id": { - "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server." - }, - "run_link": { - "description": "Run Link: Direct link to the run that generated this version" - }, - "source": { - "description": "URI indicating the location of the source model artifacts, used when creating `model_version`" - }, - "status": { - "description": "Current status of `model_version`" - }, - "status_message": { - "description": "Details on current `status`, if it is pending or failed." - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `model_version`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `model_version`." - }, - "version": { - "description": "Model's version number." - } - } - } - }, - "name": { - "description": "Unique name for the model." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `registered_model`" - } - } - } - }, - "pipelines": { - "description": "List of DLT pipelines", - "additionalproperties": { - "description": "", - "properties": { - "catalog": { - "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog." - }, - "channel": { - "description": "DLT Release Channel that specifies which version to use." - }, - "clusters": { - "description": "Cluster settings for this pipeline deployment.", - "items": { - "description": "", - "properties": { - "apply_policy_default_values": { - "description": "Note: This field won't be persisted. Only API users will check this field." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - }, - "mode": { - "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n" - } - } - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable local disk encryption for the cluster." - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "label": { - "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - } - } - } - }, - "configuration": { - "description": "String-String configuration for this pipeline execution.", - "additionalproperties": { - "description": "" - } - }, - "continuous": { - "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." - }, - "deployment": { - "description": "Deployment type of this pipeline.", - "properties": { - "kind": { - "description": "The deployment method that manages the pipeline." - }, - "metadata_file_path": { - "description": "The path to the file containing metadata about the deployment." - } - } - }, - "development": { - "description": "Whether the pipeline is in Development mode. Defaults to false." - }, - "edition": { - "description": "Pipeline product edition." - }, - "filters": { - "description": "Filters on which Pipeline packages to include in the deployed graph.", - "properties": { - "exclude": { - "description": "Paths to exclude.", - "items": { - "description": "" - } - }, - "include": { - "description": "Paths to include.", - "items": { - "description": "" - } - } - } - }, - "gateway_definition": { - "description": "The definition of a gateway pipeline to support CDC.", - "properties": { - "connection_id": { - "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." - }, - "gateway_storage_catalog": { - "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." - }, - "gateway_storage_name": { - "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" - }, - "gateway_storage_schema": { - "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." - } - } - }, - "id": { - "description": "Unique identifier for this pipeline." - }, - "ingestion_definition": { - "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", - "properties": { - "connection_name": { - "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "ingestion_gateway_id": { - "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "objects": { - "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", - "items": { - "description": "", - "properties": { - "schema": { - "description": "Select tables from a specific source schema.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store tables." - }, - "destination_schema": { - "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." - }, - "source_catalog": { - "description": "The source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Required. Schema name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "table": { - "description": "Select tables from a specific source table.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store table." - }, - "destination_schema": { - "description": "Required. Destination schema to store table." - }, - "destination_table": { - "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." - }, - "source_catalog": { - "description": "Source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Schema name in the source database. Might be optional depending on the type of source." - }, - "source_table": { - "description": "Required. Table name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - } - } - } - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "libraries": { - "description": "Libraries or code needed by this deployment.", - "items": { - "description": "", - "properties": { - "file": { - "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", - "properties": { - "path": { - "description": "The absolute path of the file." - } - } - }, - "jar": { - "description": "URI of the jar to be installed. Currently only DBFS is supported.\n" - }, - "maven": { - "description": "Specification of a maven library to be installed.\n", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "notebook": { - "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", - "properties": { - "path": { - "description": "The absolute path of the notebook." - } - } - }, - "whl": { - "description": "URI of the whl to be installed." - } - } - } - }, - "name": { - "description": "Friendly identifier for this pipeline." - }, - "notifications": { - "description": "List of notification settings for this pipeline.", - "items": { - "description": "", - "properties": { - "alerts": { - "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", - "items": { - "description": "" - } - }, - "email_recipients": { - "description": "A list of email addresses notified when a configured alert is triggered.\n", - "items": { - "description": "" - } - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "photon": { - "description": "Whether Photon is enabled for this pipeline." - }, - "serverless": { - "description": "Whether serverless compute is enabled for this pipeline." - }, - "storage": { - "description": "DBFS root directory for storing checkpoints and tables." - }, - "target": { - "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`." - }, - "trigger": { - "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", - "properties": { - "cron": { - "description": "", - "properties": { - "quartz_cron_schedule": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "manual": { - "description": "" - } - } - } - } - } - }, - "quality_monitors": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "assets_dir": { - "description": "" - }, - "baseline_table_name": { - "description": "" - }, - "custom_metrics": { - "description": "", - "items": { - "description": "", - "properties": { - "definition": { - "description": "" - }, - "input_columns": { - "description": "", - "items": { - "description": "" - } - }, - "name": { - "description": "" - }, - "output_data_type": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "data_classification_config": { - "description": "", - "properties": { - "enabled": { - "description": "" - } - } - }, - "inference_log": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "label_col": { - "description": "" - }, - "model_id_col": { - "description": "" - }, - "prediction_col": { - "description": "" - }, - "prediction_proba_col": { - "description": "" - }, - "problem_type": { - "description": "" - }, - "timestamp_col": { - "description": "" - } - } - }, - "notifications": { - "description": "", - "properties": { - "on_failure": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "on_new_classification_tag_detected": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - } - } - }, - "output_schema_name": { - "description": "" - }, - "schedule": { - "description": "", - "properties": { - "pause_status": { - "description": "" - }, - "quartz_cron_expression": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "skip_builtin_dashboard": { - "description": "" - }, - "slicing_exprs": { - "description": "", - "items": { - "description": "" - } - }, - "snapshot": { - "description": "" - }, - "time_series": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "timestamp_col": { - "description": "" - } - } - }, - "warehouse_id": { - "description": "" - } - } - } - }, - "registered_models": { - "description": "List of Registered Models", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "The name of the catalog where the schema and the registered model reside" - }, - "comment": { - "description": "The comment attached to the registered model" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "The name of the registered model" - }, - "schema_name": { - "description": "The name of the schema where the registered model resides" - }, - "storage_location": { - "description": "The storage location on the cloud under which model version data files are stored" - } - } - } - }, - "schemas": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "" - }, - "comment": { - "description": "" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "" - }, - "properties": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "storage_root": { - "description": "" - } - } - } - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - }, - "sync": { - "description": "", - "properties": { - "exclude": { - "description": "", - "items": { - "description": "" - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - }, - "paths": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "targets": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "artifacts": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "build": { - "description": "" - }, - "executable": { - "description": "" - }, - "files": { - "description": "", - "items": { - "description": "", - "properties": { - "source": { - "description": "" - } - } - } - }, - "path": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "bundle": { - "description": "", - "properties": { - "compute_id": { - "description": "" - }, - "databricks_cli_version": { - "description": "" - }, - "deployment": { - "description": "", - "properties": { - "fail_on_active_runs": { - "description": "" - }, - "lock": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "force": { - "description": "" - } - } - } - } - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "name": { - "description": "" - } - } - }, - "compute_id": { - "description": "" - }, - "default": { - "description": "" - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "mode": { - "description": "" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "presets": { - "description": "", - "properties": { - "jobs_max_concurrent_runs": { - "description": "" - }, - "name_prefix": { - "description": "" - }, - "pipelines_development": { - "description": "" - }, - "tags": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "trigger_pause_status": { - "description": "" - } - } - }, - "resources": { - "description": "Collection of Databricks resources to deploy.", - "properties": { - "experiments": { - "description": "List of MLflow experiments", - "additionalproperties": { - "description": "", - "properties": { - "artifact_location": { - "description": "Location where artifacts for the experiment are stored." - }, - "creation_time": { - "description": "Creation time" - }, - "experiment_id": { - "description": "Unique identifier for the experiment." - }, - "last_update_time": { - "description": "Last update time" - }, - "lifecycle_stage": { - "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs." - }, - "name": { - "description": "Human readable name that identifies the experiment." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - } - } - } - }, - "jobs": { - "description": "List of Databricks jobs", - "additionalproperties": { - "description": "", - "properties": { - "continuous": { - "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", - "properties": { - "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." - } - } - }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "properties": { - "kind": { - "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle." - }, - "metadata_file_path": { - "description": "Path of the file that contains deployment metadata." - } - } - }, - "description": { - "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." - }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", - "items": { - "description": "", - "properties": { - "environment_key": { - "description": "The key of an environment. It has to be unique within a job." - }, - "spec": { - "description": "", - "properties": { - "client": { - "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." - }, - "dependencies": { - "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", - "items": { - "description": "" - } - } - } - } - } - } - }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`." - }, - "git_source": { - "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", - "properties": { - "git_branch": { - "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit." - }, - "git_commit": { - "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag." - }, - "git_provider": { - "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive." - }, - "git_snapshot": { - "description": "", - "properties": { - "used_commit": { - "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to." - } - } - }, - "git_tag": { - "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit." - }, - "git_url": { - "description": "URL of the repository to be cloned by this job." - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "properties": { - "dirty_state": { - "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced." - }, - "import_from_git_branch": { - "description": "Name of the branch which the job is imported from." - }, - "job_config_path": { - "description": "Path of the job YAML file that contains the job specification." - } - } - } - } - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_clusters": { - "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", - "items": { - "description": "", - "properties": { - "job_cluster_key": { - "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution." - }, - "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - } - } - } - }, - "max_concurrent_runs": { - "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped." - }, - "name": { - "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding." - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", - "properties": { - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "parameters": { - "description": "Job-level parameter definitions", - "items": { - "description": "", - "properties": { - "default": { - "description": "Default value of the parameter." - }, - "name": { - "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`" - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "queue": { - "description": "The queue settings of the job.", - "properties": { - "enabled": { - "description": "If true, enable queueing for the job. This is a required field." - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role." - }, - "user_name": { - "description": "The email of an active workspace user. Non-admin users can only set this field to their own email." - } - } - }, - "schedule": { - "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "pause_status": { - "description": "Indicate whether this schedule is paused or not." - }, - "quartz_cron_expression": { - "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." - }, - "timezone_id": { - "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required." - } - } - }, - "tags": { - "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", - "additionalproperties": { - "description": "" - } - }, - "tasks": { - "description": "A list of task specifications to be executed by this job.", - "items": { - "description": "", - "properties": { - "condition_task": { - "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", - "properties": { - "left": { - "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference." - }, - "op": { - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison." - }, - "right": { - "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference." - } - } - }, - "dbt_task": { - "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", - "properties": { - "catalog": { - "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1." - }, - "commands": { - "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", - "items": { - "description": "" - } - }, - "profiles_directory": { - "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used." - }, - "project_directory": { - "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used." - }, - "schema": { - "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." - }, - "source": { - "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." - }, - "warehouse_id": { - "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." - } - } - }, - "depends_on": { - "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", - "items": { - "description": "", - "properties": { - "outcome": { - "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run." - }, - "task_key": { - "description": "The name of the task this task depends on." - } - } - } - }, - "description": { - "description": "An optional description for this task." - }, - "disable_auto_optimization": { - "description": "An option to disable auto optimization in serverless" - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environment_key": { - "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute." - }, - "existing_cluster_id": { - "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability" - }, - "for_each_task": { - "description": "" - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_cluster_key": { - "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`." - }, - "libraries": { - "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", - "items": { - "description": "", - "properties": { - "cran": { - "description": "Specification of a CRAN library to be installed as part of the library", - "properties": { - "package": { - "description": "The name of the CRAN package to install." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default CRAN repo is used." - } - } - }, - "egg": { - "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." - }, - "jar": { - "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - }, - "maven": { - "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "pypi": { - "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", - "properties": { - "package": { - "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\"." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default pip index is\nused." - } - } - }, - "requirements": { - "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" - }, - "whl": { - "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - } - } - } - }, - "max_retries": { - "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry." - }, - "min_retry_interval_millis": { - "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." - }, - "new_cluster": { - "description": "If new_cluster, a description of a new cluster that is created for each run.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - }, - "notebook_task": { - "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", - "properties": { - "base_parameters": { - "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", - "additionalproperties": { - "description": "" - } - }, - "notebook_path": { - "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." - }, - "warehouse_id": { - "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." - } - } - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", - "properties": { - "alert_on_last_attempt": { - "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run." - }, - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "pipeline_task": { - "description": "If pipeline_task, indicates that this task must execute a Pipeline.", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - }, - "pipeline_id": { - "description": "The full name of the pipeline task to execute." - } - } - }, - "python_wheel_task": { - "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", - "properties": { - "entry_point": { - "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`" - }, - "named_parameters": { - "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", - "additionalproperties": { - "description": "" - } - }, - "package_name": { - "description": "Name of the package to execute" - }, - "parameters": { - "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", - "items": { - "description": "" - } - } - } - }, - "retry_on_timeout": { - "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout." - }, - "run_if": { - "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed" - }, - "run_job_task": { - "description": "If run_job_task, indicates that this task must execute another job.", - "properties": { - "dbt_commands": { - "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", - "items": { - "description": "" - } - }, - "jar_params": { - "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "job_id": { - "description": "ID of the job to trigger." - }, - "job_parameters": { - "description": "Job-level parameters used to trigger the job.", - "additionalproperties": { - "description": "" - } - }, - "notebook_params": { - "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", - "additionalproperties": { - "description": "" - } - }, - "pipeline_params": { - "description": "", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - } - } - }, - "python_named_params": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "python_params": { - "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "spark_submit_params": { - "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "sql_params": { - "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - } - } - }, - "spark_jar_task": { - "description": "If spark_jar_task, indicates that this task must run a JAR.", - "properties": { - "jar_uri": { - "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create." - }, - "main_class_name": { - "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail." - }, - "parameters": { - "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "spark_python_task": { - "description": "If spark_python_task, indicates that this task must run a Python file.", - "properties": { - "parameters": { - "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "python_file": { - "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." - } - } - }, - "spark_submit_task": { - "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", - "properties": { - "parameters": { - "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "sql_task": { - "description": "If sql_task, indicates that this job must execute a SQL task.", - "properties": { - "alert": { - "description": "If alert, indicates that this job must refresh a SQL alert.", - "properties": { - "alert_id": { - "description": "The canonical identifier of the SQL alert." - }, - "pause_subscriptions": { - "description": "If true, the alert notifications are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, alert notifications are sent to subscribers.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "dashboard": { - "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", - "properties": { - "custom_subject": { - "description": "Subject of the email sent to subscribers of this task." - }, - "dashboard_id": { - "description": "The canonical identifier of the SQL dashboard." - }, - "pause_subscriptions": { - "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, dashboard snapshots are sent to subscriptions.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", - "properties": { - "path": { - "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." - }, - "source": { - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." - } - } - }, - "parameters": { - "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - }, - "query": { - "description": "If query, indicates that this job must execute a SQL query.", - "properties": { - "query_id": { - "description": "The canonical identifier of the SQL query." - } - } - }, - "warehouse_id": { - "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs." - } - } - }, - "task_key": { - "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset." - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout." - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout." - }, - "trigger": { - "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "file_arrival": { - "description": "File arrival trigger settings.", - "properties": { - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds" - }, - "url": { - "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location." - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "pause_status": { - "description": "Whether this trigger is paused or not." - }, - "periodic": { - "description": "Periodic trigger settings.", - "properties": { - "interval": { - "description": "The interval at which the trigger should run." - }, - "unit": { - "description": "The unit of time for the interval." - } - } - }, - "table": { - "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "table_update": { - "description": "", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - } - } - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "model_serving_endpoints": { - "description": "List of Model Serving Endpoints", - "additionalproperties": { - "description": "", - "properties": { - "config": { - "description": "The core config of the serving endpoint.", - "properties": { - "auto_capture_config": { - "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", - "properties": { - "catalog_name": { - "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled." - }, - "enabled": { - "description": "Indicates whether the inference table is enabled." - }, - "schema_name": { - "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled." - }, - "table_name_prefix": { - "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled." - } - } - }, - "served_entities": { - "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", - "items": { - "description": "", - "properties": { - "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n" - }, - "entity_version": { - "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC." - }, - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", - "properties": { - "ai21labs_config": { - "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", - "properties": { - "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - }, - "ai21labs_api_key_plaintext": { - "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - } - } - }, - "amazon_bedrock_config": { - "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", - "properties": { - "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_access_key_id_plaintext": { - "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_region": { - "description": "The AWS region to use. Bedrock has to be enabled there." - }, - "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "aws_secret_access_key_plaintext": { - "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "bedrock_provider": { - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." - } - } - }, - "anthropic_config": { - "description": "Anthropic Config. Only required if the provider is 'anthropic'.", - "properties": { - "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - }, - "anthropic_api_key_plaintext": { - "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - } - } - }, - "cohere_config": { - "description": "Cohere Config. Only required if the provider is 'cohere'.", - "properties": { - "cohere_api_base": { - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" - }, - "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - }, - "cohere_api_key_plaintext": { - "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - } - } - }, - "databricks_model_serving_config": { - "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", - "properties": { - "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_api_token_plaintext": { - "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_workspace_url": { - "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" - } - } - }, - "google_cloud_vertex_ai_config": { - "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", - "properties": { - "private_key": { - "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" - }, - "private_key_plaintext": { - "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." - }, - "project_id": { - "description": "This is the Google Cloud project id that the service account is associated with." - }, - "region": { - "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." - } - } - }, - "name": { - "description": "The name of the external model." - }, - "openai_config": { - "description": "OpenAI Config. Only required if the provider is 'openai'.", - "properties": { - "microsoft_entra_client_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" - }, - "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_client_secret_plaintext": { - "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_tenant_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" - }, - "openai_api_base": { - "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" - }, - "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_key_plaintext": { - "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_type": { - "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" - }, - "openai_api_version": { - "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n" - }, - "openai_deployment_name": { - "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n" - }, - "openai_organization": { - "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n" - } - } - }, - "palm_config": { - "description": "PaLM Config. Only required if the provider is 'palm'.", - "properties": { - "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - }, - "palm_api_key_plaintext": { - "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - } - } - }, - "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" - }, - "task": { - "description": "The task type of the external model." - } - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served entity uses to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served entity should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n" - }, - "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "served_models": { - "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", - "items": { - "description": "", - "properties": { - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served model will use to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "model_name": { - "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" - }, - "model_version": { - "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served." - }, - "name": { - "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served model should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n" - }, - "workload_type": { - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "traffic_config": { - "description": "The traffic config defining how invocations to the serving endpoint should be routed.", - "properties": { - "routes": { - "description": "The list of routes that define traffic to each served entity.", - "items": { - "description": "", - "properties": { - "served_model_name": { - "description": "The name of the served model this route configures traffic for." - }, - "traffic_percentage": { - "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive." - } - } - } - } - } - } - } - }, - "name": { - "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", - "items": { - "description": "", - "properties": { - "calls": { - "description": "Used to specify how many calls are allowed for a key within the renewal_period." - }, - "key": { - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified." - }, - "renewal_period": { - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported." - } - } - } - }, - "route_optimized": { - "description": "Enable route optimization for the serving endpoint." - }, - "tags": { - "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "Key field for a serving endpoint tag." - }, - "value": { - "description": "Optional value field for a serving endpoint tag." - } - } - } - } - } - } - }, - "models": { - "description": "List of MLflow models", - "additionalproperties": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `registered_model` was created." - }, - "description": { - "description": "Description of this `registered_model`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `registered_model` was last updated." - }, - "latest_versions": { - "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", - "items": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `model_version` was created." - }, - "current_stage": { - "description": "Current stage for this `model_version`." - }, - "description": { - "description": "Description of this `model_version`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `model_version` was last updated." - }, - "name": { - "description": "Unique name of the model" - }, - "run_id": { - "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server." - }, - "run_link": { - "description": "Run Link: Direct link to the run that generated this version" - }, - "source": { - "description": "URI indicating the location of the source model artifacts, used when creating `model_version`" - }, - "status": { - "description": "Current status of `model_version`" - }, - "status_message": { - "description": "Details on current `status`, if it is pending or failed." - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `model_version`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `model_version`." - }, - "version": { - "description": "Model's version number." - } - } - } - }, - "name": { - "description": "Unique name for the model." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `registered_model`" - } - } - } - }, - "pipelines": { - "description": "List of DLT pipelines", - "additionalproperties": { - "description": "", - "properties": { - "catalog": { - "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog." - }, - "channel": { - "description": "DLT Release Channel that specifies which version to use." - }, - "clusters": { - "description": "Cluster settings for this pipeline deployment.", - "items": { - "description": "", - "properties": { - "apply_policy_default_values": { - "description": "Note: This field won't be persisted. Only API users will check this field." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - }, - "mode": { - "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n" - } - } - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable local disk encryption for the cluster." - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "label": { - "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - } - } - } - }, - "configuration": { - "description": "String-String configuration for this pipeline execution.", - "additionalproperties": { - "description": "" - } - }, - "continuous": { - "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." - }, - "deployment": { - "description": "Deployment type of this pipeline.", - "properties": { - "kind": { - "description": "The deployment method that manages the pipeline." - }, - "metadata_file_path": { - "description": "The path to the file containing metadata about the deployment." - } - } - }, - "development": { - "description": "Whether the pipeline is in Development mode. Defaults to false." - }, - "edition": { - "description": "Pipeline product edition." - }, - "filters": { - "description": "Filters on which Pipeline packages to include in the deployed graph.", - "properties": { - "exclude": { - "description": "Paths to exclude.", - "items": { - "description": "" - } - }, - "include": { - "description": "Paths to include.", - "items": { - "description": "" - } - } - } - }, - "gateway_definition": { - "description": "The definition of a gateway pipeline to support CDC.", - "properties": { - "connection_id": { - "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." - }, - "gateway_storage_catalog": { - "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." - }, - "gateway_storage_name": { - "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" - }, - "gateway_storage_schema": { - "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." - } - } - }, - "id": { - "description": "Unique identifier for this pipeline." - }, - "ingestion_definition": { - "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", - "properties": { - "connection_name": { - "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "ingestion_gateway_id": { - "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "objects": { - "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", - "items": { - "description": "", - "properties": { - "schema": { - "description": "Select tables from a specific source schema.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store tables." - }, - "destination_schema": { - "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." - }, - "source_catalog": { - "description": "The source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Required. Schema name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "table": { - "description": "Select tables from a specific source table.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store table." - }, - "destination_schema": { - "description": "Required. Destination schema to store table." - }, - "destination_table": { - "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." - }, - "source_catalog": { - "description": "Source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Schema name in the source database. Might be optional depending on the type of source." - }, - "source_table": { - "description": "Required. Table name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - } - } - } - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "libraries": { - "description": "Libraries or code needed by this deployment.", - "items": { - "description": "", - "properties": { - "file": { - "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", - "properties": { - "path": { - "description": "The absolute path of the file." - } - } - }, - "jar": { - "description": "URI of the jar to be installed. Currently only DBFS is supported.\n" - }, - "maven": { - "description": "Specification of a maven library to be installed.\n", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "notebook": { - "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", - "properties": { - "path": { - "description": "The absolute path of the notebook." - } - } - }, - "whl": { - "description": "URI of the whl to be installed." - } - } - } - }, - "name": { - "description": "Friendly identifier for this pipeline." - }, - "notifications": { - "description": "List of notification settings for this pipeline.", - "items": { - "description": "", - "properties": { - "alerts": { - "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", - "items": { - "description": "" - } - }, - "email_recipients": { - "description": "A list of email addresses notified when a configured alert is triggered.\n", - "items": { - "description": "" - } - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "photon": { - "description": "Whether Photon is enabled for this pipeline." - }, - "serverless": { - "description": "Whether serverless compute is enabled for this pipeline." - }, - "storage": { - "description": "DBFS root directory for storing checkpoints and tables." - }, - "target": { - "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`." - }, - "trigger": { - "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", - "properties": { - "cron": { - "description": "", - "properties": { - "quartz_cron_schedule": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "manual": { - "description": "" - } - } - } - } - } - }, - "quality_monitors": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "assets_dir": { - "description": "" - }, - "baseline_table_name": { - "description": "" - }, - "custom_metrics": { - "description": "", - "items": { - "description": "", - "properties": { - "definition": { - "description": "" - }, - "input_columns": { - "description": "", - "items": { - "description": "" - } - }, - "name": { - "description": "" - }, - "output_data_type": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "data_classification_config": { - "description": "", - "properties": { - "enabled": { - "description": "" - } - } - }, - "inference_log": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "label_col": { - "description": "" - }, - "model_id_col": { - "description": "" - }, - "prediction_col": { - "description": "" - }, - "prediction_proba_col": { - "description": "" - }, - "problem_type": { - "description": "" - }, - "timestamp_col": { - "description": "" - } - } - }, - "notifications": { - "description": "", - "properties": { - "on_failure": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "on_new_classification_tag_detected": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - } - } - }, - "output_schema_name": { - "description": "" - }, - "schedule": { - "description": "", - "properties": { - "pause_status": { - "description": "" - }, - "quartz_cron_expression": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "skip_builtin_dashboard": { - "description": "" - }, - "slicing_exprs": { - "description": "", - "items": { - "description": "" - } - }, - "snapshot": { - "description": "" - }, - "time_series": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "timestamp_col": { - "description": "" - } - } - }, - "warehouse_id": { - "description": "" - } - } - } - }, - "registered_models": { - "description": "List of Registered Models", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "The name of the catalog where the schema and the registered model reside" - }, - "comment": { - "description": "The comment attached to the registered model" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "The name of the registered model" - }, - "schema_name": { - "description": "The name of the schema where the registered model resides" - }, - "storage_location": { - "description": "The storage location on the cloud under which model version data files are stored" - } - } - } - }, - "schemas": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "" - }, - "comment": { - "description": "" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "" - }, - "properties": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "storage_root": { - "description": "" - } - } - } - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - }, - "sync": { - "description": "", - "properties": { - "exclude": { - "description": "", - "items": { - "description": "" - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - }, - "paths": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "variables": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "default": { - "description": "" - }, - "description": { - "description": "" - }, - "lookup": { - "description": "", - "properties": { - "alert": { - "description": "" - }, - "cluster": { - "description": "" - }, - "cluster_policy": { - "description": "" - }, - "dashboard": { - "description": "" - }, - "instance_pool": { - "description": "" - }, - "job": { - "description": "" - }, - "metastore": { - "description": "" - }, - "pipeline": { - "description": "" - }, - "query": { - "description": "" - }, - "service_principal": { - "description": "" - }, - "warehouse": { - "description": "" - } - } - }, - "type": { - "description": "" - } - } - } - }, - "workspace": { - "description": "", - "properties": { - "artifact_path": { - "description": "" - }, - "auth_type": { - "description": "" - }, - "azure_client_id": { - "description": "" - }, - "azure_environment": { - "description": "" - }, - "azure_login_app_id": { - "description": "" - }, - "azure_tenant_id": { - "description": "" - }, - "azure_use_msi": { - "description": "" - }, - "azure_workspace_resource_id": { - "description": "" - }, - "client_id": { - "description": "" - }, - "file_path": { - "description": "" - }, - "google_service_account": { - "description": "" - }, - "host": { - "description": "" - }, - "profile": { - "description": "" - }, - "root_path": { - "description": "" - }, - "state_path": { - "description": "" - } - } - } - } - } - }, - "variables": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "default": { - "description": "" - }, - "description": { - "description": "" - }, - "lookup": { - "description": "", - "properties": { - "alert": { - "description": "" - }, - "cluster": { - "description": "" - }, - "cluster_policy": { - "description": "" - }, - "dashboard": { - "description": "" - }, - "instance_pool": { - "description": "" - }, - "job": { - "description": "" - }, - "metastore": { - "description": "" - }, - "pipeline": { - "description": "" - }, - "query": { - "description": "" - }, - "service_principal": { - "description": "" - }, - "warehouse": { - "description": "" - } - } - }, - "type": { - "description": "" - } - } - } - }, - "workspace": { - "description": "", - "properties": { - "artifact_path": { - "description": "" - }, - "auth_type": { - "description": "" - }, - "azure_client_id": { - "description": "" - }, - "azure_environment": { - "description": "" - }, - "azure_login_app_id": { - "description": "" - }, - "azure_tenant_id": { - "description": "" - }, - "azure_use_msi": { - "description": "" - }, - "azure_workspace_resource_id": { - "description": "" - }, - "client_id": { - "description": "" - }, - "file_path": { - "description": "" - }, - "google_service_account": { - "description": "" - }, - "host": { - "description": "" - }, - "profile": { - "description": "" - }, - "root_path": { - "description": "" - }, - "state_path": { - "description": "" - } - } - } - } -} \ No newline at end of file diff --git a/bundle/schema/docs_test.go b/bundle/schema/docs_test.go deleted file mode 100644 index 83ee681b..00000000 --- a/bundle/schema/docs_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package schema - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/libs/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSchemaToDocs(t *testing.T) { - jsonSchema := &jsonschema.Schema{ - Type: "object", - Description: "root doc", - Properties: map[string]*jsonschema.Schema{ - "foo": {Type: "number", Description: "foo doc"}, - "bar": {Type: "string"}, - "octave": { - Type: "object", - AdditionalProperties: &jsonschema.Schema{Type: "number"}, - Description: "octave docs", - }, - "scales": { - Type: "object", - Description: "scale docs", - Items: &jsonschema.Schema{Type: "string"}, - }, - }, - } - docs := schemaToDocs(jsonSchema) - docsJson, err := json.MarshalIndent(docs, " ", " ") - require.NoError(t, err) - - expected := - `{ - "description": "root doc", - "properties": { - "bar": { - "description": "" - }, - "foo": { - "description": "foo doc" - }, - "octave": { - "description": "octave docs", - "additionalproperties": { - "description": "" - } - }, - "scales": { - "description": "scale docs", - "items": { - "description": "" - } - } - } - }` - t.Log("[DEBUG] actual: ", string(docsJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(docsJson)) -} diff --git a/bundle/schema/embed.go b/bundle/schema/embed.go new file mode 100644 index 00000000..68f42a8e --- /dev/null +++ b/bundle/schema/embed.go @@ -0,0 +1,6 @@ +package schema + +import _ "embed" + +//go:embed jsonschema.json +var Bytes []byte diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go new file mode 100644 index 00000000..ee0b5a61 --- /dev/null +++ b/bundle/schema/embed_test.go @@ -0,0 +1,71 @@ +package schema_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/schema" + "github.com/databricks/cli/libs/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func walk(defs map[string]any, p ...string) jsonschema.Schema { + v, ok := defs[p[0]] + if !ok { + panic("not found: " + p[0]) + } + + if len(p) == 1 { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + res := jsonschema.Schema{} + err = json.Unmarshal(b, &res) + if err != nil { + panic(err) + } + return res + } + + return walk(v.(map[string]any), p[1:]...) +} + +func TestJsonSchema(t *testing.T) { + s := jsonschema.Schema{} + err := json.Unmarshal(schema.Bytes, &s) + require.NoError(t, err) + + // Assert job fields have their descriptions loaded. + resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job") + fields := []string{"name", "continuous", "deployment", "tasks", "trigger"} + for _, field := range fields { + assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description) + } + + // Assert descriptions were also loaded for a job task definition. + jobTask := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.Task") + fields = []string{"notebook_task", "spark_jar_task", "spark_python_task", "spark_submit_task", "description", "depends_on", "environment_key", "for_each_task", "existing_cluster_id"} + for _, field := range fields { + assert.NotEmpty(t, jobTask.AnyOf[0].Properties[field].Description) + } + + // Assert descriptions are loaded for pipelines + pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline") + fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"} + for _, field := range fields { + assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description) + } + + // Assert enum values are loaded + schedule := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "catalog.MonitorCronSchedule") + assert.Contains(t, schedule.AnyOf[0].Properties["pause_status"].Enum, "PAUSED") + assert.Contains(t, schedule.AnyOf[0].Properties["pause_status"].Enum, "UNPAUSED") + + providers := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.GitProvider") + assert.Contains(t, providers.Enum, "gitHub") + assert.Contains(t, providers.Enum, "bitbucketCloud") + assert.Contains(t, providers.Enum, "gitHubEnterprise") + assert.Contains(t, providers.Enum, "bitbucketServer") +} diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json new file mode 100644 index 00000000..4fe978b8 --- /dev/null +++ b/bundle/schema/jsonschema.json @@ -0,0 +1,5524 @@ +{ + "$defs": { + "bool": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "float64": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Grant": { + "anyOf": [ + { + "type": "object", + "properties": { + "principal": { + "$ref": "#/$defs/string" + }, + "privileges": { + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "privileges", + "principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Job": { + "anyOf": [ + { + "type": "object", + "properties": { + "continuous": { + "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" + }, + "deployment": { + "description": "Deployment information for jobs managed by external sources.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment" + }, + "description": { + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "edit_mode": { + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" + }, + "environments": { + "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + }, + "format": { + "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format" + }, + "git_source": { + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_clusters": { + "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + }, + "max_concurrent_runs": { + "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobNotificationSettings" + }, + "parameters": { + "description": "Job-level parameter definitions", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "queue": { + "description": "The queue settings of the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "schedule": { + "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.CronSchedule" + }, + "tags": { + "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", + "$ref": "#/$defs/map/string" + }, + "tasks": { + "description": "A list of task specifications to be executed by this job.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Task" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "trigger": { + "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings" + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifact_location": { + "description": "Location where artifacts for the experiment are stored.", + "$ref": "#/$defs/string" + }, + "creation_time": { + "description": "Creation time", + "$ref": "#/$defs/int64" + }, + "experiment_id": { + "description": "Unique identifier for the experiment.", + "$ref": "#/$defs/string" + }, + "last_update_time": { + "description": "Last update time", + "$ref": "#/$defs/int64" + }, + "lifecycle_stage": { + "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "Human readable name that identifies the experiment.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "creation_timestamp": { + "description": "Timestamp recorded when this `registered_model` was created.", + "$ref": "#/$defs/int64" + }, + "description": { + "description": "Description of this `registered_model`.", + "$ref": "#/$defs/string" + }, + "last_updated_timestamp": { + "description": "Timestamp recorded when metadata for this `registered_model` was last updated.", + "$ref": "#/$defs/int64" + }, + "latest_versions": { + "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" + }, + "name": { + "description": "Unique name for the model.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + }, + "user_id": { + "description": "User that created this `registered_model`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "anyOf": [ + { + "type": "object", + "properties": { + "config": { + "description": "The core config of the serving endpoint.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" + }, + "name": { + "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "rate_limits": { + "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" + }, + "route_optimized": { + "description": "Enable route optimization for the serving endpoint.", + "$ref": "#/$defs/bool" + }, + "tags": { + "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + "additionalProperties": false, + "required": [ + "config", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Permission": { + "anyOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/string" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Pipeline": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", + "$ref": "#/$defs/string" + }, + "channel": { + "description": "DLT Release Channel that specifies which version to use.", + "$ref": "#/$defs/string" + }, + "clusters": { + "description": "Cluster settings for this pipeline deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + }, + "configuration": { + "description": "String-String configuration for this pipeline execution.", + "$ref": "#/$defs/map/string" + }, + "continuous": { + "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`.", + "$ref": "#/$defs/bool" + }, + "deployment": { + "description": "Deployment type of this pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineDeployment" + }, + "development": { + "description": "Whether the pipeline is in Development mode. Defaults to false.", + "$ref": "#/$defs/bool" + }, + "edition": { + "description": "Pipeline product edition.", + "$ref": "#/$defs/string" + }, + "filters": { + "description": "Filters on which Pipeline packages to include in the deployed graph.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Filters" + }, + "gateway_definition": { + "description": "The definition of a gateway pipeline to support CDC.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition" + }, + "id": { + "description": "Unique identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "ingestion_definition": { + "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinition" + }, + "libraries": { + "description": "Libraries or code needed by this deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + }, + "name": { + "description": "Friendly identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "notifications": { + "description": "List of notification settings for this pipeline.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "photon": { + "description": "Whether Photon is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "serverless": { + "description": "Whether serverless compute is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "storage": { + "description": "DBFS root directory for storing checkpoints and tables.", + "$ref": "#/$defs/string" + }, + "target": { + "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`.", + "$ref": "#/$defs/string" + }, + "trigger": { + "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.QualityMonitor": { + "anyOf": [ + { + "type": "object", + "properties": { + "assets_dir": { + "description": "The directory to store monitoring assets (e.g. dashboard, metric tables).", + "$ref": "#/$defs/string" + }, + "baseline_table_name": { + "description": "Name of the baseline table from which drift metrics are computed from.\nColumns in the monitored table should also be present in the baseline table.\n", + "$ref": "#/$defs/string" + }, + "custom_metrics": { + "description": "Custom metrics to compute on the monitored table. These can be aggregate metrics, derived\nmetrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time\nwindows).\n", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + }, + "data_classification_config": { + "description": "The data classification config for the monitor.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDataClassificationConfig" + }, + "inference_log": { + "description": "Configuration for monitoring inference logs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLog" + }, + "notifications": { + "description": "The notification settings for the monitor.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorNotifications" + }, + "output_schema_name": { + "description": "Schema where output metric tables are created.", + "$ref": "#/$defs/string" + }, + "schedule": { + "description": "The schedule for automatically updating and refreshing metric tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedule" + }, + "skip_builtin_dashboard": { + "description": "Whether to skip creating a default dashboard summarizing data quality metrics.", + "$ref": "#/$defs/bool" + }, + "slicing_exprs": { + "description": "List of column expressions to slice data with for targeted analysis. The data is grouped by\neach expression independently, resulting in a separate slice for each predicate and its\ncomplements. For high-cardinality columns, only the top 100 unique values by frequency will\ngenerate slices.\n", + "$ref": "#/$defs/slice/string" + }, + "snapshot": { + "description": "Configuration for monitoring snapshot tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot" + }, + "time_series": { + "description": "Configuration for monitoring time series tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries" + }, + "warehouse_id": { + "description": "Optional argument to specify the warehouse for dashboard creation. If not specified, the first running\nwarehouse will be used.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "assets_dir", + "output_schema_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.RegisteredModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the registered model reside", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "The comment attached to the registered model", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "The name of the registered model", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the registered model resides", + "$ref": "#/$defs/string" + }, + "storage_location": { + "description": "The storage location on the cloud under which model version data files are stored", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name", + "schema_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "Name of parent catalog.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description.", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "Name of schema, relative to parent catalog.", + "$ref": "#/$defs/string" + }, + "properties": { + "$ref": "#/$defs/map/string" + }, + "storage_root": { + "description": "Storage root URL for managed tables within schema.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.Lookup": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert": { + "$ref": "#/$defs/string" + }, + "cluster": { + "$ref": "#/$defs/string" + }, + "cluster_policy": { + "$ref": "#/$defs/string" + }, + "dashboard": { + "$ref": "#/$defs/string" + }, + "instance_pool": { + "$ref": "#/$defs/string" + }, + "job": { + "$ref": "#/$defs/string" + }, + "metastore": { + "$ref": "#/$defs/string" + }, + "pipeline": { + "$ref": "#/$defs/string" + }, + "query": { + "$ref": "#/$defs/string" + }, + "service_principal": { + "$ref": "#/$defs/string" + }, + "warehouse": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.Variable": { + "type": "object", + "properties": { + "default": { + "$ref": "#/$defs/interface" + }, + "description": { + "$ref": "#/$defs/string" + }, + "lookup": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + "variable.VariableType": { + "type": "string" + } + }, + "config.Artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "build": { + "$ref": "#/$defs/string" + }, + "executable": { + "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" + }, + "files": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" + }, + "path": { + "$ref": "#/$defs/string" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.ArtifactFile": { + "anyOf": [ + { + "type": "object", + "properties": { + "source": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "source" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.ArtifactType": { + "type": "string" + }, + "config.Bundle": { + "anyOf": [ + { + "type": "object", + "properties": { + "compute_id": { + "$ref": "#/$defs/string" + }, + "databricks_cli_version": { + "$ref": "#/$defs/string" + }, + "deployment": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment" + }, + "git": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Command": { + "type": "string" + }, + "config.Deployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "fail_on_active_runs": { + "$ref": "#/$defs/bool" + }, + "lock": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Experimental": { + "anyOf": [ + { + "type": "object", + "properties": { + "pydabs": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs" + }, + "python_wheel_wrapper": { + "$ref": "#/$defs/bool" + }, + "scripts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" + }, + "use_legacy_run_as": { + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Git": { + "anyOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "origin_url": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Lock": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "$ref": "#/$defs/bool" + }, + "force": { + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Mode": { + "type": "string" + }, + "config.Presets": { + "anyOf": [ + { + "type": "object", + "properties": { + "jobs_max_concurrent_runs": { + "$ref": "#/$defs/int" + }, + "name_prefix": { + "$ref": "#/$defs/string" + }, + "pipelines_development": { + "$ref": "#/$defs/bool" + }, + "tags": { + "$ref": "#/$defs/map/string" + }, + "trigger_pause_status": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.PyDABs": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "$ref": "#/$defs/bool" + }, + "import": { + "$ref": "#/$defs/slice/string" + }, + "venv_path": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Resources": { + "anyOf": [ + { + "type": "object", + "properties": { + "experiments": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" + }, + "jobs": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job" + }, + "model_serving_endpoints": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint" + }, + "models": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel" + }, + "pipelines": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline" + }, + "quality_monitors": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor" + }, + "registered_models": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel" + }, + "schemas": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Sync": { + "anyOf": [ + { + "type": "object", + "properties": { + "exclude": { + "$ref": "#/$defs/slice/string" + }, + "include": { + "$ref": "#/$defs/slice/string" + }, + "paths": { + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Target": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifacts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + }, + "bundle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" + }, + "compute_id": { + "$ref": "#/$defs/string" + }, + "default": { + "$ref": "#/$defs/bool" + }, + "git": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" + }, + "mode": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Mode" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "presets": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" + }, + "resources": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "sync": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" + }, + "variables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + }, + "workspace": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Workspace": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifact_path": { + "$ref": "#/$defs/string" + }, + "auth_type": { + "$ref": "#/$defs/string" + }, + "azure_client_id": { + "$ref": "#/$defs/string" + }, + "azure_environment": { + "$ref": "#/$defs/string" + }, + "azure_login_app_id": { + "$ref": "#/$defs/string" + }, + "azure_tenant_id": { + "$ref": "#/$defs/string" + }, + "azure_use_msi": { + "$ref": "#/$defs/bool" + }, + "azure_workspace_resource_id": { + "$ref": "#/$defs/string" + }, + "client_id": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "google_service_account": { + "$ref": "#/$defs/string" + }, + "host": { + "$ref": "#/$defs/string" + }, + "profile": { + "$ref": "#/$defs/string" + }, + "root_path": { + "$ref": "#/$defs/string" + }, + "state_path": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "libs": { + "exec.ExecutableType": { + "type": "string" + } + } + }, + "databricks-sdk-go": { + "service": { + "catalog.MonitorCronSchedule": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Read only field that indicates whether a schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedulePauseStatus", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + "quartz_cron_expression": { + "description": "The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).\n", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "The timezone id (e.g., ``\"PST\"``) in which to evaluate the quartz expression.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorCronSchedulePauseStatus": { + "type": "string" + }, + "catalog.MonitorDataClassificationConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether data classification is enabled.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorDestination": { + "anyOf": [ + { + "type": "object", + "properties": { + "email_addresses": { + "description": "The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorInferenceLog": { + "anyOf": [ + { + "type": "object", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Currently the following static\ngranularities are supported:\n{``\"5 minutes\"``, ``\"30 minutes\"``, ``\"1 hour\"``, ``\"1 day\"``, ``\"\u003cn\u003e week(s)\"``, ``\"1 month\"``, ``\"1 year\"``}.\n", + "$ref": "#/$defs/slice/string" + }, + "label_col": { + "description": "Optional column that contains the ground truth for the prediction.", + "$ref": "#/$defs/string" + }, + "model_id_col": { + "description": "Column that contains the id of the model generating the predictions. Metrics will be computed per model id by\ndefault, and also across all model ids.\n", + "$ref": "#/$defs/string" + }, + "prediction_col": { + "description": "Column that contains the output/prediction from the model.", + "$ref": "#/$defs/string" + }, + "prediction_proba_col": { + "description": "Optional column that contains the prediction probabilities for each class in a classification problem type.\nThe values in this column should be a map, mapping each class label to the prediction probability for a given\nsample. The map should be of PySpark MapType().\n", + "$ref": "#/$defs/string" + }, + "problem_type": { + "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLogProblemType", + "enum": [ + "PROBLEM_TYPE_CLASSIFICATION", + "PROBLEM_TYPE_REGRESSION" + ] + }, + "timestamp_col": { + "description": "Column that contains the timestamps of requests. The column must be one of the following:\n- A ``TimestampType`` column\n- A column whose values can be converted to timestamps through the pyspark\n ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "model_id_col", + "prediction_col", + "problem_type", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorInferenceLogProblemType": { + "type": "string" + }, + "catalog.MonitorMetric": { + "anyOf": [ + { + "type": "object", + "properties": { + "definition": { + "description": "Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition).", + "$ref": "#/$defs/string" + }, + "input_columns": { + "description": "A list of column names in the input table the metric should be computed for.\nCan use ``\":table\"`` to indicate that the metric needs information from multiple columns.\n", + "$ref": "#/$defs/slice/string" + }, + "name": { + "description": "Name of the metric in the output tables.", + "$ref": "#/$defs/string" + }, + "output_data_type": { + "description": "The output type of the custom metric.", + "$ref": "#/$defs/string" + }, + "type": { + "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetricType", + "enum": [ + "CUSTOM_METRIC_TYPE_AGGREGATE", + "CUSTOM_METRIC_TYPE_DERIVED", + "CUSTOM_METRIC_TYPE_DRIFT" + ] + } + }, + "additionalProperties": false, + "required": [ + "definition", + "input_columns", + "name", + "output_data_type", + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorMetricType": { + "type": "string" + }, + "catalog.MonitorNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "on_failure": { + "description": "Who to send notifications to on monitor failure.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination" + }, + "on_new_classification_tag_detected": { + "description": "Who to send notifications to when new data classification tags are detected.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorSnapshot": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorTimeSeries": { + "anyOf": [ + { + "type": "object", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Currently the following static\ngranularities are supported:\n{``\"5 minutes\"``, ``\"30 minutes\"``, ``\"1 hour\"``, ``\"1 day\"``, ``\"\u003cn\u003e week(s)\"``, ``\"1 month\"``, ``\"1 year\"``}.\n", + "$ref": "#/$defs/slice/string" + }, + "timestamp_col": { + "description": "Column that contains the timestamps of requests. The column must be one of the following:\n- A ``TimestampType`` column\n- A column whose values can be converted to timestamps through the pyspark\n ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Adlsgen2Info": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AutoScale": { + "anyOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AwsAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAvailability" + }, + "ebs_volume_count": { + "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden.", + "$ref": "#/$defs/int" + }, + "ebs_volume_iops": { + "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_size": { + "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096.", + "$ref": "#/$defs/int" + }, + "ebs_volume_throughput": { + "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.EbsVolumeType" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "instance_profile_arn": { + "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists.", + "$ref": "#/$defs/string" + }, + "spot_bid_price_percent": { + "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AwsAvailability": { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", + "enum": [ + "SPOT", + "ON_DEMAND", + "SPOT_WITH_FALLBACK" + ] + }, + "compute.AzureAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAvailability" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "log_analytics_info": { + "description": "Defines values necessary to configure and run Azure Log Analytics agent", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo" + }, + "spot_bid_max_price": { + "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1.", + "$ref": "#/$defs/float64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AzureAvailability": { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE", + "SPOT_WITH_FALLBACK_AZURE" + ] + }, + "compute.ClientsTypes": { + "anyOf": [ + { + "type": "object", + "properties": { + "jobs": { + "description": "With jobs set, the cluster can be used for jobs", + "$ref": "#/$defs/bool" + }, + "notebooks": { + "description": "With notebooks set, this cluster can be used for notebooks", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.ClusterLogConf": { + "anyOf": [ + { + "type": "object", + "properties": { + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.ClusterSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DataSecurityMode": { + "type": "string", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", + "enum": [ + "NONE", + "SINGLE_USER", + "USER_ISOLATION", + "LEGACY_TABLE_ACL", + "LEGACY_PASSTHROUGH", + "LEGACY_SINGLE_USER", + "LEGACY_SINGLE_USER_STANDARD" + ] + }, + "compute.DbfsStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "dbfs destination, e.g. `dbfs:/my/path`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DockerBasicAuth": { + "anyOf": [ + { + "type": "object", + "properties": { + "password": { + "description": "Password of the user", + "$ref": "#/$defs/string" + }, + "username": { + "description": "Name of the user", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DockerImage": { + "anyOf": [ + { + "type": "object", + "properties": { + "basic_auth": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth" + }, + "url": { + "description": "URL of the docker image.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.EbsVolumeType": { + "type": "string", + "description": "The type of EBS volumes that will be launched with this cluster.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + "compute.Environment": { + "anyOf": [ + { + "type": "object", + "description": "The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task.\nIn this minimal environment spec, only pip dependencies are supported.", + "properties": { + "client": { + "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version.", + "$ref": "#/$defs/string" + }, + "dependencies": { + "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "client" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.GcpAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability" + }, + "boot_disk_size": { + "description": "boot disk size in GB", + "$ref": "#/$defs/int" + }, + "google_service_account": { + "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator.", + "$ref": "#/$defs/string" + }, + "local_ssd_count": { + "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.", + "$ref": "#/$defs/int" + }, + "use_preemptible_executors": { + "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead.", + "$ref": "#/$defs/bool" + }, + "zone_id": { + "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.GcpAvailability": { + "type": "string", + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "enum": [ + "PREEMPTIBLE_GCP", + "ON_DEMAND_GCP", + "PREEMPTIBLE_WITH_FALLBACK_GCP" + ] + }, + "compute.GcsStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "abfss": { + "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Adlsgen2Info" + }, + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo" + }, + "file": { + "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LocalFileInfo" + }, + "gcs": { + "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + }, + "volumes": { + "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo" + }, + "workspace": { + "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkspaceStorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Library": { + "anyOf": [ + { + "type": "object", + "properties": { + "cran": { + "description": "Specification of a CRAN library to be installed as part of the library", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RCranLibrary" + }, + "egg": { + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.", + "$ref": "#/$defs/string" + }, + "jar": { + "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + }, + "maven": { + "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary" + }, + "pypi": { + "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.PythonPyPiLibrary" + }, + "requirements": { + "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`", + "$ref": "#/$defs/string" + }, + "whl": { + "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.LocalFileInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "local file destination, e.g. `file:/my/local/file.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.LogAnalyticsInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "log_analytics_primary_key": { + "description": "\u003cneeds content added\u003e", + "$ref": "#/$defs/string" + }, + "log_analytics_workspace_id": { + "description": "\u003cneeds content added\u003e", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.MavenLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "coordinates": { + "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\".", + "$ref": "#/$defs/string" + }, + "exclusions": { + "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", + "$ref": "#/$defs/slice/string" + }, + "repo": { + "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "coordinates" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.PythonPyPiLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\".", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default pip index is\nused.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.RCranLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the CRAN package to install.", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default CRAN repo is used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.RuntimeEngine": { + "type": "string", + "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", + "enum": [ + "NULL", + "STANDARD", + "PHOTON" + ] + }, + "compute.S3StorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "canned_acl": { + "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs.", + "$ref": "#/$defs/string" + }, + "destination": { + "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs.", + "$ref": "#/$defs/string" + }, + "enable_encryption": { + "description": "(Optional) Flag to enable server side encryption, `false` by default.", + "$ref": "#/$defs/bool" + }, + "encryption_type": { + "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`.", + "$ref": "#/$defs/string" + }, + "endpoint": { + "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used.", + "$ref": "#/$defs/string" + }, + "kms_key": { + "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.", + "$ref": "#/$defs/string" + }, + "region": { + "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.VolumesStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.WorkloadType": { + "anyOf": [ + { + "type": "object", + "properties": { + "clients": { + "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClientsTypes" + } + }, + "additionalProperties": false, + "required": [ + "clients" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.WorkspaceStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Condition": { + "type": "string", + "enum": [ + "ANY_UPDATED", + "ALL_UPDATED" + ] + }, + "jobs.ConditionTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "left": { + "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + }, + "op": { + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTaskOp" + }, + "right": { + "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "left", + "op", + "right" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ConditionTaskOp": { + "type": "string", + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "enum": [ + "EQUAL_TO", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "NOT_EQUAL" + ] + }, + "jobs.Continuous": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.CronSchedule": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether this schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "quartz_cron_expression": { + "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required.", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.DbtTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1.", + "$ref": "#/$defs/string" + }, + "commands": { + "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", + "$ref": "#/$defs/slice/string" + }, + "profiles_directory": { + "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used.", + "$ref": "#/$defs/string" + }, + "project_directory": { + "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used.", + "$ref": "#/$defs/string" + }, + "schema": { + "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "commands" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.FileArrivalTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds", + "$ref": "#/$defs/int" + }, + "url": { + "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location.", + "$ref": "#/$defs/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ForEachTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "concurrency": { + "description": "An optional maximum allowed number of concurrent runs of the task.\nSet this value if you want to be able to execute multiple runs of the task concurrently.", + "$ref": "#/$defs/int" + }, + "inputs": { + "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", + "$ref": "#/$defs/string" + }, + "task": { + "description": "Configuration for the task that will be run for each element in the array", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + "additionalProperties": false, + "required": [ + "inputs", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Format": { + "type": "string", + "enum": [ + "SINGLE_TASK", + "MULTI_TASK" + ] + }, + "jobs.GitProvider": { + "type": "string", + "enum": [ + "gitHub", + "bitbucketCloud", + "azureDevOpsServices", + "gitHubEnterprise", + "bitbucketServer", + "gitLab", + "gitLabEnterpriseEdition", + "awsCodeCommit" + ] + }, + "jobs.GitSnapshot": { + "anyOf": [ + { + "type": "object", + "description": "Read-only state of the remote repository at the time the job was run. This field is only included on job runs.", + "properties": { + "used_commit": { + "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.GitSource": { + "anyOf": [ + { + "type": "object", + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "properties": { + "git_branch": { + "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit.", + "$ref": "#/$defs/string" + }, + "git_commit": { + "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag.", + "$ref": "#/$defs/string" + }, + "git_provider": { + "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" + }, + "git_snapshot": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot" + }, + "git_tag": { + "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", + "$ref": "#/$defs/string" + }, + "git_url": { + "description": "URL of the repository to be cloned by this job.", + "$ref": "#/$defs/string" + }, + "job_source": { + "description": "The source of the job specification in the remote repository when the job is source controlled.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource" + } + }, + "additionalProperties": false, + "required": [ + "git_provider", + "git_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobCluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "job_cluster_key": { + "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution.", + "$ref": "#/$defs/string" + }, + "new_cluster": { + "description": "If new_cluster, a description of a cluster that is created for each task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + } + }, + "additionalProperties": false, + "required": [ + "job_cluster_key", + "new_cluster" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobDeployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeploymentKind" + }, + "metadata_file_path": { + "description": "Path of the file that contains deployment metadata.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "kind" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobDeploymentKind": { + "type": "string", + "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "enum": [ + "BUNDLE" + ] + }, + "jobs.JobEditMode": { + "type": "string", + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "enum": [ + "UI_LOCKED", + "EDITABLE" + ] + }, + "jobs.JobEmailNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "anyOf": [ + { + "type": "object", + "properties": { + "environment_key": { + "description": "The key of an environment. It has to be unique within a job.", + "$ref": "#/$defs/string" + }, + "spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Environment" + } + }, + "additionalProperties": false, + "required": [ + "environment_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobNotificationSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "description": "Default value of the parameter.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "default", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobRunAs": { + "anyOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update/Reset and Submit calls. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Non-admin users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobSource": { + "anyOf": [ + { + "type": "object", + "description": "The source of the job specification in the remote repository when the job is source controlled.", + "properties": { + "dirty_state": { + "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSourceDirtyState" + }, + "import_from_git_branch": { + "description": "Name of the branch which the job is imported from.", + "$ref": "#/$defs/string" + }, + "job_config_path": { + "description": "Path of the job YAML file that contains the job specification.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "import_from_git_branch", + "job_config_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobSourceDirtyState": { + "type": "string", + "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "enum": [ + "NOT_SYNCED", + "DISCONNECTED" + ] + }, + "jobs.JobsHealthMetric": { + "type": "string", + "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Private Preview.", + "enum": [ + "RUN_DURATION_SECONDS", + "STREAMING_BACKLOG_BYTES", + "STREAMING_BACKLOG_RECORDS", + "STREAMING_BACKLOG_SECONDS", + "STREAMING_BACKLOG_FILES" + ] + }, + "jobs.JobsHealthOperator": { + "type": "string", + "description": "Specifies the operator used to compare the health metric value with the specified threshold.", + "enum": [ + "GREATER_THAN" + ] + }, + "jobs.JobsHealthRule": { + "anyOf": [ + { + "type": "object", + "properties": { + "metric": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthMetric" + }, + "op": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthOperator" + }, + "value": { + "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule.", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false, + "required": [ + "metric", + "op", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthRules": { + "anyOf": [ + { + "type": "object", + "description": "An optional set of health rules that can be defined for this job.", + "properties": { + "rules": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.NotebookTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "base_parameters": { + "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", + "$ref": "#/$defs/map/string" + }, + "notebook_path": { + "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "notebook_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PauseStatus": { + "type": "string", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + "jobs.PeriodicTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "interval": { + "description": "The interval at which the trigger should run.", + "$ref": "#/$defs/int" + }, + "unit": { + "description": "The unit of time for the interval.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfigurationTimeUnit" + } + }, + "additionalProperties": false, + "required": [ + "interval", + "unit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PeriodicTriggerConfigurationTimeUnit": { + "type": "string", + "enum": [ + "HOURS", + "DAYS", + "WEEKS" + ] + }, + "jobs.PipelineParams": { + "anyOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PipelineTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + }, + "pipeline_id": { + "description": "The full name of the pipeline task to execute.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "pipeline_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PythonWheelTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "entry_point": { + "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`", + "$ref": "#/$defs/string" + }, + "named_parameters": { + "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", + "$ref": "#/$defs/map/string" + }, + "package_name": { + "description": "Name of the package to execute", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "entry_point", + "package_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.QueueSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "If true, enable queueing for the job. This is a required field.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false, + "required": [ + "enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.RunIf": { + "type": "string", + "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "enum": [ + "ALL_SUCCESS", + "ALL_DONE", + "NONE_FAILED", + "AT_LEAST_ONE_SUCCESS", + "ALL_FAILED", + "AT_LEAST_ONE_FAILED" + ] + }, + "jobs.RunJobTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "dbt_commands": { + "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", + "$ref": "#/$defs/slice/string" + }, + "jar_params": { + "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "job_id": { + "description": "ID of the job to trigger.", + "$ref": "#/$defs/int64" + }, + "job_parameters": { + "description": "Job-level parameters used to trigger the job.", + "$ref": "#/$defs/map/string" + }, + "notebook_params": { + "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", + "$ref": "#/$defs/map/string" + }, + "pipeline_params": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" + }, + "python_named_params": { + "$ref": "#/$defs/map/string" + }, + "python_params": { + "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string" + }, + "spark_submit_params": { + "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string" + }, + "sql_params": { + "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string" + } + }, + "additionalProperties": false, + "required": [ + "job_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Source": { + "type": "string", + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "enum": [ + "WORKSPACE", + "GIT" + ] + }, + "jobs.SparkJarTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "jar_uri": { + "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.", + "$ref": "#/$defs/string" + }, + "main_class_name": { + "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SparkPythonTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "python_file": { + "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "python_file" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SparkSubmitTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert": { + "description": "If alert, indicates that this job must refresh a SQL alert.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskAlert" + }, + "dashboard": { + "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskDashboard" + }, + "file": { + "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskFile" + }, + "parameters": { + "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string" + }, + "query": { + "description": "If query, indicates that this job must execute a SQL query.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskQuery" + }, + "warehouse_id": { + "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "warehouse_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskAlert": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert_id": { + "description": "The canonical identifier of the SQL alert.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the alert notifications are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, alert notifications are sent to subscribers.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "alert_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskDashboard": { + "anyOf": [ + { + "type": "object", + "properties": { + "custom_subject": { + "description": "Subject of the email sent to subscribers of this task.", + "$ref": "#/$defs/string" + }, + "dashboard_id": { + "description": "The canonical identifier of the SQL dashboard.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, dashboard snapshots are sent to subscriptions.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "dashboard_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskFile": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskQuery": { + "anyOf": [ + { + "type": "object", + "properties": { + "query_id": { + "description": "The canonical identifier of the SQL query.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "query_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_id": { + "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TableUpdateTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "condition": { + "description": "The table(s) condition based on which to trigger a job run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Condition" + }, + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + }, + "table_names": { + "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", + "$ref": "#/$defs/slice/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Task": { + "anyOf": [ + { + "type": "object", + "properties": { + "condition_task": { + "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTask" + }, + "dbt_task": { + "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtTask" + }, + "depends_on": { + "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + }, + "description": { + "description": "An optional description for this task.", + "$ref": "#/$defs/string" + }, + "disable_auto_optimization": { + "description": "An option to disable auto optimization in serverless", + "$ref": "#/$defs/bool" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications" + }, + "environment_key": { + "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute.", + "$ref": "#/$defs/string" + }, + "existing_cluster_id": { + "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability", + "$ref": "#/$defs/string" + }, + "for_each_task": { + "description": "If for_each_task, indicates that this task must execute the nested task within it.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ForEachTask" + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_cluster_key": { + "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "max_retries": { + "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry.", + "$ref": "#/$defs/int" + }, + "min_retry_interval_millis": { + "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried.", + "$ref": "#/$defs/int" + }, + "new_cluster": { + "description": "If new_cluster, a description of a new cluster that is created for each run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + }, + "notebook_task": { + "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskNotificationSettings" + }, + "pipeline_task": { + "description": "If pipeline_task, indicates that this task must execute a Pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineTask" + }, + "python_wheel_task": { + "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask" + }, + "retry_on_timeout": { + "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout.", + "$ref": "#/$defs/bool" + }, + "run_if": { + "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunIf" + }, + "run_job_task": { + "description": "If run_job_task, indicates that this task must execute another job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask" + }, + "spark_jar_task": { + "description": "If spark_jar_task, indicates that this task must run a JAR.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask" + }, + "spark_python_task": { + "description": "If spark_python_task, indicates that this task must run a Python file.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask" + }, + "spark_submit_task": { + "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkSubmitTask" + }, + "sql_task": { + "description": "If sql_task, indicates that this job must execute a SQL task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTask" + }, + "task_key": { + "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset.", + "$ref": "#/$defs/string" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskDependency": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run.", + "$ref": "#/$defs/string" + }, + "task_key": { + "description": "The name of the task this task depends on.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskEmailNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskNotificationSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert_on_last_attempt": { + "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TriggerSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "file_arrival": { + "description": "File arrival trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.FileArrivalTriggerConfiguration" + }, + "pause_status": { + "description": "Whether this trigger is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "periodic": { + "description": "Periodic trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration" + }, + "table": { + "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration" + }, + "table_update": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Webhook": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.WebhookNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "on_duration_warning_threshold_exceeded": { + "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_failure": { + "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_start": { + "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_streaming_backlog_exceeded": { + "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_success": { + "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ExperimentTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelVersion": { + "anyOf": [ + { + "type": "object", + "properties": { + "creation_timestamp": { + "description": "Timestamp recorded when this `model_version` was created.", + "$ref": "#/$defs/int64" + }, + "current_stage": { + "description": "Current stage for this `model_version`.", + "$ref": "#/$defs/string" + }, + "description": { + "description": "Description of this `model_version`.", + "$ref": "#/$defs/string" + }, + "last_updated_timestamp": { + "description": "Timestamp recorded when metadata for this `model_version` was last updated.", + "$ref": "#/$defs/int64" + }, + "name": { + "description": "Unique name of the model", + "$ref": "#/$defs/string" + }, + "run_id": { + "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server.", + "$ref": "#/$defs/string" + }, + "run_link": { + "description": "Run Link: Direct link to the run that generated this version", + "$ref": "#/$defs/string" + }, + "source": { + "description": "URI indicating the location of the source model artifacts, used when creating `model_version`", + "$ref": "#/$defs/string" + }, + "status": { + "description": "Current status of `model_version`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionStatus", + "enum": [ + "PENDING_REGISTRATION", + "FAILED_REGISTRATION", + "READY" + ] + }, + "status_message": { + "description": "Details on current `status`, if it is pending or failed.", + "$ref": "#/$defs/string" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs for this `model_version`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" + }, + "user_id": { + "description": "User that created this `model_version`.", + "$ref": "#/$defs/string" + }, + "version": { + "description": "Model's version number.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelVersionStatus": { + "type": "string" + }, + "ml.ModelVersionTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.CronTrigger": { + "anyOf": [ + { + "type": "object", + "properties": { + "quartz_cron_schedule": { + "$ref": "#/$defs/string" + }, + "timezone_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.DeploymentKind": { + "type": "string", + "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", + "enum": [ + "BUNDLE" + ] + }, + "pipelines.FileLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the file.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Filters": { + "anyOf": [ + { + "type": "object", + "properties": { + "exclude": { + "description": "Paths to exclude.", + "$ref": "#/$defs/slice/string" + }, + "include": { + "description": "Paths to include.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "schema": { + "description": "Select tables from a specific source schema.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec" + }, + "table": { + "description": "Select tables from a specific source table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpec" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionGatewayPipelineDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "connection_id": { + "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source.", + "$ref": "#/$defs/string" + }, + "gateway_storage_catalog": { + "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location.", + "$ref": "#/$defs/string" + }, + "gateway_storage_name": { + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n", + "$ref": "#/$defs/string" + }, + "gateway_storage_schema": { + "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionPipelineDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "connection_name": { + "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name.", + "$ref": "#/$defs/string" + }, + "ingestion_gateway_id": { + "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name.", + "$ref": "#/$defs/string" + }, + "objects": { + "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.ManualTrigger": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.NotebookLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the notebook.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Notifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "alerts": { + "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", + "$ref": "#/$defs/slice/string" + }, + "email_recipients": { + "description": "A list of email addresses notified when a configured alert is triggered.\n", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "Note: This field won't be persisted. Only API users will check this field.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscale" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above.", + "$ref": "#/$defs/string" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster.", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "label": { + "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscale": { + "anyOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + }, + "mode": { + "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscaleMode", + "enum": [ + "ENHANCED", + "LEGACY" + ] + } + }, + "additionalProperties": false, + "required": [ + "max_workers", + "min_workers" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscaleMode": { + "type": "string" + }, + "pipelines.PipelineDeployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The deployment method that manages the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DeploymentKind" + }, + "metadata_file_path": { + "description": "The path to the file containing metadata about the deployment.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "file": { + "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileLibrary" + }, + "jar": { + "description": "URI of the jar to be installed. Currently only DBFS is supported.\n", + "$ref": "#/$defs/string" + }, + "maven": { + "description": "Specification of a maven library to be installed.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary" + }, + "notebook": { + "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary" + }, + "whl": { + "description": "URI of the whl to be installed.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineTrigger": { + "anyOf": [ + { + "type": "object", + "properties": { + "cron": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger" + }, + "manual": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.SchemaSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store tables.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "The source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Required. Schema name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store table.", + "$ref": "#/$defs/string" + }, + "destination_table": { + "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "Source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Schema name in the source database. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_table": { + "description": "Required. Table name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpecificConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "$ref": "#/$defs/slice/string" + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector", + "$ref": "#/$defs/bool" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScdType", + "enum": [ + "SCD_TYPE_1", + "SCD_TYPE_2" + ] + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpecificConfigScdType": { + "type": "string" + }, + "serving.Ai21LabsConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "ai21labs_api_key": { + "$ref": "#/$defs/string" + }, + "ai21labs_api_key_plaintext": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AmazonBedrockConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "aws_access_key_id": { + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_region": { + "description": "The AWS region to use. Bedrock has to be enabled there.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key": { + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "bedrock_provider": { + "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider", + "enum": [ + "anthropic", + "cohere", + "ai21labs", + "amazon" + ] + } + }, + "additionalProperties": false, + "required": [ + "aws_region", + "bedrock_provider" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AmazonBedrockConfigBedrockProvider": { + "type": "string" + }, + "serving.AnthropicConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "anthropic_api_key": { + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AutoCaptureConfigInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.CohereConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n", + "$ref": "#/$defs/string" + }, + "cohere_api_key": { + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.DatabricksModelServingConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "databricks_api_token": { + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "$ref": "#/$defs/string" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "$ref": "#/$defs/string" + }, + "databricks_workspace_url": { + "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "databricks_workspace_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointCoreConfigInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "auto_capture_config": { + "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput" + }, + "served_entities": { + "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + }, + "served_models": { + "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + }, + "traffic_config": { + "description": "The traffic config defining how invocations to the serving endpoint should be routed.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "Key field for a serving endpoint tag.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "Optional value field for a serving endpoint tag.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ExternalModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "ai21labs_config": { + "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig" + }, + "amazon_bedrock_config": { + "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig" + }, + "anthropic_config": { + "description": "Anthropic Config. Only required if the provider is 'anthropic'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig" + }, + "cohere_config": { + "description": "Cohere Config. Only required if the provider is 'cohere'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.CohereConfig" + }, + "databricks_model_serving_config": { + "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig" + }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig" + }, + "name": { + "description": "The name of the external model.", + "$ref": "#/$defs/string" + }, + "openai_config": { + "description": "OpenAI Config. Only required if the provider is 'openai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig" + }, + "palm_config": { + "description": "PaLM Config. Only required if the provider is 'palm'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig" + }, + "provider": { + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider", + "enum": [ + "ai21labs", + "anthropic", + "amazon-bedrock", + "cohere", + "databricks-model-serving", + "google-cloud-vertex-ai", + "openai", + "palm" + ] + }, + "task": { + "description": "The task type of the external model.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "provider", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ExternalModelProvider": { + "type": "string" + }, + "serving.GoogleCloudVertexAiConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "private_key": { + "$ref": "#/$defs/string" + }, + "private_key_plaintext": { + "$ref": "#/$defs/string" + }, + "project_id": { + "$ref": "#/$defs/string" + }, + "region": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.OpenAiConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "microsoft_entra_client_id": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret_plaintext": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_tenant_id": { + "$ref": "#/$defs/string" + }, + "openai_api_base": { + "$ref": "#/$defs/string" + }, + "openai_api_key": { + "$ref": "#/$defs/string" + }, + "openai_api_key_plaintext": { + "$ref": "#/$defs/string" + }, + "openai_api_type": { + "$ref": "#/$defs/string" + }, + "openai_api_version": { + "$ref": "#/$defs/string" + }, + "openai_deployment_name": { + "$ref": "#/$defs/string" + }, + "openai_organization": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.PaLmConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "palm_api_key": { + "$ref": "#/$defs/string" + }, + "palm_api_key_plaintext": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimit": { + "anyOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int" + }, + "key": { + "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey", + "enum": [ + "user", + "endpoint" + ] + }, + "renewal_period": { + "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod", + "enum": [ + "minute" + ] + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimitKey": { + "type": "string" + }, + "serving.RateLimitRenewalPeriod": { + "type": "string" + }, + "serving.Route": { + "anyOf": [ + { + "type": "object", + "properties": { + "served_model_name": { + "description": "The name of the served model this route configures traffic for.", + "$ref": "#/$defs/string" + }, + "traffic_percentage": { + "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "served_model_name", + "traffic_percentage" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "entity_name": { + "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n", + "$ref": "#/$defs/string" + }, + "entity_version": { + "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.", + "$ref": "#/$defs/string" + }, + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "external_model": { + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", + "$ref": "#/$defs/string" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n", + "$ref": "#/$defs/string" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served entity should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n", + "$ref": "#/$defs/string" + }, + "workload_type": { + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served model will use to access AWS resources.", + "$ref": "#/$defs/string" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, + "model_name": { + "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n", + "$ref": "#/$defs/string" + }, + "model_version": { + "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n", + "$ref": "#/$defs/string" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served model should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize", + "enum": [ + "Small", + "Medium", + "Large" + ] + }, + "workload_type": { + "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType", + "enum": [ + "CPU", + "GPU_SMALL", + "GPU_MEDIUM", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + } + }, + "additionalProperties": false, + "required": [ + "model_name", + "model_version", + "scale_to_zero_enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInputWorkloadSize": { + "type": "string" + }, + "serving.ServedModelInputWorkloadType": { + "type": "string" + }, + "serving.TrafficConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "routes": { + "description": "The list of routes that define traffic to each served entity.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "int": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "int64": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "interface": {}, + "map": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Job": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Job" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModel": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Pipeline": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Pipeline" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.QualityMonitor": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.QualityMonitor" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.RegisteredModel": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.RegisteredModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Schema": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Schema" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.Variable": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Variable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "config.Artifact": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Artifact" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Command": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Command" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Target": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Target" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "string": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "slice": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Grant": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Grant" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Permission": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Permission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "config.ArtifactFile": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactFile" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + }, + "databricks-sdk-go": { + "service": { + "catalog.MonitorMetric": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Library": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Library" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobCluster": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthRule": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Task": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskDependency": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Webhook": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ExperimentTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelVersion": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelVersionTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Notifications": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimit": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.Route": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInput": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "string": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "string": { + "type": "string" + } + }, + "type": "object", + "properties": { + "artifacts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + }, + "bundle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" + }, + "experimental": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Experimental" + }, + "include": { + "$ref": "#/$defs/slice/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "presets": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" + }, + "resources": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "sync": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" + }, + "targets": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target" + }, + "variables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + }, + "workspace": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go deleted file mode 100644 index 0d896b87..00000000 --- a/bundle/schema/openapi.go +++ /dev/null @@ -1,293 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/databricks/cli/libs/jsonschema" -) - -type OpenapiReader struct { - // OpenAPI spec to read schemas from. - OpenapiSpec *Specification - - // In-memory cache of schemas read from the OpenAPI spec. - memo map[string]jsonschema.Schema -} - -const SchemaPathPrefix = "#/components/schemas/" - -// Read a schema directly from the OpenAPI spec. -func (reader *OpenapiReader) readOpenapiSchema(path string) (jsonschema.Schema, error) { - schemaKey := strings.TrimPrefix(path, SchemaPathPrefix) - - // return early if we already have a computed schema - memoSchema, ok := reader.memo[schemaKey] - if ok { - return memoSchema, nil - } - - // check path is present in openapi spec - openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey] - if !ok { - return jsonschema.Schema{}, fmt.Errorf("schema with path %s not found in openapi spec", path) - } - - // convert openapi schema to the native schema struct - bytes, err := json.Marshal(*openapiSchema) - if err != nil { - return jsonschema.Schema{}, err - } - jsonSchema := jsonschema.Schema{} - err = json.Unmarshal(bytes, &jsonSchema) - if err != nil { - return jsonschema.Schema{}, err - } - - // A hack to convert a map[string]interface{} to *Schema - // We rely on the type of a AdditionalProperties in downstream functions - // to do reference interpolation - _, ok = jsonSchema.AdditionalProperties.(map[string]interface{}) - if ok { - b, err := json.Marshal(jsonSchema.AdditionalProperties) - if err != nil { - return jsonschema.Schema{}, err - } - additionalProperties := &jsonschema.Schema{} - err = json.Unmarshal(b, additionalProperties) - if err != nil { - return jsonschema.Schema{}, err - } - jsonSchema.AdditionalProperties = additionalProperties - } - - // store read schema into memo - reader.memo[schemaKey] = jsonSchema - - return jsonSchema, nil -} - -// Resolve all nested "$ref" references in the schema. This function unrolls a single -// level of "$ref" in the schema and calls into traverseSchema to resolve nested references. -// Thus this function and traverseSchema are mutually recursive. -// -// This function is safe against reference loops. If a reference loop is detected, an error -// is returned. -func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { - if root.Reference == nil { - return reader.traverseSchema(root, tracker) - } - key := *root.Reference - - // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper - // support for recursive types is added to the docs generator. PR: https://github.com/databricks/cli/pull/1204 - if strings.Contains(key, "ForEachTask") { - return root, nil - } - - if tracker.hasCycle(key) { - // self reference loops can be supported however the logic is non-trivial because - // cross refernce loops are not allowed (see: http://json-schema.org/understanding-json-schema/structuring.html#recursion) - return nil, fmt.Errorf("references loop detected") - } - ref := *root.Reference - description := root.Description - tracker.push(ref, ref) - - // Mark reference nil, so we do not traverse this again. This is tracked - // in the memo - root.Reference = nil - - // unroll one level of reference. - selfRef, err := reader.readOpenapiSchema(ref) - if err != nil { - return nil, err - } - root = &selfRef - root.Description = description - - // traverse again to find new references - root, err = reader.traverseSchema(root, tracker) - if err != nil { - return nil, err - } - tracker.pop(ref) - return root, err -} - -// Traverse the nested properties of the schema to resolve "$ref" references. This function -// and safeResolveRefs are mutually recursive. -func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { - // case primitive (or invalid) - if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType { - return root, nil - } - // only root references are resolved - if root.Reference != nil { - return reader.safeResolveRefs(root, tracker) - } - // case struct - if len(root.Properties) > 0 { - for k, v := range root.Properties { - childSchema, err := reader.safeResolveRefs(v, tracker) - if err != nil { - return nil, err - } - root.Properties[k] = childSchema - } - } - // case array - if root.Items != nil { - itemsSchema, err := reader.safeResolveRefs(root.Items, tracker) - if err != nil { - return nil, err - } - root.Items = itemsSchema - } - // case map - additionalProperties, ok := root.AdditionalProperties.(*jsonschema.Schema) - if ok && additionalProperties != nil { - valueSchema, err := reader.safeResolveRefs(additionalProperties, tracker) - if err != nil { - return nil, err - } - root.AdditionalProperties = valueSchema - } - return root, nil -} - -func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema, error) { - root, err := reader.readOpenapiSchema(path) - if err != nil { - return nil, err - } - tracker := newTracker() - tracker.push(path, path) - resolvedRoot, err := reader.safeResolveRefs(&root, tracker) - if err != nil { - return nil, tracker.errWithTrace(err.Error(), "") - } - return resolvedRoot, nil -} - -func (reader *OpenapiReader) jobsDocs() (*Docs, error) { - jobSettingsSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "jobs.JobSettings") - if err != nil { - return nil, err - } - jobDocs := schemaToDocs(jobSettingsSchema) - // TODO: add description for id if needed. - // Tracked in https://github.com/databricks/cli/issues/242 - jobsDocs := &Docs{ - Description: "List of Databricks jobs", - AdditionalProperties: jobDocs, - } - return jobsDocs, nil -} - -func (reader *OpenapiReader) pipelinesDocs() (*Docs, error) { - pipelineSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "pipelines.PipelineSpec") - if err != nil { - return nil, err - } - pipelineDocs := schemaToDocs(pipelineSpecSchema) - // TODO: Two fields in resources.Pipeline have the json tag id. Clarify the - // semantics and then add a description if needed. (https://github.com/databricks/cli/issues/242) - pipelinesDocs := &Docs{ - Description: "List of DLT pipelines", - AdditionalProperties: pipelineDocs, - } - return pipelinesDocs, nil -} - -func (reader *OpenapiReader) experimentsDocs() (*Docs, error) { - experimentSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "ml.Experiment") - if err != nil { - return nil, err - } - experimentDocs := schemaToDocs(experimentSpecSchema) - experimentsDocs := &Docs{ - Description: "List of MLflow experiments", - AdditionalProperties: experimentDocs, - } - return experimentsDocs, nil -} - -func (reader *OpenapiReader) modelsDocs() (*Docs, error) { - modelSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "ml.Model") - if err != nil { - return nil, err - } - modelDocs := schemaToDocs(modelSpecSchema) - modelsDocs := &Docs{ - Description: "List of MLflow models", - AdditionalProperties: modelDocs, - } - return modelsDocs, nil -} - -func (reader *OpenapiReader) modelServingEndpointsDocs() (*Docs, error) { - modelServingEndpointsSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "serving.CreateServingEndpoint") - if err != nil { - return nil, err - } - modelServingEndpointsDocs := schemaToDocs(modelServingEndpointsSpecSchema) - modelServingEndpointsAllDocs := &Docs{ - Description: "List of Model Serving Endpoints", - AdditionalProperties: modelServingEndpointsDocs, - } - return modelServingEndpointsAllDocs, nil -} - -func (reader *OpenapiReader) registeredModelDocs() (*Docs, error) { - registeredModelsSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "catalog.CreateRegisteredModelRequest") - if err != nil { - return nil, err - } - registeredModelsDocs := schemaToDocs(registeredModelsSpecSchema) - registeredModelsAllDocs := &Docs{ - Description: "List of Registered Models", - AdditionalProperties: registeredModelsDocs, - } - return registeredModelsAllDocs, nil -} - -func (reader *OpenapiReader) ResourcesDocs() (*Docs, error) { - jobsDocs, err := reader.jobsDocs() - if err != nil { - return nil, err - } - pipelinesDocs, err := reader.pipelinesDocs() - if err != nil { - return nil, err - } - experimentsDocs, err := reader.experimentsDocs() - if err != nil { - return nil, err - } - modelsDocs, err := reader.modelsDocs() - if err != nil { - return nil, err - } - modelServingEndpointsDocs, err := reader.modelServingEndpointsDocs() - if err != nil { - return nil, err - } - registeredModelsDocs, err := reader.registeredModelDocs() - if err != nil { - return nil, err - } - - return &Docs{ - Description: "Collection of Databricks resources to deploy.", - Properties: map[string]*Docs{ - "jobs": jobsDocs, - "pipelines": pipelinesDocs, - "experiments": experimentsDocs, - "models": modelsDocs, - "model_serving_endpoints": modelServingEndpointsDocs, - "registered_models": registeredModelsDocs, - }, - }, nil -} diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go deleted file mode 100644 index 4d393cf3..00000000 --- a/bundle/schema/openapi_test.go +++ /dev/null @@ -1,493 +0,0 @@ -package schema - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/libs/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadSchemaForObject(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mango": { - "type": "object", - "description": "a mango for my schema", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "properties": { - "foo": { - "$ref": "#/components/schemas/foo" - } - } - } - } - } - } - ` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mango": { - "type": "object", - "description": "a mango for my schema", - "properties": { - "foo": { - "type": "number" - } - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReadSchemaForArray(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "fruits": { - "type": "object", - "description": "fruits that are cool", - "items": { - "description": "some papayas, because papayas are fruits too", - "$ref": "#/components/schemas/papaya" - } - }, - "papaya": { - "type": "number" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "items": { - "type": "number", - "description": "some papayas, because papayas are fruits too" - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReadSchemaForMap(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "fruits": { - "type": "object", - "description": "fruits that are meh", - "additionalProperties": { - "description": "watermelons. watermelons.", - "$ref": "#/components/schemas/watermelon" - } - }, - "watermelon": { - "type": "number" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are meh", - "additionalProperties": { - "type": "number", - "description": "watermelons. watermelons." - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestRootReferenceIsResolved(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "abc": { - "type": "string" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - schema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - fruitsSchemaJson, err := json.MarshalIndent(schema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "foo fighters fighting fruits", - "properties": { - "abc": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestSelfReferenceLoopErrors(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "bar": { - "type": "object", - "$ref": "#/components/schemas/foo" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - _, err = reader.readResolvedSchema("#/components/schemas/fruits") - assert.ErrorContains(t, err, "references loop detected. traversal trace: -> #/components/schemas/fruits -> #/components/schemas/foo") -} - -func TestCrossReferenceLoopErrors(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "bar": { - "type": "object", - "$ref": "#/components/schemas/fruits" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - _, err = reader.readResolvedSchema("#/components/schemas/fruits") - assert.ErrorContains(t, err, "references loop detected. traversal trace: -> #/components/schemas/fruits -> #/components/schemas/foo") -} - -func TestReferenceResolutionForMapInObject(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "additionalProperties": { - "description": "a single mango", - "$ref": "#/components/schemas/foo" - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "additionalProperties": { - "type": "number", - "description": "a single mango" - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReferenceResolutionForArrayInObject(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "items": { - "description": "a single mango", - "$ref": "#/components/schemas/foo" - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "items": { - "type": "number", - "description": "a single mango" - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "properties": { - "guava": { - "type": "object", - "description": "Guava is a fruit", - "$ref": "#/components/schemas/foo" - }, - "mango": { - "type": "object", - "description": "What is a mango?", - "$ref": "#/components/schemas/foo" - } - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "properties": { - "guava": { - "type": "number", - "description": "Guava is a fruit" - }, - "mango": { - "type": "number", - "description": "What is a mango?" - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go deleted file mode 100644 index ac0b4f2e..00000000 --- a/bundle/schema/schema.go +++ /dev/null @@ -1,287 +0,0 @@ -package schema - -import ( - "container/list" - "fmt" - "reflect" - "strings" - - "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/jsonschema" -) - -// Fields tagged "readonly" should not be emitted in the schema as they are -// computed at runtime, and should not be assigned a value by the bundle author. -const readonlyTag = "readonly" - -// Annotation for internal bundle fields that should not be exposed to customers. -// Fields can be tagged as "internal" to remove them from the generated schema. -const internalTag = "internal" - -// Annotation for bundle fields that have been deprecated. -// Fields tagged as "deprecated" are removed/omitted from the generated schema. -const deprecatedTag = "deprecated" - -// This function translates golang types into json schema. Here is the mapping -// between json schema types and golang types -// -// - GolangType -> Javascript type / Json Schema2 -// -// - bool -> boolean -// -// - string -> string -// -// - int (all variants) -> number -// -// - float (all variants) -> number -// -// - map[string]MyStruct -> { type: object, additionalProperties: {}} -// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#additional-properties -// -// - []MyStruct -> {type: array, items: {}} -// for details visit: https://json-schema.org/understanding-json-schema/reference/array.html#items -// -// - []MyStruct -> {type: object, properties: {}, additionalProperties: false} -// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties -func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { - tracker := newTracker() - schema, err := safeToSchema(golangType, docs, "", tracker) - if err != nil { - return nil, tracker.errWithTrace(err.Error(), "root") - } - return schema, nil -} - -func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { - switch golangType.Kind() { - case reflect.Bool: - return jsonschema.BooleanType, nil - case reflect.String: - return jsonschema.StringType, nil - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: - - return jsonschema.NumberType, nil - case reflect.Struct: - return jsonschema.ObjectType, nil - case reflect.Map: - if golangType.Key().Kind() != reflect.String { - return jsonschema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) - } - return jsonschema.ObjectType, nil - case reflect.Array, reflect.Slice: - return jsonschema.ArrayType, nil - default: - return jsonschema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) - } -} - -// A wrapper over toSchema function to: -// 1. Detect cycles in the bundle config struct. -// 2. Update tracker -// -// params: -// -// - golangType: Golang type to generate json schema for -// -// - docs: Contains documentation to be injected into the generated json schema -// -// - traceId: An identifier for the current type, to trace recursive traversal. -// Its value is the first json tag in case of struct fields and "" in other cases -// like array, map or no json tags -// -// - tracker: Keeps track of types / traceIds seen during recursive traversal -func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*jsonschema.Schema, error) { - // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper - // support for recursive types is added to the schema generator. PR: https://github.com/databricks/cli/pull/1204 - if traceId == "for_each_task" { - return &jsonschema.Schema{ - Type: jsonschema.ObjectType, - }, nil - } - - // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA - // There are mechanisms to deal with cycles though recursive identifiers in json - // schema. However if we use them, we would need to make sure we are able to detect - // cycles where two properties (directly or indirectly) pointing to each other - // - // see: https://json-schema.org/understanding-json-schema/structuring.html#recursion - // for details - if tracker.hasCycle(golangType) { - return nil, fmt.Errorf("cycle detected") - } - - tracker.push(golangType, traceId) - props, err := toSchema(golangType, docs, tracker) - if err != nil { - return nil, err - } - tracker.pop(golangType) - return props, nil -} - -// This function returns all member fields of the provided type. -// If the type has embedded (aka anonymous) fields, this function traverses -// those in a breadth first manner -func getStructFields(golangType reflect.Type) []reflect.StructField { - fields := []reflect.StructField{} - bfsQueue := list.New() - - for i := 0; i < golangType.NumField(); i++ { - bfsQueue.PushBack(golangType.Field(i)) - } - for bfsQueue.Len() > 0 { - front := bfsQueue.Front() - field := front.Value.(reflect.StructField) - bfsQueue.Remove(front) - - if !field.Anonymous { - fields = append(fields, field) - continue - } - - fieldType := field.Type - if fieldType.Kind() == reflect.Pointer { - fieldType = fieldType.Elem() - } - - for i := 0; i < fieldType.NumField(); i++ { - bfsQueue.PushBack(fieldType.Field(i)) - } - } - return fields -} - -func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschema.Schema, error) { - // *Struct and Struct generate identical json schemas - if golangType.Kind() == reflect.Pointer { - return safeToSchema(golangType.Elem(), docs, "", tracker) - } - if golangType.Kind() == reflect.Interface { - return &jsonschema.Schema{}, nil - } - - rootJavascriptType, err := jsonSchemaType(golangType) - if err != nil { - return nil, err - } - jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} - - // If the type is a non-string primitive, then we allow it to be a string - // provided it's a pure variable reference (ie only a single variable reference). - if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType { - jsonSchema = &jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: rootJavascriptType, - }, - { - Type: jsonschema.StringType, - Pattern: dynvar.VariableRegex, - }, - }, - } - } - - if docs != nil { - jsonSchema.Description = docs.Description - } - - // case array/slice - if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { - elemGolangType := golangType.Elem() - elemJavascriptType, err := jsonSchemaType(elemGolangType) - if err != nil { - return nil, err - } - var childDocs *Docs - if docs != nil { - childDocs = docs.Items - } - elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker) - if err != nil { - return nil, err - } - jsonSchema.Items = &jsonschema.Schema{ - Type: elemJavascriptType, - Properties: elemProps.Properties, - AdditionalProperties: elemProps.AdditionalProperties, - Items: elemProps.Items, - Required: elemProps.Required, - } - } - - // case map - if golangType.Kind() == reflect.Map { - if golangType.Key().Kind() != reflect.String { - return nil, fmt.Errorf("only string keyed maps allowed") - } - var childDocs *Docs - if docs != nil { - childDocs = docs.AdditionalProperties - } - jsonSchema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) - if err != nil { - return nil, err - } - } - - // case struct - if golangType.Kind() == reflect.Struct { - children := getStructFields(golangType) - properties := map[string]*jsonschema.Schema{} - required := []string{} - for _, child := range children { - bundleTag := child.Tag.Get("bundle") - // Fields marked as "readonly", "internal" or "deprecated" are skipped - // while generating the schema - if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag { - continue - } - - // get child json tags - childJsonTag := strings.Split(child.Tag.Get("json"), ",") - childName := childJsonTag[0] - - // skip children that have no json tags, the first json tag is "" - // or the first json tag is "-" - if childName == "" || childName == "-" { - continue - } - - // get docs for the child if they exist - var childDocs *Docs - if docs != nil { - if val, ok := docs.Properties[childName]; ok { - childDocs = val - } - } - - // compute if the child is a required field. Determined by the - // presence of "omitempty" in the json tags - hasOmitEmptyTag := false - for i := 1; i < len(childJsonTag); i++ { - if childJsonTag[i] == "omitempty" { - hasOmitEmptyTag = true - } - } - if !hasOmitEmptyTag { - required = append(required, childName) - } - - // compute Schema.Properties for the child recursively - fieldProps, err := safeToSchema(child.Type, childDocs, childName, tracker) - if err != nil { - return nil, err - } - properties[childName] = fieldProps - } - - jsonSchema.AdditionalProperties = false - jsonSchema.Properties = properties - jsonSchema.Required = required - } - - return jsonSchema, nil -} diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go deleted file mode 100644 index 6d9df0cc..00000000 --- a/bundle/schema/schema_test.go +++ /dev/null @@ -1,1900 +0,0 @@ -package schema - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestIntSchema(t *testing.T) { - var elemInt int - - expected := - `{ - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }` - - schema, err := New(reflect.TypeOf(elemInt), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBooleanSchema(t *testing.T) { - var elem bool - - expected := - `{ - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }` - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStringSchema(t *testing.T) { - var elem string - - expected := - `{ - "type": "string" - }` - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfPrimitivesSchema(t *testing.T) { - type Foo struct { - IntVal int `json:"int_val"` - Int8Val int8 `json:"int8_val"` - Int16Val int16 `json:"int16_val"` - Int32Val int32 `json:"int32_val"` - Int64Val int64 `json:"int64_val"` - - UIntVal uint `json:"uint_val"` - Uint8Val uint8 `json:"uint8_val"` - Uint16Val uint16 `json:"uint16_val"` - Uint32Val uint32 `json:"uint32_val"` - Uint64Val uint64 `json:"uint64_val"` - - Float32Val float32 `json:"float32_val"` - Float64Val float64 `json:"float64_val"` - - StringVal string `json:"string_val"` - - BoolVal bool `json:"bool_val"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bool_val": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "float32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "float64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int16_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int8_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "string_val": { - "type": "string" - }, - "uint16_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint8_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "int_val", - "int8_val", - "int16_val", - "int32_val", - "int64_val", - "uint_val", - "uint8_val", - "uint16_val", - "uint32_val", - "uint64_val", - "float32_val", - "float64_val", - "string_val", - "bool_val" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfStructsSchema(t *testing.T) { - type Bar struct { - A int `json:"a"` - B string `json:"b,string"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - type MyStruct struct { - Foo Foo `json:"foo"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "foo": { - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "a": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "b": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "a", - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - } - }, - "additionalProperties": false, - "required": [ - "foo" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfMapsSchema(t *testing.T) { - type Bar struct { - MyMap map[string]int `json:"my_map"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "my_map": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_map" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfSliceSchema(t *testing.T) { - type Bar struct { - MySlice []string `json:"my_slice"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "my_slice": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "my_slice" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfPrimitivesSchema(t *testing.T) { - var elem map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfStructSchema(t *testing.T) { - type Foo struct { - MyInt int `json:"my_int"` - } - - var elem map[string]Foo - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "my_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_int" - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfMapSchema(t *testing.T) { - var elem map[string]map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfSliceSchema(t *testing.T) { - var elem map[string][]string - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfPrimitivesSchema(t *testing.T) { - var elem []float32 - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "number" - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfSliceSchema(t *testing.T) { - var elem [][]string - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfMapSchema(t *testing.T) { - var elem []map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfStructSchema(t *testing.T) { - type Foo struct { - MyInt int `json:"my_int"` - } - - var elem []Foo - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "object", - "properties": { - "my_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_int" - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestEmbeddedStructSchema(t *testing.T) { - type Location struct { - Country string `json:"country"` - State string `json:"state,omitempty"` - } - - type Person struct { - Name string `json:"name"` - Age int `json:"age,omitempty"` - Home Location `json:"home"` - } - - type Plot struct { - Events map[string]Person `json:"events"` - } - - type Story struct { - Plot Plot `json:"plot"` - *Person - Location - } - - elem := Story{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "country": { - "type": "string" - }, - "home": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "country" - ] - }, - "name": { - "type": "string" - }, - "plot": { - "type": "object", - "properties": { - "events": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "home": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "country" - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name", - "home" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "events" - ] - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "plot", - "name", - "home", - "country" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestErrorWithTrace(t *testing.T) { - tracker := newTracker() - dummyType := reflect.TypeOf(struct{}{}) - err := tracker.errWithTrace("with empty trace", "root") - assert.ErrorContains(t, err, "with empty trace. traversal trace: root") - - tracker.push(dummyType, "resources") - err = tracker.errWithTrace("with depth = 1", "root") - assert.ErrorContains(t, err, "with depth = 1. traversal trace: root -> resources") - - tracker.push(dummyType, "pipelines") - tracker.push(dummyType, "datasets") - err = tracker.errWithTrace("with depth = 4", "root") - assert.ErrorContains(t, err, "with depth = 4. traversal trace: root -> resources -> pipelines -> datasets") -} - -func TestNonAnnotatedFieldsAreSkipped(t *testing.T) { - type MyStruct struct { - Foo string - Bar int `json:"bar"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDashFieldsAreSkipped(t *testing.T) { - type MyStruct struct { - Foo string `json:"-"` - Bar int `json:"bar"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestPointerInStructSchema(t *testing.T) { - - type Bar struct { - PtrVal2 *int `json:"ptr_val2"` - } - - type Foo struct { - PtrInt *int `json:"ptr_int"` - PtrString *string `json:"ptr_string"` - FloatVal float32 `json:"float_val"` - PtrBar *Bar `json:"ptr_bar"` - Bar *Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "ptr_val2": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "ptr_val2" - ] - }, - "float_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "ptr_bar": { - "type": "object", - "properties": { - "ptr_val2": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "ptr_val2" - ] - }, - "ptr_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "ptr_string": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "ptr_int", - "ptr_string", - "float_val", - "ptr_bar", - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestGenericSchema(t *testing.T) { - type Person struct { - Name string `json:"name"` - Age int `json:"age,omitempty"` - } - - type Plot struct { - Stakes []string `json:"stakes"` - Deaths []Person `json:"deaths"` - Murders map[string]Person `json:"murders"` - } - - type Wedding struct { - Hidden string `json:","` - Groom Person `json:"groom"` - Bride Person `json:"bride"` - Plots []Plot `json:"plots"` - } - - type Story struct { - Hero *Person `json:"hero"` - Villian Person `json:"villian,omitempty"` - Weddings []Wedding `json:"weddings"` - } - - elem := Story{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "hero": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "villian": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "weddings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "bride": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "groom": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "plots": { - "type": "array", - "items": { - "type": "object", - "properties": { - "deaths": { - "type": "array", - "items": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - } - }, - "murders": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - } - }, - "stakes": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "stakes", - "deaths", - "murders" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "groom", - "bride", - "plots" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "hero", - "weddings" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) { - - type Papaya struct { - A int `json:"a,string,omitempty"` - B string `json:"b"` - } - - type MyStruct struct { - Foo string `json:"-,omitempty"` - Bar int `json:"bar"` - Apple int `json:"apple,omitempty"` - Mango int `json:",omitempty"` - Guava int `json:","` - Papaya *Papaya `json:"papaya,"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "papaya": { - "type": "object", - "properties": { - "a": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "b": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar", - "papaya" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForObject(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_struct": { - Description: "docs for my struct", - Properties: map[string]*Docs{ - "a": { - Description: "docs for a", - }, - "c": { - Description: "docs for c which does not exist on my_struct", - }, - }, - }, - }, - } - - type MyStruct struct { - A string `json:"a"` - B int `json:"b"` - } - - type Root struct { - MyStruct *MyStruct `json:"my_struct"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_struct": { - "type": "object", - "description": "docs for my struct", - "properties": { - "a": { - "type": "string", - "description": "docs for a" - }, - "b": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "a", - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "my_struct" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForSlice(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_slice": { - Description: "docs for my slice", - Items: &Docs{ - Properties: map[string]*Docs{ - "guava": { - Description: "docs for guava", - }, - "pineapple": { - Description: "docs for pineapple", - }, - "watermelon": { - Description: "docs for watermelon which does not exist in schema", - }, - }, - }, - }, - }, - } - - type Bar struct { - Guava int `json:"guava"` - Pineapple int `json:"pineapple"` - } - - type Root struct { - MySlice []Bar `json:"my_slice"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_slice": { - "type": "array", - "description": "docs for my slice", - "items": { - "type": "object", - "properties": { - "guava": { - "description": "docs for guava", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pineapple": { - "description": "docs for pineapple", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "guava", - "pineapple" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_slice" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForMap(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_map": { - Description: "docs for my map", - AdditionalProperties: &Docs{ - Properties: map[string]*Docs{ - "apple": { - Description: "docs for apple", - }, - "mango": { - Description: "docs for mango", - }, - "watermelon": { - Description: "docs for watermelon which does not exist in schema", - }, - "papaya": { - Description: "docs for papaya which does not exist in schema", - }, - }, - }, - }, - }, - } - - type Foo struct { - Apple int `json:"apple"` - Mango int `json:"mango"` - } - - type Root struct { - MyMap map[string]*Foo `json:"my_map"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_map": { - "type": "object", - "description": "docs for my map", - "additionalProperties": { - "type": "object", - "properties": { - "apple": { - "description": "docs for apple", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "mango": { - "description": "docs for mango", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "apple", - "mango" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_map" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForTopLevelPrimitive(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_val": { - Description: "docs for my val", - }, - }, - } - - type Root struct { - MyVal int `json:"my_val"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_val": { - "description": "docs for my val", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_val" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestErrorOnMapWithoutStringKey(t *testing.T) { - type Foo struct { - Bar map[int]string `json:"bar"` - } - elem := Foo{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "only strings map keys are valid. key type: int") -} - -func TestErrorIfStructRefersToItself(t *testing.T) { - type Foo struct { - MyFoo *Foo `json:"my_foo"` - } - - elem := Foo{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_foo") -} - -func TestErrorIfStructHasLoop(t *testing.T) { - type Apple struct { - MyVal int `json:"my_val"` - MyMango struct { - MyGuava struct { - MyPapaya struct { - MyApple *Apple `json:"my_apple"` - } `json:"my_papaya"` - } `json:"my_guava"` - } `json:"my_mango"` - } - - elem := Apple{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_mango -> my_guava -> my_papaya -> my_apple") -} - -func TestInterfaceGeneratesEmptySchema(t *testing.T) { - type Foo struct { - Apple int `json:"apple"` - Mango interface{} `json:"mango"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "mango": {} - }, - "additionalProperties": false, - "required": [ - "apple", - "mango" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBundleReadOnlytag(t *testing.T) { - type Pokemon struct { - Pikachu string `json:"pikachu" bundle:"readonly"` - Raichu string `json:"raichu"` - } - - type Foo struct { - Pokemon *Pokemon `json:"pokemon"` - Apple int `json:"apple"` - Mango string `json:"mango" bundle:"readonly"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pokemon": { - "type": "object", - "properties": { - "raichu": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "raichu" - ] - } - }, - "additionalProperties": false, - "required": [ - "pokemon", - "apple" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBundleInternalTag(t *testing.T) { - type Pokemon struct { - Pikachu string `json:"pikachu" bundle:"internal"` - Raichu string `json:"raichu"` - } - - type Foo struct { - Pokemon *Pokemon `json:"pokemon"` - Apple int `json:"apple"` - Mango string `json:"mango" bundle:"internal"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pokemon": { - "type": "object", - "properties": { - "raichu": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "raichu" - ] - } - }, - "additionalProperties": false, - "required": [ - "pokemon", - "apple" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} diff --git a/bundle/schema/spec.go b/bundle/schema/spec.go deleted file mode 100644 index fdc31a4c..00000000 --- a/bundle/schema/spec.go +++ /dev/null @@ -1,11 +0,0 @@ -package schema - -import "github.com/databricks/cli/libs/jsonschema" - -type Specification struct { - Components *Components `json:"components"` -} - -type Components struct { - Schemas map[string]*jsonschema.Schema `json:"schemas,omitempty"` -} diff --git a/bundle/schema/tracker.go b/bundle/schema/tracker.go deleted file mode 100644 index ace6559b..00000000 --- a/bundle/schema/tracker.go +++ /dev/null @@ -1,53 +0,0 @@ -package schema - -import ( - "container/list" - "fmt" -) - -type tracker struct { - // Nodes encountered in current path during the recursive traversal. Used to - // check for cycles - seenNodes map[interface{}]struct{} - - // List of node names encountered in order in current path during the recursive traversal. - // Used to hydrate errors with path to the exact node where error occured. - // - // NOTE: node and node names can be the same - listOfNodes *list.List -} - -func newTracker() *tracker { - return &tracker{ - seenNodes: map[interface{}]struct{}{}, - listOfNodes: list.New(), - } -} - -func (t *tracker) errWithTrace(prefix string, initTrace string) error { - traceString := initTrace - curr := t.listOfNodes.Front() - for curr != nil { - if curr.Value.(string) != "" { - traceString += " -> " + curr.Value.(string) - } - curr = curr.Next() - } - return fmt.Errorf(prefix + ". traversal trace: " + traceString) -} - -func (t *tracker) hasCycle(node interface{}) bool { - _, ok := t.seenNodes[node] - return ok -} - -func (t *tracker) push(node interface{}, name string) { - t.seenNodes[node] = struct{}{} - t.listOfNodes.PushBack(name) -} - -func (t *tracker) pop(nodeType interface{}) { - back := t.listOfNodes.Back() - t.listOfNodes.Remove(back) - delete(t.seenNodes, nodeType) -} diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 813aebba..480618ed 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -1,13 +1,8 @@ package bundle import ( - "encoding/json" - "reflect" - - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/schema" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/jsonschema" "github.com/spf13/cobra" ) @@ -19,32 +14,8 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - // Load embedded schema descriptions. - docs, err := schema.LoadBundleDescriptions() - if err != nil { - return err - } - - // Generate the JSON schema from the bundle configuration struct in Go. - schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) - if err != nil { - return err - } - - // Target variable value overrides can be primitives, maps or sequences. - // Set an empty schema for them. - err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) - if err != nil { - return err - } - - // Print the JSON schema to stdout. - result, err := json.MarshalIndent(schema, "", " ") - if err != nil { - return err - } - cmd.OutOrStdout().Write(result) - return nil + _, err := cmd.OutOrStdout().Write(schema.Bytes) + return err } return cmd diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index bf160fa8..338ac8ce 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,9 +6,7 @@ import ( "github.com/databricks/cli/libs/dyn" ) -const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}` - -var re = regexp.MustCompile(VariableRegex) +var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go new file mode 100644 index 00000000..18a2b3ba --- /dev/null +++ b/libs/jsonschema/from_type.go @@ -0,0 +1,356 @@ +package jsonschema + +import ( + "container/list" + "fmt" + "maps" + "path" + "reflect" + "slices" + "strings" +) + +var skipTags = []string{ + // Fields tagged "readonly" should not be emitted in the schema as they are + // computed at runtime, and should not be assigned a value by the bundle author. + "readonly", + + // Annotation for internal bundle fields that should not be exposed to customers. + // Fields can be tagged as "internal" to remove them from the generated schema. + "internal", + + // Annotation for bundle fields that have been deprecated. + // Fields tagged as "deprecated" are omitted from the generated schema. + "deprecated", +} + +type constructor struct { + // Map of typ.PkgPath() + "." + typ.Name() to the schema for that type. + // Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings + definitions map[string]Schema + + // Map of typ.PkgPath() + "." + typ.Name() to the corresponding type. Used to + // track types that have been seen to avoid infinite recursion. + seen map[string]reflect.Type + + // The root type for which the schema is being generated. + root reflect.Type +} + +// JSON pointers use "/" as a delimiter to represent nested objects. This means +// we would instead need to use "~1" to represent "/" if we wish to refer to a +// key in a JSON object with a "/" in it. Instead of doing that we replace "/" with an +// additional level of nesting in the output map. Thus the $refs in the generated +// JSON schema can contain "/" without any issues. +// see: https://datatracker.ietf.org/doc/html/rfc6901 +// +// For example: +// {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} +// the $ref for "value" would be "#/$defs/a/b/c" in the generated JSON schema. +func (c *constructor) Definitions() map[string]any { + defs := maps.Clone(c.definitions) + + // Remove the root type from the definitions. We don't need to include it in + // the definitions because it will be inlined as the root of the generated JSON schema. + delete(defs, typePath(c.root)) + + if len(defs) == 0 { + return nil + } + + res := make(map[string]any) + for k, v := range defs { + parts := strings.Split(k, "/") + cur := res + for i, p := range parts { + // Set the value for the last part. + if i == len(parts)-1 { + cur[p] = v + break + } + + // For all but the last part, create a new map value to add a level + // of nesting. + if _, ok := cur[p]; !ok { + cur[p] = make(map[string]any) + } + cur = cur[p].(map[string]any) + } + } + + return res +} + +// FromType converts a [reflect.Type] to a [Schema]. Nodes in the final JSON +// schema are guaranteed to be one level deep, which is done using defining $defs +// for every Go type and referring them using $ref in the corresponding node in +// the JSON schema. +// +// fns is a list of transformation functions that will be applied in order to all $defs +// in the schema. +func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (Schema, error) { + c := constructor{ + definitions: make(map[string]Schema), + seen: make(map[string]reflect.Type), + root: typ, + } + + _, err := c.walk(typ) + if err != nil { + return Schema{}, err + } + + for _, fn := range fns { + for k := range c.definitions { + c.definitions[k] = fn(c.seen[k], c.definitions[k]) + } + } + + res := c.definitions[typePath(typ)] + res.Definitions = c.Definitions() + return res, nil +} + +// typePath computes a unique string representation of the type. $ref in the generated +// JSON schema will refer to this path. See TestTypePath for examples outputs. +func typePath(typ reflect.Type) string { + // Pointers have a typ.Name() of "". Dereference them to get the underlying type. + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + + if typ.Kind() == reflect.Interface { + return "interface" + } + + // Recursively call typePath, to handle slices of slices / maps. + if typ.Kind() == reflect.Slice { + return path.Join("slice", typePath(typ.Elem())) + } + + if typ.Kind() == reflect.Map { + if typ.Key().Kind() != reflect.String { + panic(fmt.Sprintf("found map with non-string key: %v", typ.Key())) + } + + // Recursively call typePath, to handle maps of maps / slices. + return path.Join("map", typePath(typ.Elem())) + } + + switch { + case typ.PkgPath() != "" && typ.Name() != "": + return typ.PkgPath() + "." + typ.Name() + case typ.Name() != "": + return typ.Name() + default: + // Invariant. This function should return a non-empty string + // for all types. + panic("unexpected empty type name for type: " + typ.String()) + } +} + +// Walk the Go type, generating $defs for every type encountered, and populating +// the corresponding $ref in the JSON schema. +func (c *constructor) walk(typ reflect.Type) (string, error) { + // Dereference pointers if necessary. + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + + typPath := typePath(typ) + + // Return early if the type has already been seen, to avoid infinite recursion. + if _, ok := c.seen[typPath]; ok { + return typPath, nil + } + c.seen[typPath] = typ + + var s Schema + var err error + + switch typ.Kind() { + case reflect.Struct: + s, err = c.fromTypeStruct(typ) + case reflect.Slice: + s, err = c.fromTypeSlice(typ) + case reflect.Map: + s, err = c.fromTypeMap(typ) + case reflect.String: + s = Schema{Type: StringType} + case reflect.Bool: + s = Schema{Type: BooleanType} + case reflect.Int, reflect.Int32, reflect.Int64: + s = Schema{Type: IntegerType} + case reflect.Float32, reflect.Float64: + s = Schema{Type: NumberType} + case reflect.Interface: + // We cannot determine the schema for fields of interface type just based + // on the type information. Thus we'll set the empty schema here and allow + // arbitrary values. + s = Schema{} + default: + return "", fmt.Errorf("unsupported type: %s", typ.Kind()) + } + if err != nil { + return "", err + } + + // Store the computed JSON schema for the type. + c.definitions[typPath] = s + return typPath, nil +} + +// This function returns all member fields of the provided type. +// If the type has embedded (aka anonymous) fields, this function traverses +// those in a breadth first manner +// +// BFS is important because we want the a field defined at a higher level embedded +// struct to be given preference over a field with the same name defined at a lower +// level embedded struct. For example see: TestHigherLevelEmbeddedFieldIsInSchema +func getStructFields(typ reflect.Type) []reflect.StructField { + fields := []reflect.StructField{} + bfsQueue := list.New() + + for i := 0; i < typ.NumField(); i++ { + bfsQueue.PushBack(typ.Field(i)) + } + for bfsQueue.Len() > 0 { + front := bfsQueue.Front() + field := front.Value.(reflect.StructField) + bfsQueue.Remove(front) + + if !field.Anonymous { + fields = append(fields, field) + continue + } + + fieldType := field.Type + + // Embedded types can only be struct{} or pointer to struct{}. Multiple + // levels of pointers are not allowed by the Go compiler. So we only + // dereference pointers once. + if fieldType.Kind() == reflect.Pointer { + fieldType = fieldType.Elem() + } + + for i := 0; i < fieldType.NumField(); i++ { + bfsQueue.PushBack(fieldType.Field(i)) + } + } + return fields +} + +func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { + if typ.Kind() != reflect.Struct { + return Schema{}, fmt.Errorf("expected struct, got %s", typ.Kind()) + } + + res := Schema{ + Type: ObjectType, + Properties: make(map[string]*Schema), + Required: []string{}, + AdditionalProperties: false, + } + + structFields := getStructFields(typ) + for _, structField := range structFields { + bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") + // Fields marked as "readonly", "internal" or "deprecated" are skipped + // while generating the schema + skip := false + for _, tag := range skipTags { + if slices.Contains(bundleTags, tag) { + skip = true + break + } + } + if skip { + continue + } + + jsonTags := strings.Split(structField.Tag.Get("json"), ",") + fieldName := jsonTags[0] + // Do not include fields in the schema that will not be serialized during + // JSON marshalling. + if fieldName == "" || fieldName == "-" || !structField.IsExported() { + continue + } + + // Skip property if it is already present in the schema. + // This can happen if the same field is defined multiple times across + // a tree of embedded structs. For example see: TestHigherLevelEmbeddedFieldIsInSchema + if _, ok := res.Properties[fieldName]; ok { + continue + } + + // "omitempty" tags in the Go SDK structs represent fields that not are + // required to be present in the API payload. Thus its absence in the + // tags list indicates that the field is required. + if !slices.Contains(jsonTags, "omitempty") { + res.Required = append(res.Required, fieldName) + } + + // Walk the fields of the struct. + typPath, err := c.walk(structField.Type) + if err != nil { + return Schema{}, err + } + + // For every property in the struct, add a $ref to the corresponding + // $defs block. + refPath := path.Join("#/$defs", typPath) + res.Properties[fieldName] = &Schema{ + Reference: &refPath, + } + } + + return res, nil +} + +func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { + if typ.Kind() != reflect.Slice { + return Schema{}, fmt.Errorf("expected slice, got %s", typ.Kind()) + } + + res := Schema{ + Type: ArrayType, + } + + // Walk the slice element type. + typPath, err := c.walk(typ.Elem()) + if err != nil { + return Schema{}, err + } + + refPath := path.Join("#/$defs", typPath) + + // Add a $ref to the corresponding $defs block for the slice element type. + res.Items = &Schema{ + Reference: &refPath, + } + return res, nil +} + +func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { + if typ.Kind() != reflect.Map { + return Schema{}, fmt.Errorf("expected map, got %s", typ.Kind()) + } + + res := Schema{ + Type: ObjectType, + } + + // Walk the map value type. + typPath, err := c.walk(typ.Elem()) + if err != nil { + return Schema{}, err + } + + refPath := path.Join("#/$defs", typPath) + + // Add a $ref to the corresponding $defs block for the map value type. + res.AdditionalProperties = &Schema{ + Reference: &refPath, + } + return res, nil +} diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go new file mode 100644 index 00000000..174ffad8 --- /dev/null +++ b/libs/jsonschema/from_type_test.go @@ -0,0 +1,521 @@ +package jsonschema + +import ( + "reflect" + "testing" + + "github.com/databricks/cli/libs/jsonschema/test_types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFromTypeBasic(t *testing.T) { + type myStruct struct { + S string `json:"s"` + I *int `json:"i,omitempty"` + V interface{} `json:"v,omitempty"` + TriplePointer ***int `json:"triple_pointer,omitempty"` + + // These fields should be ignored in the resulting schema. + NotAnnotated string + DashedTag string `json:"-"` + InternalTagged string `json:"internal_tagged" bundle:"internal"` + DeprecatedTagged string `json:"deprecated_tagged" bundle:"deprecated"` + ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"` + } + + strRef := "#/$defs/string" + boolRef := "#/$defs/bool" + intRef := "#/$defs/int" + interfaceRef := "#/$defs/interface" + + tcases := []struct { + name string + typ reflect.Type + expected Schema + }{ + { + name: "int", + typ: reflect.TypeOf(int(0)), + expected: Schema{ + Type: "integer", + }, + }, + { + name: "string", + typ: reflect.TypeOf(string("")), + expected: Schema{ + Type: "string", + }, + }, + { + name: "bool", + typ: reflect.TypeOf(bool(true)), + expected: Schema{ + Type: "boolean", + }, + }, + { + name: "float64", + typ: reflect.TypeOf(float64(0)), + expected: Schema{ + Type: "number", + }, + }, + { + name: "struct", + typ: reflect.TypeOf(myStruct{}), + expected: Schema{ + Type: "object", + Definitions: map[string]any{ + "interface": Schema{}, + "string": Schema{ + Type: "string", + }, + "int": Schema{ + Type: "integer", + }, + }, + Properties: map[string]*Schema{ + "s": { + Reference: &strRef, + }, + "i": { + Reference: &intRef, + }, + "v": { + Reference: &interfaceRef, + }, + "triple_pointer": { + Reference: &intRef, + }, + }, + AdditionalProperties: false, + Required: []string{"s"}, + }, + }, + { + name: "slice", + typ: reflect.TypeOf([]bool{}), + expected: Schema{ + Type: "array", + Definitions: map[string]any{ + "bool": Schema{ + Type: "boolean", + }, + }, + Items: &Schema{ + Reference: &boolRef, + }, + }, + }, + { + name: "map", + typ: reflect.TypeOf(map[string]int{}), + expected: Schema{ + Type: "object", + Definitions: map[string]any{ + "int": Schema{ + Type: "integer", + }, + }, + AdditionalProperties: &Schema{ + Reference: &intRef, + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + s, err := FromType(tc.typ, nil) + assert.NoError(t, err) + assert.Equal(t, tc.expected, s) + }) + } +} + +func TestGetStructFields(t *testing.T) { + type InnerEmbeddedStruct struct { + InnerField float64 + } + + type EmbeddedStructOne struct { + FieldOne int + + *InnerEmbeddedStruct + } + + type EmbeddedStructTwo struct { + FieldTwo bool + } + + type MyStruct struct { + *EmbeddedStructOne + EmbeddedStructTwo + + OuterField string + } + + fields := getStructFields(reflect.TypeOf(MyStruct{})) + assert.Len(t, fields, 4) + assert.Equal(t, "OuterField", fields[0].Name) + assert.Equal(t, "FieldOne", fields[1].Name) + + // InnerField occurring after FieldTwo ensures BFS as opposed to DFS traversal. + assert.Equal(t, "FieldTwo", fields[2].Name) + assert.Equal(t, "InnerField", fields[3].Name) +} + +func TestHigherLevelEmbeddedFieldIsInSchema(t *testing.T) { + type Inner struct { + Override string `json:"override,omitempty"` + } + + type EmbeddedOne struct { + Inner + } + + type EmbeddedTwo struct { + Override int `json:"override,omitempty"` + } + + type Outer struct { + EmbeddedOne + EmbeddedTwo + } + + intRef := "#/$defs/int" + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "int": Schema{ + Type: "integer", + }, + }, + Properties: map[string]*Schema{ + "override": { + Reference: &intRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + } + + s, err := FromType(reflect.TypeOf(Outer{}), nil) + require.NoError(t, err) + assert.Equal(t, expected, s) +} + +func TestFromTypeNested(t *testing.T) { + type Inner struct { + S string `json:"s"` + } + + type Outer struct { + I string `json:"i"` + Inner Inner `json:"inner"` + } + + innerRef := "#/$defs/github.com/databricks/cli/libs/jsonschema.Inner" + strRef := "#/$defs/string" + + expectedDefinitions := map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema.Inner": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "s": { + Reference: &strRef, + }, + }, + AdditionalProperties: false, + Required: []string{"s"}, + }, + }, + }, + }, + }, + "string": Schema{ + Type: "string", + }, + } + + tcases := []struct { + name string + typ reflect.Type + expected Schema + }{ + { + name: "struct in struct", + typ: reflect.TypeOf(Outer{}), + expected: Schema{ + Type: "object", + Definitions: expectedDefinitions, + Properties: map[string]*Schema{ + "i": { + Reference: &strRef, + }, + "inner": { + Reference: &innerRef, + }, + }, + AdditionalProperties: false, + Required: []string{"i", "inner"}, + }, + }, + { + name: "struct as a map value", + typ: reflect.TypeOf(map[string]*Inner{}), + expected: Schema{ + Type: "object", + Definitions: expectedDefinitions, + AdditionalProperties: &Schema{ + Reference: &innerRef, + }, + }, + }, + { + name: "struct as a slice element", + typ: reflect.TypeOf([]Inner{}), + expected: Schema{ + Type: "array", + Definitions: expectedDefinitions, + Items: &Schema{ + Reference: &innerRef, + }, + }, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + s, err := FromType(tc.typ, nil) + assert.NoError(t, err) + assert.Equal(t, tc.expected, s) + }) + } +} + +func TestFromTypeRecursive(t *testing.T) { + fooRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Foo" + barRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Bar" + + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema": map[string]any{ + "test_types.Bar": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "foo": { + Reference: &fooRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + "test_types.Foo": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "bar": { + Reference: &barRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + }, + }, + }, + }, + }, + }, + Properties: map[string]*Schema{ + "foo": { + Reference: &fooRef, + }, + }, + AdditionalProperties: false, + Required: []string{"foo"}, + } + + s, err := FromType(reflect.TypeOf(test_types.Outer{}), nil) + assert.NoError(t, err) + assert.Equal(t, expected, s) +} + +func TestFromTypeSelfReferential(t *testing.T) { + selfRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Self" + stringRef := "#/$defs/string" + + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema": map[string]any{ + "test_types.Self": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "self": { + Reference: &selfRef, + }, + "s": { + Reference: &stringRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + }, + }, + }, + }, + }, + "string": Schema{ + Type: "string", + }, + }, + Properties: map[string]*Schema{ + "self": { + Reference: &selfRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + } + + s, err := FromType(reflect.TypeOf(test_types.OuterSelf{}), nil) + assert.NoError(t, err) + assert.Equal(t, expected, s) +} + +func TestFromTypeError(t *testing.T) { + // Maps with non-string keys should panic. + type mapOfInts map[int]int + assert.PanicsWithValue(t, "found map with non-string key: int", func() { + FromType(reflect.TypeOf(mapOfInts{}), nil) + }) + + // Unsupported types should return an error. + _, err := FromType(reflect.TypeOf(complex64(0)), nil) + assert.EqualError(t, err, "unsupported type: complex64") +} + +func TestFromTypeFunctionsArg(t *testing.T) { + type myStruct struct { + S string `json:"s"` + } + + strRef := "#/$defs/string" + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "string": Schema{ + Type: "string", + Description: "a string", + Enum: []any{"a", "b", "c"}, + }, + }, + Properties: map[string]*Schema{ + "s": { + Reference: &strRef, + }, + }, + AdditionalProperties: false, + Required: []string{"s"}, + } + + addDescription := func(typ reflect.Type, s Schema) Schema { + if typ.Kind() != reflect.String { + return s + } + s.Description = "a string" + return s + } + + addEnums := func(typ reflect.Type, s Schema) Schema { + if typ.Kind() != reflect.String { + return s + } + s.Enum = []any{"a", "b", "c"} + return s + } + + s, err := FromType(reflect.TypeOf(myStruct{}), []func(reflect.Type, Schema) Schema{ + addDescription, + addEnums, + }) + assert.NoError(t, err) + assert.Equal(t, expected, s) +} + +func TestTypePath(t *testing.T) { + type myStruct struct{} + + tcases := []struct { + typ reflect.Type + path string + }{ + { + typ: reflect.TypeOf(""), + path: "string", + }, + { + typ: reflect.TypeOf(int(0)), + path: "int", + }, + { + typ: reflect.TypeOf(true), + path: "bool", + }, + { + typ: reflect.TypeOf(float64(0)), + path: "float64", + }, + { + typ: reflect.TypeOf(myStruct{}), + path: "github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf([]int{}), + path: "slice/int", + }, + { + typ: reflect.TypeOf(map[string]int{}), + path: "map/int", + }, + { + typ: reflect.TypeOf([]myStruct{}), + path: "slice/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf([][]map[string]map[string]myStruct{}), + path: "slice/slice/map/map/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf(map[string]myStruct{}), + path: "map/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + } + + for _, tc := range tcases { + t.Run(tc.typ.String(), func(t *testing.T) { + assert.Equal(t, tc.path, typePath(tc.typ)) + }) + } + + // Maps with non-string keys should panic. + assert.PanicsWithValue(t, "found map with non-string key: int", func() { + typePath(reflect.TypeOf(map[int]int{})) + }) +} diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index f1e223ec..7690ec2f 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -6,7 +6,6 @@ import ( "os" "regexp" "slices" - "strings" "github.com/databricks/cli/internal/build" "golang.org/x/mod/semver" @@ -14,6 +13,10 @@ import ( // defines schema for a json object type Schema struct { + // Definitions that can be reused and referenced throughout the schema. The + // syntax for a reference is $ref: #/$defs/ + Definitions map[string]any `json:"$defs,omitempty"` + // Type of the object Type Type `json:"type,omitempty"` @@ -63,7 +66,7 @@ type Schema struct { Extension // Schema that must match any of the schemas in the array - AnyOf []*Schema `json:"anyOf,omitempty"` + AnyOf []Schema `json:"anyOf,omitempty"` } // Default value defined in a JSON Schema, represented as a string. @@ -82,41 +85,6 @@ func (s *Schema) ParseString(v string) (any, error) { return fromString(v, s.Type) } -func (s *Schema) getByPath(path string) (*Schema, error) { - p := strings.Split(path, ".") - - res := s - for _, node := range p { - if node == "*" { - res = res.AdditionalProperties.(*Schema) - continue - } - var ok bool - res, ok = res.Properties[node] - if !ok { - return nil, fmt.Errorf("property %q not found in schema. Query path: %s", node, path) - } - } - return res, nil -} - -func (s *Schema) GetByPath(path string) (Schema, error) { - v, err := s.getByPath(path) - if err != nil { - return Schema{}, err - } - return *v, nil -} - -func (s *Schema) SetByPath(path string, v Schema) error { - dst, err := s.getByPath(path) - if err != nil { - return err - } - *dst = v - return nil -} - type Type string const ( diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go index c365cf23..cf1f1276 100644 --- a/libs/jsonschema/schema_test.go +++ b/libs/jsonschema/schema_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestSchemaValidateTypeNames(t *testing.T) { @@ -306,92 +305,3 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) { err = s.validate() assert.NoError(t, err) } - -func testSchema() *Schema { - return &Schema{ - Type: "object", - Properties: map[string]*Schema{ - "int_val": { - Type: "integer", - Default: int64(123), - }, - "string_val": { - Type: "string", - }, - "object_val": { - Type: "object", - Properties: map[string]*Schema{ - "bar": { - Type: "string", - Default: "baz", - }, - }, - AdditionalProperties: &Schema{ - Type: "object", - Properties: map[string]*Schema{ - "foo": { - Type: "string", - Default: "zab", - }, - }, - }, - }, - }, - } - -} - -func TestSchemaGetByPath(t *testing.T) { - s := testSchema() - - ss, err := s.GetByPath("int_val") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: IntegerType, - Default: int64(123), - }, ss) - - ss, err = s.GetByPath("string_val") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - }, ss) - - ss, err = s.GetByPath("object_val.bar") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "baz", - }, ss) - - ss, err = s.GetByPath("object_val.*.foo") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "zab", - }, ss) -} - -func TestSchemaSetByPath(t *testing.T) { - s := testSchema() - - err := s.SetByPath("int_val", Schema{ - Type: IntegerType, - Default: int64(456), - }) - require.NoError(t, err) - assert.Equal(t, int64(456), s.Properties["int_val"].Default) - - err = s.SetByPath("object_val.*.foo", Schema{ - Type: StringType, - Default: "zooby", - }) - require.NoError(t, err) - - ns, err := s.GetByPath("object_val.*.foo") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "zooby", - }, ns) -} diff --git a/libs/jsonschema/test_types/test_types.go b/libs/jsonschema/test_types/test_types.go new file mode 100644 index 00000000..75e81595 --- /dev/null +++ b/libs/jsonschema/test_types/test_types.go @@ -0,0 +1,25 @@ +package test_types + +// Recursive types cannot be defined inline without making them anonymous, +// so we define them here instead. +type Foo struct { + Bar *Bar `json:"bar,omitempty"` +} + +type Bar struct { + Foo Foo `json:"foo,omitempty"` +} + +type Outer struct { + Foo Foo `json:"foo"` +} + +type Self struct { + Self *Self `json:"self,omitempty"` + + S string `json:"s,omitempty"` +} + +type OuterSelf struct { + Self Self `json:"self,omitempty"` +} diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 73b47f28..ab9dbeb5 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -461,7 +461,7 @@ func TestPromptIsSkippedAnyOf(t *testing.T) { Default: "hello-world", Extension: jsonschema.Extension{ SkipPromptIf: &jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ { Properties: map[string]*jsonschema.Schema{ "abc": { From 5d2c0e3885070359c22310dd79fa5fe594c8e0f5 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 10 Sep 2024 20:19:34 +0530 Subject: [PATCH 33/78] Alias variables block in the `Target` struct (#1748) ## Changes This PR aliases and overrides the schema associated with the variables block in `target` to allow for directly specifying a variable value in the JSON schema (without an levels of nesting). This is needed because this direct value is resolved by dynamically parsing the configuration tree. https://github.com/databricks/cli/blob/ca6332a5a4325aff1be848536f45d13bd74d93b3/bundle/config/root.go#L424 ## Tests Existing unit tests. --- bundle/config/root_test.go | 2 +- bundle/config/target.go | 21 +++++++++++++++- bundle/config/variable/variable.go | 5 ++++ bundle/internal/schema/main.go | 16 ++++++++++++ bundle/schema/jsonschema.json | 39 +++++++++++++++++++++++++++++- 5 files changed, 80 insertions(+), 3 deletions(-) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index c95e6e86..d2c7a9b1 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -139,7 +139,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, Targets: map[string]*Target{ "development": { - Variables: map[string]*variable.Variable{ + Variables: map[string]*variable.TargetVariable{ "foo": { Default: "bar", Description: "wrong", diff --git a/bundle/config/target.go b/bundle/config/target.go index a2ef4d73..fc6ba7b5 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -38,7 +38,26 @@ type Target struct { // Override default values or lookup name for defined variables // Does not permit defining new variables or redefining existing ones // in the scope of an target - Variables map[string]*variable.Variable `json:"variables,omitempty"` + // + // There are two valid ways to define a variable override in a target: + // 1. Direct value override. We normalize this to the variable.Variable + // struct format when loading the configuration YAML: + // + // variables: + // foo: "value" + // + // 2. Override matching the variable.Variable struct. + // + // variables: + // foo: + // default: "value" + // + // OR + // + // variables: + // foo: + // lookup: "resource_name" + Variables map[string]*variable.TargetVariable `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index ba94f9c8..2362ad10 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -16,6 +16,11 @@ const ( VariableTypeComplex VariableType = "complex" ) +// We alias it here to override the JSON schema associated with a variable value +// in a target override. This is because we allow for directly specifying the value +// in addition to the variable.Variable struct format in a target override. +type TargetVariable Variable + // An input variable for the bundle config type Variable struct { // A type of the variable. This is used to validate the value of the variable diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 3c1fb5da..4a237147 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -21,6 +21,22 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. return s } + // The variables block in a target override allows for directly specifying + // the value of the variable. + if typ == reflect.TypeOf(variable.TargetVariable{}) { + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ + // We keep the original schema so that autocomplete suggestions + // continue to work. + s, + // All values are valid for a variable value, be it primitive types + // like string/bool or complex ones like objects/arrays. Thus we override + // the schema to allow all valid JSON values. + {}, + }, + } + } + switch s.Type { case jsonschema.ArrayType, jsonschema.ObjectType: // arrays and objects can have complex variable values specified. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4fe978b8..2db1a5ab 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -642,6 +642,29 @@ } ] }, + "variable.TargetVariable": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "$ref": "#/$defs/interface" + }, + "description": { + "$ref": "#/$defs/string" + }, + "lookup": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + {} + ] + }, "variable.Variable": { "type": "object", "properties": { @@ -995,7 +1018,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable" }, "workspace": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" @@ -4993,6 +5016,20 @@ } ] }, + "variable.TargetVariable": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.TargetVariable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "variable.Variable": { "anyOf": [ { From c61358407fade291678ff9e68f9ccd982ea99124 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:45:56 +0530 Subject: [PATCH 34/78] Add end to end integration tests for bundle JSON schema (#1726) --- .github/workflows/push.yml | 11 ++- .../internal/schema/testdata/fail/basic.yml | 3 + .../fail/invalid_enum_value_in_job.yml | 4 ++ .../fail/invalid_enum_value_in_model.yml | 6 ++ .../fail/invalid_reference_in_job.yml | 8 +++ .../fail/invalid_reference_in_model.yml | 5 ++ .../fail/required_field_missing_in_job.yml | 9 +++ .../testdata/fail/unknown_field_in_job.yml | 5 ++ .../testdata/fail/unknown_field_in_model.yml | 6 ++ .../testdata/fail/unknown_top_level_field.yml | 1 + .../testdata/pass/artifact_references.yml | 11 +++ .../internal/schema/testdata/pass/basic.yml | 2 + .../testdata/pass/direct_value_in_target.yml | 4 ++ bundle/internal/schema/testdata/pass/job.yml | 63 ++++++++++++++++ bundle/internal/schema/testdata/pass/ml.yml | 72 +++++++++++++++++++ .../schema/testdata/pass/pipeline.yml | 54 ++++++++++++++ .../schema/testdata/pass/quality_monitor.yml | 16 +++++ .../schema/testdata/pass/run_job_task.yml | 56 +++++++++++++++ .../internal/schema/testdata/pass/schema.yml | 24 +++++++ 19 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 bundle/internal/schema/testdata/fail/basic.yml create mode 100644 bundle/internal/schema/testdata/fail/invalid_enum_value_in_job.yml create mode 100644 bundle/internal/schema/testdata/fail/invalid_enum_value_in_model.yml create mode 100644 bundle/internal/schema/testdata/fail/invalid_reference_in_job.yml create mode 100644 bundle/internal/schema/testdata/fail/invalid_reference_in_model.yml create mode 100644 bundle/internal/schema/testdata/fail/required_field_missing_in_job.yml create mode 100644 bundle/internal/schema/testdata/fail/unknown_field_in_job.yml create mode 100644 bundle/internal/schema/testdata/fail/unknown_field_in_model.yml create mode 100644 bundle/internal/schema/testdata/fail/unknown_top_level_field.yml create mode 100644 bundle/internal/schema/testdata/pass/artifact_references.yml create mode 100644 bundle/internal/schema/testdata/pass/basic.yml create mode 100644 bundle/internal/schema/testdata/pass/direct_value_in_target.yml create mode 100644 bundle/internal/schema/testdata/pass/job.yml create mode 100644 bundle/internal/schema/testdata/pass/ml.yml create mode 100644 bundle/internal/schema/testdata/pass/pipeline.yml create mode 100644 bundle/internal/schema/testdata/pass/quality_monitor.yml create mode 100644 bundle/internal/schema/testdata/pass/run_job_task.yml create mode 100644 bundle/internal/schema/testdata/pass/schema.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 08edfb9d..02bf7378 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -107,11 +107,18 @@ jobs: run: npm install -g ajv-cli@5.0.0 # Assert that the generated bundle schema is a valid JSON schema by using - # ajv-cli to validate it against a sample configuration file. + # ajv-cli to validate it against bundle configuration files. # By default the ajv-cli runs in strict mode which will fail if the schema # itself is not valid. Strict mode is more strict than the JSON schema # specification. See for details: https://ajv.js.org/options.html#strict-mode-options - name: Validate bundle schema run: | go run main.go bundle schema > schema.json - ajv -s schema.json -d ./bundle/tests/basic/databricks.yml + + for file in ./bundle/internal/schema/testdata/pass/*.yml; do + ajv test -s schema.json -d $file --valid + done + + for file in ./bundle/internal/schema/testdata/fail/*.yml; do + ajv test -s schema.json -d $file --invalid + done diff --git a/bundle/internal/schema/testdata/fail/basic.yml b/bundle/internal/schema/testdata/fail/basic.yml new file mode 100644 index 00000000..5ab981e3 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/basic.yml @@ -0,0 +1,3 @@ +bundle: + # expected type is 'string' + name: 1234 diff --git a/bundle/internal/schema/testdata/fail/invalid_enum_value_in_job.yml b/bundle/internal/schema/testdata/fail/invalid_enum_value_in_job.yml new file mode 100644 index 00000000..92b1e9fc --- /dev/null +++ b/bundle/internal/schema/testdata/fail/invalid_enum_value_in_job.yml @@ -0,0 +1,4 @@ +resources: + jobs: + myjob: + format: INVALID_VALUE diff --git a/bundle/internal/schema/testdata/fail/invalid_enum_value_in_model.yml b/bundle/internal/schema/testdata/fail/invalid_enum_value_in_model.yml new file mode 100644 index 00000000..278b238f --- /dev/null +++ b/bundle/internal/schema/testdata/fail/invalid_enum_value_in_model.yml @@ -0,0 +1,6 @@ +resources: + models: + mymodel: + latest_versions: + - creation_timestamp: 123 + status: INVALID_VALUE diff --git a/bundle/internal/schema/testdata/fail/invalid_reference_in_job.yml b/bundle/internal/schema/testdata/fail/invalid_reference_in_job.yml new file mode 100644 index 00000000..2e0e2d84 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/invalid_reference_in_job.yml @@ -0,0 +1,8 @@ +resources: + jobs: + outer: + name: outer job + tasks: + - task_key: run job task 1 + run_job_task: + job_id: ${invalid.reference} diff --git a/bundle/internal/schema/testdata/fail/invalid_reference_in_model.yml b/bundle/internal/schema/testdata/fail/invalid_reference_in_model.yml new file mode 100644 index 00000000..899d6d85 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/invalid_reference_in_model.yml @@ -0,0 +1,5 @@ +resources: + models: + mymodel: + latest_versions: + - creation_timestamp: ${invalid.reference} diff --git a/bundle/internal/schema/testdata/fail/required_field_missing_in_job.yml b/bundle/internal/schema/testdata/fail/required_field_missing_in_job.yml new file mode 100644 index 00000000..ebb8ecf6 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/required_field_missing_in_job.yml @@ -0,0 +1,9 @@ +resources: + jobs: + foo: + name: my job + tasks: + # All tasks need to have a task_key. + - notebook_task: + notebook_path: /Users/abc/notebooks/inner + existing_cluster_id: abcd diff --git a/bundle/internal/schema/testdata/fail/unknown_field_in_job.yml b/bundle/internal/schema/testdata/fail/unknown_field_in_job.yml new file mode 100644 index 00000000..7e7e0d2d --- /dev/null +++ b/bundle/internal/schema/testdata/fail/unknown_field_in_job.yml @@ -0,0 +1,5 @@ +resources: + jobs: + myjob: + # unknown fields should cause schema failure. + unknown_field: "value" diff --git a/bundle/internal/schema/testdata/fail/unknown_field_in_model.yml b/bundle/internal/schema/testdata/fail/unknown_field_in_model.yml new file mode 100644 index 00000000..a00c2093 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/unknown_field_in_model.yml @@ -0,0 +1,6 @@ +resources: + models: + mymodel: + creation_timestamp: 123 + description: "my model" + unknown: "value" diff --git a/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml b/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml new file mode 100644 index 00000000..e8a8866b --- /dev/null +++ b/bundle/internal/schema/testdata/fail/unknown_top_level_field.yml @@ -0,0 +1 @@ +unknown: value diff --git a/bundle/internal/schema/testdata/pass/artifact_references.yml b/bundle/internal/schema/testdata/pass/artifact_references.yml new file mode 100644 index 00000000..c9b13763 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/artifact_references.yml @@ -0,0 +1,11 @@ +artifacts: + abc: + path: /Workspace/a/b/c + type: wheel + files: + - source: ./x.whl + +resources: + jobs: + foo: + name: ${artifacts.abc.type} diff --git a/bundle/internal/schema/testdata/pass/basic.yml b/bundle/internal/schema/testdata/pass/basic.yml new file mode 100644 index 00000000..de02d20b --- /dev/null +++ b/bundle/internal/schema/testdata/pass/basic.yml @@ -0,0 +1,2 @@ +bundle: + name: basic diff --git a/bundle/internal/schema/testdata/pass/direct_value_in_target.yml b/bundle/internal/schema/testdata/pass/direct_value_in_target.yml new file mode 100644 index 00000000..5033d8cd --- /dev/null +++ b/bundle/internal/schema/testdata/pass/direct_value_in_target.yml @@ -0,0 +1,4 @@ +targets: + development: + variables: + myvar: value diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml new file mode 100644 index 00000000..d9b0e832 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/job.yml @@ -0,0 +1,63 @@ +bundle: + name: a job + +workspace: + host: "https://myworkspace.com" + root_path: /abc + +presets: + name_prefix: "[DEV]" + jobs_max_concurrent_runs: 10 + +variables: + simplevar: + default: true + description: "simplevar description" + + complexvar: + default: + key1: value1 + key2: value2 + key3: + - value3 + - value4 + description: "complexvar description" + +run_as: + service_principal_name: myserviceprincipal + +resources: + jobs: + myjob: + name: myjob + continuous: + pause_status: PAUSED + edit_mode: EDITABLE + max_concurrent_runs: 10 + description: "my job description" + email_notifications: + no_alert_for_skipped_runs: true + environments: + - environment_key: venv + spec: + dependencies: + - python=3.7 + client: "myclient" + format: MULTI_TASK + tags: + foo: bar + bar: baz + tasks: + - task_key: mytask + notebook_task: + notebook_path: ${var.simplevar} + existing_cluster_id: abcd + - task_key: mytask2 + for_each_task: + inputs: av + concurrency: 10 + task: + task_key: inside_for_each + notebook_task: + notebook_path: ${var.complexvar.key3[0]} + - ${var.complexvar} diff --git a/bundle/internal/schema/testdata/pass/ml.yml b/bundle/internal/schema/testdata/pass/ml.yml new file mode 100644 index 00000000..b1558f10 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/ml.yml @@ -0,0 +1,72 @@ +bundle: + name: ML + +workspace: + host: "https://myworkspace.com" + root_path: /abc + +presets: + name_prefix: "[DEV]" + jobs_max_concurrent_runs: 10 + +variables: + simplevar: + default: "true" + description: "simplevar description" + + complexvar: + default: + key1: value1 + key2: value2 + key3: + - value3 + - value4 + description: "complexvar description" + +resources: + models: + mymodel: + creation_timestamp: 123 + description: "my model" + latest_versions: + - creation_timestamp: 123 + tags: ${var.complexvar.key1} + status: READY + permissions: + - service_principal_name: myserviceprincipal + level: CAN_MANAGE + + experiments: + myexperiment: + artifact_location: /dbfs/myexperiment + last_update_time: ${var.complexvar.key2} + lifecycle_stage: ${var.simplevar} + permissions: + - service_principal_name: myserviceprincipal + level: CAN_MANAGE + + model_serving_endpoints: + myendpoint: + config: + served_models: + - model_name: ${resources.models.mymodel.name} + model_version: abc + scale_to_zero_enabled: true + workload_size: Large + name: myendpoint + + schemas: + myschema: + catalog_name: mycatalog + name: myschema + + registered_models: + myregisteredmodel: + catalog_name: mycatalog + name: myregisteredmodel + schema_name: ${resources.schemas.myschema.name} + grants: + - principal: abcd + privileges: + - SELECT + - INSERT diff --git a/bundle/internal/schema/testdata/pass/pipeline.yml b/bundle/internal/schema/testdata/pass/pipeline.yml new file mode 100644 index 00000000..1b2b1a10 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/pipeline.yml @@ -0,0 +1,54 @@ +bundle: + name: a pipeline + +workspace: + host: "https://myworkspace.com" + root_path: /abc + +presets: + name_prefix: "[DEV]" + jobs_max_concurrent_runs: 10 + +variables: + simplevar: + default: true + description: "simplevar description" + + complexvar: + default: + key1: value1 + key2: value2 + key3: + - value3 + - value4 + description: "complexvar description" + +artifacts: + mywheel: + path: ./mywheel.whl + type: WHEEL + +run_as: + service_principal_name: myserviceprincipal + +resources: + jobs: + myjob: + name: myjob + tasks: + - task_key: ${bundle.name} pipeline trigger + pipeline_task: + pipeline_id: ${resources.mypipeline.id} + + pipelines: + mypipeline: + name: mypipeline + libraries: + - whl: ./mywheel.whl + catalog: 3{var.complexvar.key2} + development: true + clusters: + - autoscale: + mode: ENHANCED + max_workers: 10 + min_workers: 1 diff --git a/bundle/internal/schema/testdata/pass/quality_monitor.yml b/bundle/internal/schema/testdata/pass/quality_monitor.yml new file mode 100644 index 00000000..a9be5932 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/quality_monitor.yml @@ -0,0 +1,16 @@ +bundle: + name: quality_monitor + +resources: + quality_monitors: + myqualitymonitor: + inference_log: + granularities: + - a + - b + model_id_col: a + prediction_col: b + timestamp_col: c + problem_type: PROBLEM_TYPE_CLASSIFICATION + assets_dir: /dbfs/mnt/abc + output_schema_name: default diff --git a/bundle/internal/schema/testdata/pass/run_job_task.yml b/bundle/internal/schema/testdata/pass/run_job_task.yml new file mode 100644 index 00000000..be2ca22c --- /dev/null +++ b/bundle/internal/schema/testdata/pass/run_job_task.yml @@ -0,0 +1,56 @@ +bundle: + name: a run job task + databricks_cli_version: 0.200.0 + compute_id: "mycompute" + + +variables: + simplevar: + default: 5678 + description: "simplevar description" + + complexvar: + default: + key1: 1234 + key2: value2 + key3: + - value3 + - 9999 + description: "complexvar description" + +resources: + jobs: + inner: + permissions: + - user_name: user1 + level: CAN_MANAGE + + name: inner job + tasks: + - task_key: inner notebook task + notebook_task: + notebook_path: /Users/abc/notebooks/inner + existing_cluster_id: abcd + + outer: + name: outer job + tasks: + - task_key: run job task 1 + run_job_task: + job_id: 1234 + + - task_key: run job task 2 + run_job_task: + job_id: ${var.complexvar.key1} + + - task_key: run job task 3 + run_job_task: + job_id: ${var.simplevar} + + - task_key: run job task 4 + run_job_task: + job_id: ${resources.inner.id} + + - task_key: run job task 5 + run_job_task: + job_id: ${var.complexvar.key3[1]} diff --git a/bundle/internal/schema/testdata/pass/schema.yml b/bundle/internal/schema/testdata/pass/schema.yml new file mode 100644 index 00000000..37d0f6f7 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/schema.yml @@ -0,0 +1,24 @@ +bundle: + name: basic + +variables: + complexvar: + default: + key1: 1234 + key2: value2 + key3: + - value3 + - 9999 + description: complexvar description + +resources: + schemas: + myschema: + name: myschema + catalog_name: main + grants: + - ${var.complexvar} + - principal: ${workspace.current_user.me} + privileges: + - ${var.complexvar.key3[0]} + - ${var.complexvar.key2} From 66307134c1d34e34fc64e71fcc5c01acf819e677 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Sep 2024 11:49:58 +0200 Subject: [PATCH 35/78] Fixed generated YAML missing 'default' for empty values (#1765) ## Changes Fixed generated YAML missing 'default' for empty values ## Tests Added unit test --- bundle/config/generate/job.go | 14 ++++++++++++++ cmd/bundle/generate/generate_test.go | 9 +++++++++ libs/dyn/yamlsaver/saver.go | 2 ++ 3 files changed, 25 insertions(+) diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 28bc8641..6cd7c1b3 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -25,6 +25,20 @@ func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { value["tasks"] = dyn.NewValue(tasks, []dyn.Location{{Line: jobOrder.Get("tasks")}}) } + // We're processing job.Settings.Parameters separately to retain empty default values. + if len(job.Settings.Parameters) > 0 { + params := make([]dyn.Value, 0) + for _, parameter := range job.Settings.Parameters { + p := map[string]dyn.Value{ + "name": dyn.NewValue(parameter.Name, []dyn.Location{{Line: 0}}), // We use Line: 0 to ensure that the name goes first. + "default": dyn.NewValue(parameter.Default, []dyn.Location{{Line: 1}}), + } + params = append(params, dyn.NewValue(p, []dyn.Location{})) + } + + value["parameters"] = dyn.NewValue(params, []dyn.Location{{Line: jobOrder.Get("parameters")}}) + } + return yamlsaver.ConvertToMapValue(job.Settings, jobOrder, []string{"format", "new_cluster", "existing_cluster_id"}, value) } diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index ae3710ac..7de6805f 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -152,6 +152,12 @@ func TestGenerateJobCommand(t *testing.T) { }, }, }, + Parameters: []jobs.JobParameterDefinition{ + { + Name: "empty", + Default: "", + }, + }, }, }, nil) @@ -198,6 +204,9 @@ func TestGenerateJobCommand(t *testing.T) { - task_key: notebook_task notebook_task: notebook_path: %s + parameters: + - name: empty + default: "" `, filepath.Join("..", "src", "notebook.py")), string(data)) data, err = os.ReadFile(filepath.Join(srcDir, "notebook.py")) diff --git a/libs/dyn/yamlsaver/saver.go b/libs/dyn/yamlsaver/saver.go index f4c7157f..0fd81d53 100644 --- a/libs/dyn/yamlsaver/saver.go +++ b/libs/dyn/yamlsaver/saver.go @@ -151,6 +151,8 @@ func isScalarValueInString(v dyn.Value) bool { switch v.MustString() { case "true", "false": return true + case "": + return true default: _, err := parseNumber(v.MustString()) return err == nil From fb077a85d25e051342185193ed165bf7c61338d3 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 11 Sep 2024 14:16:18 +0200 Subject: [PATCH 36/78] Fix artifact upload integration tests (#1767) ## Changes I didn't run integration tests on #1756. ## Tests Manually confirmed integration tests pass. --- internal/bundle/artifacts_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index bae8073f..fa052e22 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -36,7 +36,8 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, + RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -100,7 +101,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, + RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -169,7 +171,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { touchEmptyFile(t, whlPath) b := &bundle.Bundle{ - RootPath: dir, + RootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", From f2dee890b8b858c94f7b140acf14eb4406111b46 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 12 Sep 2024 10:33:00 +0200 Subject: [PATCH 37/78] Use periodic triggers in all templates (#1739) ## Summary Simplifies template by using the periodic trigger syntax instead of the cron schedule syntax. Periodic triggers are simpler to configure, simpler to read, and make sure that workloads are spread out through the day. We only recommend cron syntax for advanced cases or when more control is needed. ## Testing * Templates validation via unit tests * Manual validation that the new triggers work as expected in dev/prod --- .../resources/{{.project_name}}_job.yml.tmpl | 9 +++++---- .../resources/{{.project_name}}_job.yml.tmpl | 9 +++++---- .../resources/{{.project_name}}_sql_job.yml.tmpl | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index bad12c75..e23c8dbc 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -3,10 +3,11 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 9:27 AM - quartz_cron_expression: 21 27 9 * * ? - timezone_id: UTC + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS email_notifications: on_failure: diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index dc79e3a1..d2100e90 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -10,10 +10,11 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 8:37 AM - quartz_cron_expression: '44 37 8 * * ?' - timezone_id: Europe/Amsterdam + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 31d2d21a..4e6803da 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -4,10 +4,11 @@ resources: {{.project_name}}_sql_job: name: {{.project_name}}_sql_job - schedule: - # Run every day at 7:17 AM - quartz_cron_expression: '44 17 7 * * ?' - timezone_id: Europe/Amsterdam + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} From e220f9ddd6745a9b3ae10b2dff2b16a4b72d1626 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 16 Sep 2024 20:35:07 +0200 Subject: [PATCH 38/78] Use the friendly name of service principals when shortening their name (#1770) ## Summary Use the friendly name of service principals when shortening their name. This change is helpful for the prefix in development mode. Instead of adding a prefix like `[dev 1706906c-c0a2-4c25-9f57-3a7aa3cb8123]`, we'll prefix like `[dev my_principal]`. --- .../config/mutator/populate_current_user.go | 2 +- internal/init_test.go | 2 +- libs/auth/user.go | 9 ++- libs/auth/user_test.go | 71 +++++++++++++++---- libs/template/helpers.go | 2 +- 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index b5e0bd43..1e99b327 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } b.Config.Workspace.CurrentUser = &config.User{ - ShortName: auth.GetShortUserName(me.UserName), + ShortName: auth.GetShortUserName(me), User: me, } diff --git a/internal/init_test.go b/internal/init_test.go index c3cb0127..d1a89f7b 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -126,7 +126,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }{ { funcName: "{{short_name}}", - expected: auth.GetShortUserName(me.UserName), + expected: auth.GetShortUserName(me), }, { funcName: "{{user_name}}", diff --git a/libs/auth/user.go b/libs/auth/user.go index 8eaa8763..c6aa974f 100644 --- a/libs/auth/user.go +++ b/libs/auth/user.go @@ -4,12 +4,17 @@ import ( "strings" "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/service/iam" ) // Get a short-form username, based on the user's primary email address. // We leave the full range of unicode letters in tact, but remove all "special" characters, // including dots, which are not supported in e.g. experiment names. -func GetShortUserName(emailAddress string) string { - local, _, _ := strings.Cut(emailAddress, "@") +func GetShortUserName(user *iam.User) string { + name := user.UserName + if IsServicePrincipal(user.UserName) && user.DisplayName != "" { + name = user.DisplayName + } + local, _, _ := strings.Cut(name, "@") return textutil.NormalizeString(local) } diff --git a/libs/auth/user_test.go b/libs/auth/user_test.go index 62b2d29a..24b61464 100644 --- a/libs/auth/user_test.go +++ b/libs/auth/user_test.go @@ -3,70 +3,111 @@ package auth import ( "testing" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) func TestGetShortUserName(t *testing.T) { tests := []struct { name string - email string + user *iam.User expected string }{ { - email: "test.user.1234@example.com", + user: &iam.User{ + UserName: "test.user.1234@example.com", + }, expected: "test_user_1234", }, { - email: "tést.üser@example.com", + user: &iam.User{ + UserName: "tést.üser@example.com", + }, expected: "tést_üser", }, { - email: "test$.user@example.com", + user: &iam.User{ + UserName: "test$.user@example.com", + }, expected: "test_user", }, { - email: `jöhn.dœ@domain.com`, // Using non-ASCII characters. + user: &iam.User{ + UserName: `jöhn.dœ@domain.com`, // Using non-ASCII characters. + }, expected: "jöhn_dœ", }, { - email: `first+tag@email.com`, // The plus (+) sign is used for "sub-addressing" in some email services. + user: &iam.User{ + UserName: `first+tag@email.com`, // The plus (+) sign is used for "sub-addressing" in some email services. + }, expected: "first_tag", }, { - email: `email@sub.domain.com`, // Using a sub-domain. + user: &iam.User{ + UserName: `email@sub.domain.com`, // Using a sub-domain. + }, expected: "email", }, { - email: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. + user: &iam.User{ + UserName: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. + }, expected: "quoted", }, { - email: `name-o'mally@website.org`, // Single quote in the local-part. + user: &iam.User{ + UserName: `name-o'mally@website.org`, // Single quote in the local-part. + }, expected: "name_o_mally", }, { - email: `user%domain@external.com`, // Percent sign can be used for email routing in legacy systems. + user: &iam.User{ + UserName: `user%domain@external.com`, // Percent sign can be used for email routing in legacy systems. + }, expected: "user_domain", }, { - email: `long.name.with.dots@domain.net`, // Multiple dots in the local-part. + user: &iam.User{ + UserName: `long.name.with.dots@domain.net`, // Multiple dots in the local-part. + }, expected: "long_name_with_dots", }, { - email: `me&you@together.com`, // Using an ampersand (&) in the local-part. + user: &iam.User{ + UserName: `me&you@together.com`, // Using an ampersand (&) in the local-part. + }, expected: "me_you", }, { - email: `user!def!xyz@domain.org`, // The exclamation mark can be valid in some legacy systems. + user: &iam.User{ + UserName: `user!def!xyz@domain.org`, // The exclamation mark can be valid in some legacy systems. + }, expected: "user_def_xyz", }, { - email: `admin@ιντερνετ.com`, // Domain in non-ASCII characters (IDN or Internationalized Domain Name). + user: &iam.User{ + UserName: `admin@ιντερνετ.com`, // Domain in non-ASCII characters (IDN or Internationalized Domain Name). + }, expected: "admin", }, + { + user: &iam.User{ + UserName: `1706906c-c0a2-4c25-9f57-3a7aa3cb8123`, + DisplayName: "my-service-principal", + }, + expected: "my_service_principal", + }, + { + user: &iam.User{ + UserName: `1706906c-c0a2-4c25-9f57-3a7aa3cb8123`, + // This service princpal has DisplayName (it's an optional property) + }, + expected: "1706906c_c0a2_4c25_9f57_3a7aa3cb8123", + }, } for _, tt := range tests { - assert.Equal(t, tt.expected, GetShortUserName(tt.email)) + assert.Equal(t, tt.expected, GetShortUserName(tt.user)) } } diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 1dfe74d7..88c73cc4 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -119,7 +119,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return "", err } } - return auth.GetShortUserName(cachedUser.UserName), nil + return auth.GetShortUserName(cachedUser), nil }, // Get the default workspace catalog. If there is no default, or if // Unity Catalog is not enabled, return an empty string. From bcab6ca37b27c71156cdb3a9119db9becef4f869 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Sep 2024 12:23:07 +0200 Subject: [PATCH 39/78] Fixed detecting full syntax variable override which includes type field (#1775) ## Changes Fixes #1773 ## Tests Confirmed manually --- bundle/config/root.go | 21 ++++++++++++++++--- bundle/tests/complex_variables_test.go | 18 ++++++++++++++++ bundle/tests/variables/complex/databricks.yml | 13 ++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 46578769..884c2e1c 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -409,18 +409,33 @@ func (r *Root) MergeTargetOverrides(name string) error { var variableKeywords = []string{"default", "lookup"} // isFullVariableOverrideDef checks if the given value is a full syntax varaible override. -// A full syntax variable override is a map with only one of the following -// keys: "default", "lookup". +// A full syntax variable override is a map with either 1 of 2 keys. +// If it's 2 keys, the keys should be "default" and "type". +// If it's 1 key, the key should be one of the following keys: "default", "lookup". func isFullVariableOverrideDef(v dyn.Value) bool { mv, ok := v.AsMap() if !ok { return false } - if mv.Len() != 1 { + // If the map has more than 2 keys, it is not a full variable override. + if mv.Len() > 2 { return false } + // If the map has 2 keys, one of them should be "default" and the other is "type" + if mv.Len() == 2 { + if _, ok := mv.GetByString("type"); !ok { + return false + } + + if _, ok := mv.GetByString("default"); !ok { + return false + } + + return true + } + for _, keyword := range variableKeywords { if _, ok := mv.GetByString(keyword); ok { return true diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index 6371071c..7a9a53a7 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -88,3 +88,21 @@ func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) { require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", cluster.JobClusterKey) } } + +func TestComplexVariablesOverrideWithFullSyntax(t *testing.T) { + b, diags := loadTargetWithDiags("variables/complex", "dev") + require.Empty(t, diags) + + diags = bundle.Apply(context.Background(), b, bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferencesInComplexVariables(), + mutator.ResolveVariableReferences( + "variables", + ), + )) + require.NoError(t, diags.Error()) + require.Empty(t, diags) + + complexvar := b.Config.Variables["complexvar"].Value + require.Equal(t, map[string]interface{}{"key1": "1", "key2": "2", "key3": "3"}, complexvar) +} diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml index ca27f606..3b32a7c8 100644 --- a/bundle/tests/variables/complex/databricks.yml +++ b/bundle/tests/variables/complex/databricks.yml @@ -35,6 +35,13 @@ variables: - jar: "/path/to/jar" - egg: "/path/to/egg" - whl: "/path/to/whl" + complexvar: + type: complex + description: "A complex variable" + default: + key1: "value1" + key2: "value2" + key3: "value3" targets: @@ -49,3 +56,9 @@ targets: spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false + complexvar: + type: complex + default: + key1: "1" + key2: "2" + key3: "3" From e2c1d51d8437963bec84c857b74bb210b78b26b0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Sep 2024 13:26:16 +0200 Subject: [PATCH 40/78] [Release] Release v0.228.1 (#1778) Bundles: * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)). * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)). * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)). * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)). * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)). * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)). Internal: * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)). * Make bundle JSON schema modular with `$defs` ([#1700](https://github.com/databricks/cli/pull/1700)). * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)). * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)). * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)). API Changes: * Added `databricks quality-monitors regenerate-dashboard` command. OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04) Dependency updates: * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)). * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)). * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)). * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)). --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6383125..32a7e5cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Version changelog +## [Release] Release v0.228.1 + +Bundles: + * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)). + * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)). + * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)). + * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)). + * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)). + * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)). + +Internal: + * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)). + * Make bundle JSON schema modular with `` ([#1700](https://github.com/databricks/cli/pull/1700)). + * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)). + * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)). + * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)). + +API Changes: + * Added `databricks quality-monitors regenerate-dashboard` command. + +OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04) +Dependency updates: + * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)). + * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)). + * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)). + * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)). + ## [Release] Release v0.228.0 CLI: From cf989a7e10e56f0b021eb4ffc5a7b793da25b540 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 19 Sep 2024 13:21:32 +0200 Subject: [PATCH 41/78] Upgrade to TF provider 1.52 (#1781) ## Changes Upgrade to TF provider 1.52 We also temporarily skip generating plugin framework structs to unblock upgrade as generation does not work yet and need to be fixed separately --- .../tf/codegen/generator/generator.go | 14 +++++++++++++- bundle/internal/tf/codegen/schema/version.go | 2 +- .../tf/schema/data_source_clusters.go | 16 ++++++++++++---- .../schema/data_source_external_location.go | 1 + .../internal/tf/schema/data_source_share.go | 2 ++ ...omatic_cluster_update_workspace_setting.go | 18 ++++++------------ bundle/internal/tf/schema/resource_cluster.go | 1 + ...ance_security_profile_workspace_setting.go | 4 ++-- ...d_security_monitoring_workspace_setting.go | 2 +- .../tf/schema/resource_model_serving.go | 18 ++++++++++-------- bundle/internal/tf/schema/resource_share.go | 19 +++++++++++++------ .../internal/tf/schema/resource_sql_table.go | 1 + bundle/internal/tf/schema/root.go | 2 +- 13 files changed, 64 insertions(+), 36 deletions(-) diff --git a/bundle/internal/tf/codegen/generator/generator.go b/bundle/internal/tf/codegen/generator/generator.go index 86d76243..b31fdf15 100644 --- a/bundle/internal/tf/codegen/generator/generator.go +++ b/bundle/internal/tf/codegen/generator/generator.go @@ -51,9 +51,15 @@ func (r *root) Generate(path string) error { } func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error { - // Generate types for resources. + // Generate types for resources var resources []*namedBlock for _, k := range sortKeys(schema.ResourceSchemas) { + // Skipping all plugin framework struct generation. + // TODO: This is a temporary fix, generation should be fixed in the future. + if strings.HasSuffix(k, "_pluginframework") { + continue + } + v := schema.ResourceSchemas[k] b := &namedBlock{ filePattern: "resource_%s.go", @@ -71,6 +77,12 @@ func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error // Generate types for data sources. var dataSources []*namedBlock for _, k := range sortKeys(schema.DataSourceSchemas) { + // Skipping all plugin framework struct generation. + // TODO: This is a temporary fix, generation should be fixed in the future. + if strings.HasSuffix(k, "_pluginframework") { + continue + } + v := schema.DataSourceSchemas[k] b := &namedBlock{ filePattern: "data_source_%s.go", diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index efb29724..b71ea7d1 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.50.0" +const ProviderVersion = "1.52.0" diff --git a/bundle/internal/tf/schema/data_source_clusters.go b/bundle/internal/tf/schema/data_source_clusters.go index 7a5f3053..8c5f9578 100644 --- a/bundle/internal/tf/schema/data_source_clusters.go +++ b/bundle/internal/tf/schema/data_source_clusters.go @@ -2,8 +2,16 @@ package schema -type DataSourceClusters struct { - ClusterNameContains string `json:"cluster_name_contains,omitempty"` - Id string `json:"id,omitempty"` - Ids []string `json:"ids,omitempty"` +type DataSourceClustersFilterBy struct { + ClusterSources []string `json:"cluster_sources,omitempty"` + ClusterStates []string `json:"cluster_states,omitempty"` + IsPinned bool `json:"is_pinned,omitempty"` + PolicyId string `json:"policy_id,omitempty"` +} + +type DataSourceClusters struct { + ClusterNameContains string `json:"cluster_name_contains,omitempty"` + Id string `json:"id,omitempty"` + Ids []string `json:"ids,omitempty"` + FilterBy *DataSourceClustersFilterBy `json:"filter_by,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_external_location.go b/bundle/internal/tf/schema/data_source_external_location.go index a3e78cbd..e1ad9dc3 100644 --- a/bundle/internal/tf/schema/data_source_external_location.go +++ b/bundle/internal/tf/schema/data_source_external_location.go @@ -19,6 +19,7 @@ type DataSourceExternalLocationExternalLocationInfo struct { CreatedBy string `json:"created_by,omitempty"` CredentialId string `json:"credential_id,omitempty"` CredentialName string `json:"credential_name,omitempty"` + Fallback bool `json:"fallback,omitempty"` IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_share.go b/bundle/internal/tf/schema/data_source_share.go index 3b40fbb5..da9afaae 100644 --- a/bundle/internal/tf/schema/data_source_share.go +++ b/bundle/internal/tf/schema/data_source_share.go @@ -18,12 +18,14 @@ type DataSourceShareObject struct { AddedBy string `json:"added_by,omitempty"` CdfEnabled bool `json:"cdf_enabled,omitempty"` Comment string `json:"comment,omitempty"` + Content string `json:"content,omitempty"` DataObjectType string `json:"data_object_type"` HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"` Name string `json:"name"` SharedAs string `json:"shared_as,omitempty"` StartVersion int `json:"start_version,omitempty"` Status string `json:"status,omitempty"` + StringSharedAs string `json:"string_shared_as,omitempty"` Partition []DataSourceShareObjectPartition `json:"partition,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go index e95639de..5d7f6a14 100644 --- a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go @@ -2,20 +2,14 @@ package schema -type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct { - ForcedForComplianceMode bool `json:"forced_for_compliance_mode,omitempty"` - UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"` - UnavailableForNonEnterpriseTier bool `json:"unavailable_for_non_enterprise_tier,omitempty"` -} - type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct { - Hours int `json:"hours,omitempty"` - Minutes int `json:"minutes,omitempty"` + Hours int `json:"hours"` + Minutes int `json:"minutes"` } type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct { - DayOfWeek string `json:"day_of_week,omitempty"` - Frequency string `json:"frequency,omitempty"` + DayOfWeek string `json:"day_of_week"` + Frequency string `json:"frequency"` WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"` } @@ -25,9 +19,9 @@ type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspa type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct { CanToggle bool `json:"can_toggle,omitempty"` - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled"` + EnablementDetails []any `json:"enablement_details,omitempty"` RestartEvenIfNoUpdatesAvailable bool `json:"restart_even_if_no_updates_available,omitempty"` - EnablementDetails *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"` MaintenanceWindow *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go index e4106d04..4ae063c8 100644 --- a/bundle/internal/tf/schema/resource_cluster.go +++ b/bundle/internal/tf/schema/resource_cluster.go @@ -176,6 +176,7 @@ type ResourceCluster struct { IdempotencyToken string `json:"idempotency_token,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` IsPinned bool `json:"is_pinned,omitempty"` + NoWait bool `json:"no_wait,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` PolicyId string `json:"policy_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go index 50815f75..8265adae 100644 --- a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go @@ -3,8 +3,8 @@ package schema type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct { - ComplianceStandards []string `json:"compliance_standards,omitempty"` - IsEnabled bool `json:"is_enabled,omitempty"` + ComplianceStandards []string `json:"compliance_standards"` + IsEnabled bool `json:"is_enabled"` } type ResourceComplianceSecurityProfileWorkspaceSetting struct { diff --git a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go index 2f552402..e9c3b0ab 100644 --- a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go @@ -3,7 +3,7 @@ package schema type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct { - IsEnabled bool `json:"is_enabled,omitempty"` + IsEnabled bool `json:"is_enabled"` } type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct { diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 379807a5..29d55cd5 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -95,14 +95,16 @@ type ResourceModelServingConfigServedEntities struct { } type ResourceModelServingConfigServedModels struct { - EnvironmentVars map[string]string `json:"environment_vars,omitempty"` - InstanceProfileArn string `json:"instance_profile_arn,omitempty"` - ModelName string `json:"model_name"` - ModelVersion string `json:"model_version"` - Name string `json:"name,omitempty"` - ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"` - WorkloadSize string `json:"workload_size"` - WorkloadType string `json:"workload_type,omitempty"` + EnvironmentVars map[string]string `json:"environment_vars,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + MaxProvisionedThroughput int `json:"max_provisioned_throughput,omitempty"` + MinProvisionedThroughput int `json:"min_provisioned_throughput,omitempty"` + ModelName string `json:"model_name"` + ModelVersion string `json:"model_version"` + Name string `json:"name,omitempty"` + ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"` + WorkloadSize string `json:"workload_size,omitempty"` + WorkloadType string `json:"workload_type,omitempty"` } type ResourceModelServingConfigTrafficConfigRoutes struct { diff --git a/bundle/internal/tf/schema/resource_share.go b/bundle/internal/tf/schema/resource_share.go index e531e777..37f4d454 100644 --- a/bundle/internal/tf/schema/resource_share.go +++ b/bundle/internal/tf/schema/resource_share.go @@ -18,20 +18,27 @@ type ResourceShareObject struct { AddedBy string `json:"added_by,omitempty"` CdfEnabled bool `json:"cdf_enabled,omitempty"` Comment string `json:"comment,omitempty"` + Content string `json:"content,omitempty"` DataObjectType string `json:"data_object_type"` HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"` Name string `json:"name"` SharedAs string `json:"shared_as,omitempty"` StartVersion int `json:"start_version,omitempty"` Status string `json:"status,omitempty"` + StringSharedAs string `json:"string_shared_as,omitempty"` Partition []ResourceShareObjectPartition `json:"partition,omitempty"` } type ResourceShare struct { - CreatedAt int `json:"created_at,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name"` - Owner string `json:"owner,omitempty"` - Object []ResourceShareObject `json:"object,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + Owner string `json:"owner,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Object []ResourceShareObject `json:"object,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 51fb3bc0..4f305c52 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -15,6 +15,7 @@ type ResourceSqlTable struct { ClusterKeys []string `json:"cluster_keys,omitempty"` Comment string `json:"comment,omitempty"` DataSourceFormat string `json:"data_source_format,omitempty"` + EffectiveProperties map[string]string `json:"effective_properties,omitempty"` Id string `json:"id,omitempty"` Name string `json:"name"` Options map[string]string `json:"options,omitempty"` diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index ebdb7f09..5fc34d6b 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.50.0" +const ProviderVersion = "1.52.0" func NewRoot() *Root { return &Root{ From 6c57683dc6077282dd95e03b19396f602dd5d635 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Sat, 21 Sep 2024 08:36:47 +0200 Subject: [PATCH 42/78] Reduce time until the prompt is shown for bundle run (#1727) ## Summary Makes the `databricks bundle run` command use local state before showing the menu prompt, which makes it show more quickly. For large/busy workspaces this means the prompt can show 2-3 seconds earlier. --- cmd/bundle/run.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 63458f85..9ef5eb8f 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -55,13 +55,7 @@ task or a Python wheel task, the second example applies. return diags.Error() } - diags = bundle.Apply(ctx, b, bundle.Seq( - phases.Initialize(), - terraform.Interpolate(), - terraform.Write(), - terraform.StatePull(), - terraform.Load(terraform.ErrorOnEmptyState), - )) + diags = bundle.Apply(ctx, b, phases.Initialize()) if err := diags.Error(); err != nil { return err } @@ -84,6 +78,16 @@ task or a Python wheel task, the second example applies. return fmt.Errorf("expected a KEY of the resource to run") } + diags = bundle.Apply(ctx, b, bundle.Seq( + terraform.Interpolate(), + terraform.Write(), + terraform.StatePull(), + terraform.Load(terraform.ErrorOnEmptyState), + )) + if err := diags.Error(); err != nil { + return err + } + runner, err := run.Find(b, args[0]) if err != nil { return err From 7665c639bd34392f5d95c177b520d48b9ffa40f4 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 23 Sep 2024 11:52:04 +0200 Subject: [PATCH 43/78] Use Unity Catalog for pipelines in the default-python template (#1766) ## Summary Enables Unity Catalog for pipelines in the default template. Pipelines will default to non-Unity Catalog pipelines if a catalog is not specified. *Small caveat*: there are cases where admins lock down the default catalog of a workspace and don't allow the creation of a new schema there. If that happens, the pipeline would fail at runtime with a clear error indicating what happened. ("PERMISSION_DENIED: User does not have CREATE SCHEMA on Catalog 'main'."). I've seen this with an internal Databricks workspace, where creating new non-UC schemas wasn't locked down, but creation in the `main` was. ## Testing - Validated on a non-UC + UC workspace. The catalog selection logic here is the same as applied for the SQL templates. --- .../resources/{{.project_name}}_pipeline.yml.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index 4b8f74d1..bf469046 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -3,6 +3,12 @@ resources: pipelines: {{.project_name}}_pipeline: name: {{.project_name}}_pipeline + {{- if eq default_catalog ""}} + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name + {{- else}} + catalog: {{default_catalog}} + {{- end}} target: {{.project_name}}_${bundle.environment} libraries: - notebook: From ac80d3dfcb648c26c131ad21aec45449ac5c3b31 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Mon, 23 Sep 2024 12:09:11 +0200 Subject: [PATCH 44/78] Add verbose flag to the "bundle deploy" command (#1774) ## Changes - Extract sync output logic from `cmd/sync` into `lib/sync` - Add hidden `verbose` flag to the `bundle deploy` command, it's false by default and hidden from the `--help` output - Pass output handler to the `deploy/files/upload` mutator if the verbose option is true The was an idea to use in-place output overriding each past file sync event in the output, bit that wont work for the extension, since it doesn't display deploy logs in the terminal. Example output: ``` ~/tmp/defpy: ~/cli/cli bundle deploy --sync-progress Building defpy... Uploading defpy-0.0.1+20240917.112755-py3-none-any.whl... Uploading bundle files to /Users/ilia.babanov@databricks.com/.bundle/defpy/dev/files... Action: PUT: requirements-dev.txt, resources/defpy_pipeline.yml, pytest.ini, src/defpy/main.py, src/defpy/__init__.py, src/dlt_pipeline.ipynb, tests/main_test.py, src/notebook.ipynb, setup.py, resources/defpy_job.yml, .vscode/extensions.json, .vscode/settings.json, fixtures/.gitkeep, .vscode/__builtins__.pyi, README.md, .gitignore, databricks.yml Uploaded tests Uploaded resources Uploaded fixtures Uploaded .vscode Uploaded src/defpy Uploaded requirements-dev.txt Uploaded .gitignore Uploaded fixtures/.gitkeep Uploaded src/defpy/__init__.py Uploaded databricks.yml Uploaded README.md Uploaded setup.py Uploaded .vscode/__builtins__.pyi Uploaded .vscode/extensions.json Uploaded src/dlt_pipeline.ipynb Uploaded .vscode/settings.json Uploaded resources/defpy_job.yml Uploaded pytest.ini Uploaded src/defpy/main.py Uploaded tests/main_test.py Uploaded resources/defpy_pipeline.yml Uploaded src/notebook.ipynb Initial Sync Complete Deploying resources... Updating deployment state... Deployment complete! ``` Output example in the extension: Screenshot 2024-09-19 at 11 07 48 ## Tests Manually for the `sync` and `bundle deploy` commands + vscode extension sync and deploy flows --- bundle/deploy/files/upload.go | 18 ++++++++++---- bundle/phases/deploy.go | 5 ++-- cmd/bundle/deploy.go | 14 ++++++++++- cmd/sync/sync.go | 38 ++++++++++++++--------------- {cmd => libs}/sync/output.go | 6 ++--- libs/sync/sync.go | 45 +++++++++++++++++++++++++---------- 6 files changed, 82 insertions(+), 44 deletions(-) rename {cmd => libs}/sync/output.go (83%) diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index 2c126623..77b83611 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -8,9 +8,12 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/sync" ) -type upload struct{} +type upload struct { + outputHandler sync.OutputHandler +} func (m *upload) Name() string { return "files.Upload" @@ -18,11 +21,18 @@ func (m *upload) Name() string { func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath)) - sync, err := GetSync(ctx, bundle.ReadOnly(b)) + opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b)) if err != nil { return diag.FromErr(err) } + opts.OutputHandler = m.outputHandler + sync, err := sync.New(ctx, *opts) + if err != nil { + return diag.FromErr(err) + } + defer sync.Close() + b.Files, err = sync.RunOnce(ctx) if err != nil { return diag.FromErr(err) @@ -32,6 +42,6 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return nil } -func Upload() bundle.Mutator { - return &upload{} +func Upload(outputHandler sync.OutputHandler) bundle.Mutator { + return &upload{outputHandler} } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 49544227..097c561e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/sync" terraformlib "github.com/databricks/cli/libs/terraform" tfjson "github.com/hashicorp/terraform-json" ) @@ -128,7 +129,7 @@ properties such as the 'catalog' or 'storage' are changed:` } // The deploy phase deploys artifacts and resources. -func Deploy() bundle.Mutator { +func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. deployCore := bundle.Defer( @@ -157,7 +158,7 @@ func Deploy() bundle.Mutator { libraries.ExpandGlobReferences(), libraries.Upload(), python.TransformWheelTask(), - files.Upload(), + files.Upload(outputHandler), deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 1166875a..49231734 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/sync" "github.com/spf13/cobra" ) @@ -25,11 +26,15 @@ func newDeployCommand() *cobra.Command { var failOnActiveRuns bool var computeID string var autoApprove bool + var verbose bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") + cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") + // Verbose flag currently only affects file sync output, it's used by the vscode extension + cmd.Flags().MarkHidden("verbose") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -51,11 +56,18 @@ func newDeployCommand() *cobra.Command { return nil }) + var outputHandler sync.OutputHandler + if verbose { + outputHandler = func(ctx context.Context, c <-chan sync.Event) { + sync.TextOutput(ctx, c, cmd.OutOrStdout()) + } + } + diags = diags.Extend( bundle.Apply(ctx, b, bundle.Seq( phases.Initialize(), phases.Build(), - phases.Deploy(), + phases.Deploy(outputHandler), )), ) } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 23a4c018..2092d9e3 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "path/filepath" - stdsync "sync" "time" "github.com/databricks/cli/bundle" @@ -46,6 +45,21 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn return nil, flag.ErrHelp } + var outputFunc func(context.Context, <-chan sync.Event, io.Writer) + switch f.output { + case flags.OutputText: + outputFunc = sync.TextOutput + case flags.OutputJSON: + outputFunc = sync.JsonOutput + } + + var outputHandler sync.OutputHandler + if outputFunc != nil { + outputHandler = func(ctx context.Context, events <-chan sync.Event) { + outputFunc(ctx, events, cmd.OutOrStdout()) + } + } + opts := sync.SyncOptions{ LocalRoot: vfs.MustNew(args[0]), Paths: []string{"."}, @@ -62,6 +76,8 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn // exist and add it to the `.gitignore` file in the root. SnapshotBasePath: filepath.Join(args[0], ".databricks"), WorkspaceClient: root.WorkspaceClient(cmd.Context()), + + OutputHandler: outputHandler, } return &opts, nil } @@ -118,23 +134,7 @@ func New() *cobra.Command { if err != nil { return err } - - var outputFunc func(context.Context, <-chan sync.Event, io.Writer) - switch f.output { - case flags.OutputText: - outputFunc = textOutput - case flags.OutputJSON: - outputFunc = jsonOutput - } - - var wg stdsync.WaitGroup - if outputFunc != nil { - wg.Add(1) - go func() { - defer wg.Done() - outputFunc(ctx, s.Events(), cmd.OutOrStdout()) - }() - } + defer s.Close() if f.watch { err = s.RunContinuous(ctx) @@ -142,8 +142,6 @@ func New() *cobra.Command { _, err = s.RunOnce(ctx) } - s.Close() - wg.Wait() return err } diff --git a/cmd/sync/output.go b/libs/sync/output.go similarity index 83% rename from cmd/sync/output.go rename to libs/sync/output.go index 2785343f..c01b25ef 100644 --- a/cmd/sync/output.go +++ b/libs/sync/output.go @@ -5,12 +5,10 @@ import ( "context" "encoding/json" "io" - - "github.com/databricks/cli/libs/sync" ) // Read synchronization events and write them as JSON to the specified writer (typically stdout). -func jsonOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { +func JsonOutput(ctx context.Context, ch <-chan Event, w io.Writer) { enc := json.NewEncoder(w) for { select { @@ -31,7 +29,7 @@ func jsonOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { } // Read synchronization events and write them as text to the specified writer (typically stdout). -func textOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { +func TextOutput(ctx context.Context, ch <-chan Event, w io.Writer) { bw := bufio.NewWriter(w) for { diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 9eaebf2a..cc9c7394 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -3,6 +3,7 @@ package sync import ( "context" "fmt" + stdsync "sync" "time" "github.com/databricks/cli/libs/filer" @@ -15,6 +16,8 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +type OutputHandler func(context.Context, <-chan Event) + type SyncOptions struct { LocalRoot vfs.Path Paths []string @@ -34,6 +37,8 @@ type SyncOptions struct { CurrentUser *iam.User Host string + + OutputHandler OutputHandler } type Sync struct { @@ -49,6 +54,10 @@ type Sync struct { // Synchronization progress events are sent to this event notifier. notifier EventNotifier seq int + + // WaitGroup is automatically created when an output handler is provided in the SyncOptions. + // Close call is required to ensure the output handler goroutine handles all events in time. + outputWaitGroup *stdsync.WaitGroup } // New initializes and returns a new [Sync] instance. @@ -106,31 +115,41 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } + var notifier EventNotifier + var outputWaitGroup = &stdsync.WaitGroup{} + if opts.OutputHandler != nil { + ch := make(chan Event, MaxRequestsInFlight) + notifier = &ChannelNotifier{ch} + outputWaitGroup.Add(1) + go func() { + defer outputWaitGroup.Done() + opts.OutputHandler(ctx, ch) + }() + } else { + notifier = &NopNotifier{} + } + return &Sync{ SyncOptions: &opts, - fileSet: fileSet, - includeFileSet: includeFileSet, - excludeFileSet: excludeFileSet, - snapshot: snapshot, - filer: filer, - notifier: &NopNotifier{}, - seq: 0, + fileSet: fileSet, + includeFileSet: includeFileSet, + excludeFileSet: excludeFileSet, + snapshot: snapshot, + filer: filer, + notifier: notifier, + outputWaitGroup: outputWaitGroup, + seq: 0, }, nil } -func (s *Sync) Events() <-chan Event { - ch := make(chan Event, MaxRequestsInFlight) - s.notifier = &ChannelNotifier{ch} - return ch -} - func (s *Sync) Close() { if s.notifier == nil { return } s.notifier.Close() s.notifier = nil + s.outputWaitGroup.Wait() } func (s *Sync) notifyStart(ctx context.Context, d diff) { From 56ed9bebf39b2ef6430e42dbb840684285217436 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 23 Sep 2024 12:42:34 +0200 Subject: [PATCH 45/78] Added support for creating all-purpose clusters (#1698) ## Changes Added support for creating all-purpose clusters Example of configuration ``` bundle: name: clusters resources: clusters: test_cluster: cluster_name: "Test Cluster" num_workers: 2 node_type_id: "i3.xlarge" autoscale: min_workers: 2 max_workers: 7 spark_version: "13.3.x-scala2.12" spark_conf: "spark.executor.memory": "2g" jobs: test_job: name: "Test Job" tasks: - task_key: test_task existing_cluster_id: ${resources.clusters.test_cluster.id} notebook_task: notebook_path: "./src/test.py" targets: development: mode: development compute_id: ${resources.clusters.test_cluster.id} ``` ## Tests Added unit, config and E2E tests --- bundle/config/bundle.go | 7 +- bundle/config/mutator/apply_presets.go | 15 +++ bundle/config/mutator/compute_id_compat.go | 87 +++++++++++++++++ .../config/mutator/compute_id_compate_test.go | 57 +++++++++++ bundle/config/mutator/mutator.go | 1 + bundle/config/mutator/override_compute.go | 8 +- .../config/mutator/override_compute_test.go | 4 +- .../mutator/process_target_mode_test.go | 10 ++ bundle/config/mutator/run_as_test.go | 2 + bundle/config/resources.go | 1 + bundle/config/resources/clusters.go | 39 ++++++++ bundle/config/root.go | 6 +- bundle/config/target.go | 7 +- bundle/deploy/terraform/convert.go | 22 +++++ bundle/deploy/terraform/convert_test.go | 56 +++++++++++ bundle/deploy/terraform/interpolate.go | 2 + bundle/deploy/terraform/interpolate_test.go | 2 + .../deploy/terraform/tfdyn/convert_cluster.go | 52 ++++++++++ .../terraform/tfdyn/convert_cluster_test.go | 97 +++++++++++++++++++ bundle/tests/clusters/databricks.yml | 36 +++++++ bundle/tests/clusters_test.go | 36 +++++++ cmd/bundle/deploy.go | 11 ++- .../clusters/databricks_template_schema.json | 16 +++ .../clusters/template/databricks.yml.tmpl | 24 +++++ .../bundles/clusters/template/hello_world.py | 1 + internal/bundle/clusters_test.go | 56 +++++++++++ internal/testutil/cloud.go | 4 + 27 files changed, 643 insertions(+), 16 deletions(-) create mode 100644 bundle/config/mutator/compute_id_compat.go create mode 100644 bundle/config/mutator/compute_id_compate_test.go create mode 100644 bundle/config/resources/clusters.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_cluster.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_cluster_test.go create mode 100644 bundle/tests/clusters/databricks.yml create mode 100644 bundle/tests/clusters_test.go create mode 100644 internal/bundle/bundles/clusters/databricks_template_schema.json create mode 100644 internal/bundle/bundles/clusters/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/clusters/template/hello_world.py create mode 100644 internal/bundle/clusters_test.go diff --git a/bundle/config/bundle.go b/bundle/config/bundle.go index 78648dfd..f533c4d1 100644 --- a/bundle/config/bundle.go +++ b/bundle/config/bundle.go @@ -38,8 +38,11 @@ type Bundle struct { // Annotated readonly as this should be set at the target level. Mode Mode `json:"mode,omitempty" bundle:"readonly"` - // Overrides the compute used for jobs and other supported assets. - ComputeID string `json:"compute_id,omitempty"` + // DEPRECATED: Overrides the compute used for jobs and other supported assets. + ComputeId string `json:"compute_id,omitempty"` + + // Overrides the cluster used for jobs and other supported assets. + ClusterId string `json:"cluster_id,omitempty"` // Deployment section specifies deployment related configuration for bundle Deployment Deployment `json:"deployment,omitempty"` diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 28d015c1..27af82e5 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -160,6 +160,21 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // the Databricks UI and via the SQL API. } + // Clusters: Prefix, Tags + for _, c := range r.Clusters { + c.ClusterName = prefix + c.ClusterName + if c.CustomTags == nil { + c.CustomTags = make(map[string]string) + } + for _, tag := range tags { + normalisedKey := b.Tagging.NormalizeKey(tag.Key) + normalisedValue := b.Tagging.NormalizeValue(tag.Value) + if _, ok := c.CustomTags[normalisedKey]; !ok { + c.CustomTags[normalisedKey] = normalisedValue + } + } + } + return nil } diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go new file mode 100644 index 00000000..3afe02e9 --- /dev/null +++ b/bundle/config/mutator/compute_id_compat.go @@ -0,0 +1,87 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type computeIdToClusterId struct{} + +func ComputeIdToClusterId() bundle.Mutator { + return &computeIdToClusterId{} +} + +func (m *computeIdToClusterId) Name() string { + return "ComputeIdToClusterId" +} + +func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // The "compute_id" key is set; rewrite it to "cluster_id". + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle"))) + diags = diags.Extend(d) + + // Check if the "compute_id" key is set in any target overrides. + return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + v, d := rewriteComputeIdToClusterId(v, dyn.Path{}) + diags = diags.Extend(d) + return v, nil + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} + +func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) { + var diags diag.Diagnostics + computeIdPath := p.Append(dyn.Key("compute_id")) + computeId, err := dyn.GetByPath(v, computeIdPath) + + // If the "compute_id" key is not set, we don't need to do anything. + if err != nil { + return v, nil + } + + if computeId.Kind() == dyn.KindInvalid { + return v, nil + } + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "compute_id is deprecated, please use cluster_id instead", + Locations: computeId.Locations(), + Paths: []dyn.Path{computeIdPath}, + }) + + clusterIdPath := p.Append(dyn.Key("cluster_id")) + nv, err := dyn.SetByPath(v, clusterIdPath, computeId) + if err != nil { + return dyn.InvalidValue, diag.FromErr(err) + } + // Drop the "compute_id" key. + vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + switch len(p) { + case 0: + return v, nil + case 1: + if p[0] == dyn.Key("compute_id") { + return v, dyn.ErrDrop + } + return v, nil + case 2: + if p[1] == dyn.Key("compute_id") { + return v, dyn.ErrDrop + } + } + return v, dyn.ErrSkip + }) + + diags = diags.Extend(diag.FromErr(err)) + return vout, diags +} diff --git a/bundle/config/mutator/compute_id_compate_test.go b/bundle/config/mutator/compute_id_compate_test.go new file mode 100644 index 00000000..e59d37e3 --- /dev/null +++ b/bundle/config/mutator/compute_id_compate_test.go @@ -0,0 +1,57 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" +) + +func TestComputeIdToClusterId(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + ComputeId: "compute-id", + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId()) + assert.NoError(t, diags.Error()) + assert.Equal(t, "compute-id", b.Config.Bundle.ClusterId) + assert.Empty(t, b.Config.Bundle.ComputeId) + + assert.Len(t, diags, 1) + assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary) + assert.Equal(t, diag.Warning, diags[0].Severity) +} + +func TestComputeIdToClusterIdInTargetOverride(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Targets: map[string]*config.Target{ + "dev": { + ComputeId: "compute-id-dev", + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId()) + assert.NoError(t, diags.Error()) + assert.Empty(t, b.Config.Targets["dev"].ComputeId) + + diags = diags.Extend(bundle.Apply(context.Background(), b, mutator.SelectTarget("dev"))) + assert.NoError(t, diags.Error()) + + assert.Equal(t, "compute-id-dev", b.Config.Bundle.ClusterId) + assert.Empty(t, b.Config.Bundle.ComputeId) + + assert.Len(t, diags, 1) + assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary) + assert.Equal(t, diag.Warning, diags[0].Severity) +} diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 0458beff..faf50ae6 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -23,6 +23,7 @@ func DefaultMutators() []bundle.Mutator { VerifyCliVersion(), EnvironmentsToTargets(), + ComputeIdToClusterId(), InitializeVariables(), DefineDefaultTarget(), LoadGitDetails(), diff --git a/bundle/config/mutator/override_compute.go b/bundle/config/mutator/override_compute.go index 73fbad36..5700cdf2 100644 --- a/bundle/config/mutator/override_compute.go +++ b/bundle/config/mutator/override_compute.go @@ -39,22 +39,22 @@ func overrideJobCompute(j *resources.Job, compute string) { func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Bundle.Mode != config.Development { - if b.Config.Bundle.ComputeID != "" { + if b.Config.Bundle.ClusterId != "" { return diag.Errorf("cannot override compute for an target that does not use 'mode: development'") } return nil } if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" { - b.Config.Bundle.ComputeID = v + b.Config.Bundle.ClusterId = v } - if b.Config.Bundle.ComputeID == "" { + if b.Config.Bundle.ClusterId == "" { return nil } r := b.Config.Resources for i := range r.Jobs { - overrideJobCompute(r.Jobs[i], b.Config.Bundle.ComputeID) + overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId) } return nil diff --git a/bundle/config/mutator/override_compute_test.go b/bundle/config/mutator/override_compute_test.go index 152ee543..369447d7 100644 --- a/bundle/config/mutator/override_compute_test.go +++ b/bundle/config/mutator/override_compute_test.go @@ -20,7 +20,7 @@ func TestOverrideDevelopment(t *testing.T) { Config: config.Root{ Bundle: config.Bundle{ Mode: config.Development, - ComputeID: "newClusterID", + ClusterId: "newClusterID", }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -144,7 +144,7 @@ func TestOverrideProduction(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Bundle: config.Bundle{ - ComputeID: "newClusterID", + ClusterId: "newClusterID", }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 42f1929c..b0eb57ee 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/tags" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" @@ -119,6 +120,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Schemas: map[string]*resources.Schema{ "schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}}, }, + Clusters: map[string]*resources.Cluster{ + "cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, + }, }, }, // Use AWS implementation for testing. @@ -177,6 +181,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Schema 1 assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) + + // Clusters + assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { @@ -281,6 +288,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) + assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeProduction(t *testing.T) { @@ -312,6 +320,7 @@ func TestProcessTargetModeProduction(t *testing.T) { b.Config.Resources.Experiments["experiment2"].Permissions = permissions b.Config.Resources.Models["model1"].Permissions = permissions b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Permissions = permissions + b.Config.Resources.Clusters["cluster1"].Permissions = permissions diags = validateProductionMode(context.Background(), b, false) require.NoError(t, diags.Error()) @@ -322,6 +331,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) + assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) { diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index e6cef9ba..abeea45d 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string { // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource require.Equal(t, []string{ + "clusters", "experiments", "jobs", "model_serving_endpoints", @@ -133,6 +134,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { // some point in the future. These resources are (implicitly) on the deny list, since // they are not on the allow list below. allowList := []string{ + "clusters", "jobs", "models", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 22d69ffb..a3afb7fc 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -19,6 +19,7 @@ type Resources struct { RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` Schemas map[string]*resources.Schema `json:"schemas,omitempty"` + Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` } type ConfigResource interface { diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go new file mode 100644 index 00000000..63234566 --- /dev/null +++ b/bundle/config/resources/clusters.go @@ -0,0 +1,39 @@ +package resources + +import ( + "context" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type Cluster struct { + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + + *compute.ClusterSpec +} + +func (s *Cluster) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Cluster) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Clusters.GetByClusterId(ctx, id) + if err != nil { + log.Debugf(ctx, "cluster %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *Cluster) TerraformResourceName() string { + return "databricks_cluster" +} diff --git a/bundle/config/root.go b/bundle/config/root.go index 884c2e1c..92d834f0 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -366,9 +366,9 @@ func (r *Root) MergeTargetOverrides(name string) error { } } - // Merge `compute_id`. This field must be overwritten if set, not merged. - if v := target.Get("compute_id"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("compute_id")), v) + // Merge `cluster_id`. This field must be overwritten if set, not merged. + if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid { + root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v) if err != nil { return err } diff --git a/bundle/config/target.go b/bundle/config/target.go index fc6ba7b5..fae9c940 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -24,8 +24,11 @@ type Target struct { // name prefix of deployed resources. Presets Presets `json:"presets,omitempty"` - // Overrides the compute used for jobs and other supported assets. - ComputeID string `json:"compute_id,omitempty"` + // DEPRECATED: Overrides the compute used for jobs and other supported assets. + ComputeId string `json:"compute_id,omitempty"` + + // Overrides the cluster used for jobs and other supported assets. + ClusterId string `json:"cluster_id,omitempty"` Bundle *Bundle `json:"bundle,omitempty"` diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index f13c241c..5a548e3b 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -231,6 +231,13 @@ func BundleToTerraform(config *config.Root) *schema.Root { tfroot.Resource.QualityMonitor[k] = &dst } + for k, src := range config.Resources.Clusters { + noResources = false + var dst schema.ResourceCluster + conv(src, &dst) + tfroot.Resource.Cluster[k] = &dst + } + // We explicitly set "resource" to nil to omit it from a JSON encoding. // This is required because the terraform CLI requires >= 1 resources defined // if the "resource" property is used in a .tf.json file. @@ -394,6 +401,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.Schemas[resource.Name] = cur + case "databricks_cluster": + if config.Resources.Clusters == nil { + config.Resources.Clusters = make(map[string]*resources.Cluster) + } + cur := config.Resources.Clusters[resource.Name] + if cur == nil { + cur = &resources.Cluster{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Clusters[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -443,6 +460,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Clusters { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index e4ef6114..4c6866d9 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -663,6 +663,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -692,6 +700,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -754,6 +765,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Clusters: map[string]*resources.Cluster{ + "test_cluster": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster", + }, + }, + }, }, } var tfState = resourcesState{ @@ -786,6 +804,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -888,6 +909,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Clusters: map[string]*resources.Cluster{ + "test_cluster": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster", + }, + }, + "test_cluster_new": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -1020,6 +1053,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "2"}}, }, }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1081,6 +1130,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Clusters["test_cluster_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index faa098e1..12894c68 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -58,6 +58,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) case dyn.Key("schemas"): path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) + case dyn.Key("clusters"): + path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 5ceb243b..630a904a 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -31,6 +31,7 @@ func TestInterpolate(t *testing.T) { "other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}", "other_registered_model": "${resources.registered_models.other_registered_model.id}", "other_schema": "${resources.schemas.other_schema.id}", + "other_cluster": "${resources.clusters.other_cluster.id}", }, Tasks: []jobs.Task{ { @@ -67,6 +68,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"]) assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) + assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go new file mode 100644 index 00000000..f25f09ea --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -0,0 +1,52 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func convertClusterResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(compute.ClusterSpec{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "cluster normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type clusterConverter struct{} + +func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertClusterResource(ctx, vin) + if err != nil { + return err + } + + // We always set no_wait as it allows DABs not to wait for cluster to be started. + vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Cluster[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key) + out.Permissions["cluster_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("clusters", clusterConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go new file mode 100644 index 00000000..e7d2542f --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go @@ -0,0 +1,97 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertCluster(t *testing.T) { + var src = resources.Cluster{ + ClusterSpec: &compute.ClusterSpec{ + NumWorkers: 3, + SparkVersion: "13.3.x-scala2.12", + ClusterName: "cluster", + SparkConf: map[string]string{ + "spark.executor.memory": "2g", + }, + AwsAttributes: &compute.AwsAttributes{ + Availability: "ON_DEMAND", + }, + AzureAttributes: &compute.AzureAttributes{ + Availability: "SPOT", + }, + DataSecurityMode: "USER_ISOLATION", + NodeTypeId: "m5.xlarge", + Autoscale: &compute.AutoScale{ + MinWorkers: 1, + MaxWorkers: 10, + }, + }, + + Permissions: []resources.Permission{ + { + Level: "CAN_RUN", + UserName: "jack@gmail.com", + }, + { + Level: "CAN_MANAGE", + ServicePrincipalName: "sp", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = clusterConverter{}.Convert(ctx, "my_cluster", vin, out) + require.NoError(t, err) + + cluster := out.Cluster["my_cluster"] + assert.Equal(t, map[string]any{ + "num_workers": int64(3), + "spark_version": "13.3.x-scala2.12", + "cluster_name": "cluster", + "spark_conf": map[string]any{ + "spark.executor.memory": "2g", + }, + "aws_attributes": map[string]any{ + "availability": "ON_DEMAND", + }, + "azure_attributes": map[string]any{ + "availability": "SPOT", + }, + "data_security_mode": "USER_ISOLATION", + "no_wait": true, + "node_type_id": "m5.xlarge", + "autoscale": map[string]any{ + "min_workers": int64(1), + "max_workers": int64(10), + }, + }, cluster) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + JobId: "${databricks_cluster.my_cluster.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_RUN", + UserName: "jack@gmail.com", + }, + { + PermissionLevel: "CAN_MANAGE", + ServicePrincipalName: "sp", + }, + }, + }, out.Permissions["cluster_my_cluster"]) + +} diff --git a/bundle/tests/clusters/databricks.yml b/bundle/tests/clusters/databricks.yml new file mode 100644 index 00000000..1074462a --- /dev/null +++ b/bundle/tests/clusters/databricks.yml @@ -0,0 +1,36 @@ +bundle: + name: clusters + +workspace: + host: https://acme.cloud.databricks.com/ + +resources: + clusters: + foo: + cluster_name: foo + num_workers: 2 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 2 + max_workers: 7 + spark_version: "13.3.x-scala2.12" + spark_conf: + "spark.executor.memory": "2g" + +targets: + default: + + development: + resources: + clusters: + foo: + cluster_name: foo-override + num_workers: 3 + node_type_id: "m5.xlarge" + autoscale: + min_workers: 1 + max_workers: 3 + spark_version: "15.2.x-scala2.12" + spark_conf: + "spark.executor.memory": "4g" + "spark.executor.memory2": "4g" diff --git a/bundle/tests/clusters_test.go b/bundle/tests/clusters_test.go new file mode 100644 index 00000000..def8a2a3 --- /dev/null +++ b/bundle/tests/clusters_test.go @@ -0,0 +1,36 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestClusters(t *testing.T) { + b := load(t, "./clusters") + assert.Equal(t, "clusters", b.Config.Bundle.Name) + + cluster := b.Config.Resources.Clusters["foo"] + assert.Equal(t, "foo", cluster.ClusterName) + assert.Equal(t, "13.3.x-scala2.12", cluster.SparkVersion) + assert.Equal(t, "i3.xlarge", cluster.NodeTypeId) + assert.Equal(t, 2, cluster.NumWorkers) + assert.Equal(t, "2g", cluster.SparkConf["spark.executor.memory"]) + assert.Equal(t, 2, cluster.Autoscale.MinWorkers) + assert.Equal(t, 7, cluster.Autoscale.MaxWorkers) +} + +func TestClustersOverride(t *testing.T) { + b := loadTarget(t, "./clusters", "development") + assert.Equal(t, "clusters", b.Config.Bundle.Name) + + cluster := b.Config.Resources.Clusters["foo"] + assert.Equal(t, "foo-override", cluster.ClusterName) + assert.Equal(t, "15.2.x-scala2.12", cluster.SparkVersion) + assert.Equal(t, "m5.xlarge", cluster.NodeTypeId) + assert.Equal(t, 3, cluster.NumWorkers) + assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory"]) + assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory2"]) + assert.Equal(t, 1, cluster.Autoscale.MinWorkers) + assert.Equal(t, 3, cluster.Autoscale.MaxWorkers) +} diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 49231734..f1c85cb3 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -24,14 +24,16 @@ func newDeployCommand() *cobra.Command { var force bool var forceLock bool var failOnActiveRuns bool - var computeID string + var clusterId string var autoApprove bool var verbose bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") - cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") + cmd.Flags().StringVar(&clusterId, "compute-id", "", "Override cluster in the deployment with the given compute ID.") + cmd.Flags().StringVarP(&clusterId, "cluster-id", "c", "", "Override cluster in the deployment with the given cluster ID.") cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") + cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead") cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") // Verbose flag currently only affects file sync output, it's used by the vscode extension cmd.Flags().MarkHidden("verbose") @@ -47,7 +49,10 @@ func newDeployCommand() *cobra.Command { b.AutoApprove = autoApprove if cmd.Flag("compute-id").Changed { - b.Config.Bundle.ComputeID = computeID + b.Config.Bundle.ClusterId = clusterId + } + if cmd.Flag("cluster-id").Changed { + b.Config.Bundle.ClusterId = clusterId } if cmd.Flag("fail-on-active-runs").Changed { b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns diff --git a/internal/bundle/bundles/clusters/databricks_template_schema.json b/internal/bundle/bundles/clusters/databricks_template_schema.json new file mode 100644 index 00000000..c1c5cf12 --- /dev/null +++ b/internal/bundle/bundles/clusters/databricks_template_schema.json @@ -0,0 +1,16 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + } + } +} diff --git a/internal/bundle/bundles/clusters/template/databricks.yml.tmpl b/internal/bundle/bundles/clusters/template/databricks.yml.tmpl new file mode 100644 index 00000000..e0d6320a --- /dev/null +++ b/internal/bundle/bundles/clusters/template/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: basic + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +resources: + clusters: + test_cluster: + cluster_name: "test-cluster-{{.unique_id}}" + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + num_workers: 2 + spark_conf: + "spark.executor.memory": "2g" + + jobs: + foo: + name: test-job-with-cluster-{{.unique_id}} + tasks: + - task_key: my_notebook_task + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + spark_python_task: + python_file: ./hello_world.py diff --git a/internal/bundle/bundles/clusters/template/hello_world.py b/internal/bundle/bundles/clusters/template/hello_world.py new file mode 100644 index 00000000..f301245e --- /dev/null +++ b/internal/bundle/bundles/clusters/template/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/internal/bundle/clusters_test.go b/internal/bundle/clusters_test.go new file mode 100644 index 00000000..a961f3ea --- /dev/null +++ b/internal/bundle/clusters_test.go @@ -0,0 +1,56 @@ +package bundle + +import ( + "fmt" + "testing" + + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestAccDeployBundleWithCluster(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + if testutil.IsAWSCloud(wt.T) { + t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters") + } + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + root, err := initTestTemplate(t, ctx, "clusters", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + t.Cleanup(func() { + err = destroyBundle(t, ctx, root) + require.NoError(t, err) + + cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId)) + if err != nil { + require.ErrorContains(t, err, "does not exist") + } else { + require.Contains(t, []compute.State{compute.StateTerminated, compute.StateTerminating}, cluster.State) + } + + }) + + err = deployBundle(t, ctx, root) + require.NoError(t, err) + + // Cluster should exists after bundle deployment + cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId)) + require.NoError(t, err) + require.NotNil(t, cluster) + + out, err := runResource(t, ctx, root, "foo") + require.NoError(t, err) + require.Contains(t, out, "Hello World!") +} diff --git a/internal/testutil/cloud.go b/internal/testutil/cloud.go index e547069f..ba5b75ec 100644 --- a/internal/testutil/cloud.go +++ b/internal/testutil/cloud.go @@ -49,3 +49,7 @@ func GetCloud(t *testing.T) Cloud { } return -1 } + +func IsAWSCloud(t *testing.T) bool { + return GetCloud(t) == AWS +} From 0cc35ca05693e5989308f432f22bb0a28f8cb1dd Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:12:30 +0530 Subject: [PATCH 46/78] Assert tokens are redacted in origin URL when username is not specified (#1785) TSIA --- libs/git/repository_test.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/libs/git/repository_test.go b/libs/git/repository_test.go index a28038ee..93d9a03d 100644 --- a/libs/git/repository_test.go +++ b/libs/git/repository_test.go @@ -209,7 +209,26 @@ func TestRepositoryGitConfigWhenNotARepo(t *testing.T) { } func TestRepositoryOriginUrlRemovesUserCreds(t *testing.T) { - repo := newTestRepository(t) - repo.addOriginUrl("https://username:token@github.com/databricks/foobar.git") - repo.assertOriginUrl("https://github.com/databricks/foobar.git") + tcases := []struct { + url string + expected string + }{ + { + url: "https://username:token@github.com/databricks/foobar.git", + expected: "https://github.com/databricks/foobar.git", + }, + { + // Note: The token is still considered and parsed as a username here. + // However credentials integrations by Git providers like GitHub + // allow for setting a PAT token as a username. + url: "https://token@github.com/databricks/foobar.git", + expected: "https://github.com/databricks/foobar.git", + }, + } + + for _, tc := range tcases { + repo := newTestRepository(t) + repo.addOriginUrl(tc.url) + repo.assertOriginUrl(tc.expected) + } } From 490259a14aec0a53fe6bd97f9d1ea5a384e74773 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 24 Sep 2024 15:51:54 +0200 Subject: [PATCH 47/78] Refactor jobs path translation (#1782) ## Changes Extract package for other modules to transform different kinds of paths in job resources. ## Tests Unit tests --- .../config/mutator/paths/job_paths_visitor.go | 115 ++++++++++++ .../mutator/paths/job_paths_visitor_test.go | 168 ++++++++++++++++++ bundle/config/mutator/paths/visitor.go | 26 +++ bundle/config/mutator/translate_paths_jobs.go | 137 ++++---------- 4 files changed, 340 insertions(+), 106 deletions(-) create mode 100644 bundle/config/mutator/paths/job_paths_visitor.go create mode 100644 bundle/config/mutator/paths/job_paths_visitor_test.go create mode 100644 bundle/config/mutator/paths/visitor.go diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go new file mode 100644 index 00000000..275a8fa5 --- /dev/null +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -0,0 +1,115 @@ +package paths + +import ( + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/dyn" +) + +type jobRewritePattern struct { + pattern dyn.Pattern + kind PathKind + skipRewrite func(string) bool +} + +func noSkipRewrite(string) bool { + return false +} + +func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { + return []jobRewritePattern{ + { + base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + PathKindNotebook, + noSkipRewrite, + }, + { + base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + { + base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + PathKindDirectory, + noSkipRewrite, + }, + { + base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), + PathKindLibrary, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), + PathKindLibrary, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + } +} + +func jobRewritePatterns() []jobRewritePattern { + // Base pattern to match all tasks in all jobs. + base := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + ) + + // Compile list of patterns and their respective rewrite functions. + jobEnvironmentsPatterns := []jobRewritePattern{ + { + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("dependencies"), + dyn.AnyIndex(), + ), + PathKindWithPrefix, + func(s string) bool { + return !libraries.IsLibraryLocal(s) + }, + }, + } + + taskPatterns := jobTaskRewritePatterns(base) + forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) + allPatterns = append(allPatterns, forEachPatterns...) + return allPatterns +} + +// VisitJobPaths visits all paths in job resources and applies a function to each path. +func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + var err error + var newValue = value + + for _, rewritePattern := range jobRewritePatterns() { + newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if rewritePattern.skipRewrite(v.MustString()) { + return v, nil + } + + return fn(p, rewritePattern.kind, v) + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + + return newValue, nil +} diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go new file mode 100644 index 00000000..7f020157 --- /dev/null +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -0,0 +1,168 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestVisitJobPaths(t *testing.T) { + task0 := jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "abc", + }, + } + task1 := jobs.Task{ + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "abc", + }, + } + task2 := jobs.Task{ + DbtTask: &jobs.DbtTask{ + ProjectDirectory: "abc", + }, + } + task3 := jobs.Task{ + SqlTask: &jobs.SqlTask{ + File: &jobs.SqlTaskFile{ + Path: "abc", + }, + }, + } + task4 := jobs.Task{ + Libraries: []compute.Library{ + {Whl: "dist/foo.whl"}, + }, + } + task5 := jobs.Task{ + Libraries: []compute.Library{ + {Jar: "dist/foo.jar"}, + }, + } + task6 := jobs.Task{ + Libraries: []compute.Library{ + {Requirements: "requirements.txt"}, + }, + } + + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + task0, + task1, + task2, + task3, + task4, + task5, + task6, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"), + dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"), + dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"), + dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"), + dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"), + dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"), + dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func TestVisitJobPaths_environments(t *testing.T) { + environment0 := jobs.JobEnvironment{ + Spec: &compute.Environment{ + Dependencies: []string{ + "dist_0/*.whl", + "dist_1/*.whl", + }, + }, + } + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Environments: []jobs.JobEnvironment{ + environment0, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"), + dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func TestVisitJobPaths_foreach(t *testing.T) { + task0 := jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "abc", + }, + }, + }, + } + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + task0, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func visitJobPaths(t *testing.T, root config.Root) []dyn.Path { + var actual []dyn.Path + err := root.Mutate(func(value dyn.Value) (dyn.Value, error) { + return VisitJobPaths(value, func(p dyn.Path, kind PathKind, v dyn.Value) (dyn.Value, error) { + actual = append(actual, p) + return v, nil + }) + }) + require.NoError(t, err) + return actual +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go new file mode 100644 index 00000000..40d1f14e --- /dev/null +++ b/bundle/config/mutator/paths/visitor.go @@ -0,0 +1,26 @@ +package paths + +import "github.com/databricks/cli/libs/dyn" + +type PathKind int + +const ( + // PathKindLibrary is a path to a library file + PathKindLibrary = iota + + // PathKindNotebook is a path to a notebook file + PathKindNotebook + + // PathKindWorkspaceFile is a path to a regular workspace file, + // notebooks are not allowed because they are uploaded a special + // kind of workspace object. + PathKindWorkspaceFile + + // PathKindWithPrefix is a path that starts with './' + PathKindWithPrefix + + // PathKindDirectory is a path to directory + PathKindDirectory +) + +type VisitFunc func(path dyn.Path, kind PathKind, value dyn.Value) (dyn.Value, error) diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index e34eeb2f..c29ff0ea 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -4,97 +4,11 @@ import ( "fmt" "slices" - "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/dyn" ) -type jobRewritePattern struct { - pattern dyn.Pattern - fn rewriteFunc - skipRewrite func(string) bool -} - -func noSkipRewrite(string) bool { - return false -} - -func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern { - return []jobRewritePattern{ - { - base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), - t.translateNotebookPath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), - t.translateFilePath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), - t.translateDirectoryPath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), - t.translateFilePath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), - t.translateNoOp, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), - t.translateNoOp, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), - t.translateFilePath, - noSkipRewrite, - }, - } -} - -func (t *translateContext) jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) - - // Compile list of patterns and their respective rewrite functions. - jobEnvironmentsPatterns := []jobRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - t.translateNoOpWithPrefix, - func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, - } - - taskPatterns := rewritePatterns(t, base) - forEachPatterns := rewritePatterns(t, base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) - allPatterns = append(allPatterns, forEachPatterns...) - return allPatterns -} - func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) { var err error @@ -111,30 +25,41 @@ func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) } } - for _, rewritePattern := range t.jobRewritePatterns() { - v, err = dyn.MapByPattern(v, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + return paths.VisitJobPaths(v, func(p dyn.Path, kind paths.PathKind, v dyn.Value) (dyn.Value, error) { + key := p[2].Key() - // Skip path translation if the job is using git source. - if slices.Contains(ignore, key) { - return v, nil - } + // Skip path translation if the job is using git source. + if slices.Contains(ignore, key) { + return v, nil + } - dir, err := v.Location().Directory() - if err != nil { - return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err) - } + dir, err := v.Location().Directory() + if err != nil { + return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err) + } - sv := v.MustString() - if rewritePattern.skipRewrite(sv) { - return v, nil - } - return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, fallback[key]) - }) + rewritePatternFn, err := t.getRewritePatternFn(kind) if err != nil { return dyn.InvalidValue, err } + + return t.rewriteRelativeTo(p, v, rewritePatternFn, dir, fallback[key]) + }) +} + +func (t *translateContext) getRewritePatternFn(kind paths.PathKind) (rewriteFunc, error) { + switch kind { + case paths.PathKindLibrary: + return t.translateNoOp, nil + case paths.PathKindNotebook: + return t.translateNotebookPath, nil + case paths.PathKindWorkspaceFile: + return t.translateFilePath, nil + case paths.PathKindDirectory: + return t.translateDirectoryPath, nil + case paths.PathKindWithPrefix: + return t.translateNoOpWithPrefix, nil } - return v, nil + return nil, fmt.Errorf("unsupported path kind: %d", kind) } From 3d9decdda9638fb5495212611307d70257b3d3e6 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 25 Sep 2024 13:30:14 +0200 Subject: [PATCH 48/78] Add JobTaskClusterSpec validate mutator (#1784) ## Changes Add JobTaskClusterSpec validate mutator. It catches the case when tasks don't which cluster to use. For example, we can get this error with minor modifications to `default-python` template: ```yaml tasks: - task_key: python_file_task spark_python_task: python_file: ../src/my_project_10/main.py ``` ``` % databricks bundle validate Error: Missing required cluster or environment settings at resources.jobs.my_project_10_job.tasks[0] in resources/my_project_10_job.yml:17:11 Task "print_github_stars" requires a cluster or an environment to run. Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. ``` We implicitly rely on "one of" validation, which does not exist. Many bundle fields can't co-exist, for instance, specifying: `JobTask.{existing_cluster_id,job_cluster_key}`, `Library.{whl,pypi}`, `JobTask.{notebook_task,python_wheel_task}`, etc. ## Tests Unit tests --------- Co-authored-by: Pieter Noordhuis --- .../config/validate/job_task_cluster_spec.go | 161 ++++++++++++++ .../validate/job_task_cluster_spec_test.go | 203 ++++++++++++++++++ bundle/config/validate/validate.go | 1 + 3 files changed, 365 insertions(+) create mode 100644 bundle/config/validate/job_task_cluster_spec.go create mode 100644 bundle/config/validate/job_task_cluster_spec_test.go diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go new file mode 100644 index 00000000..b80befcd --- /dev/null +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -0,0 +1,161 @@ +package validate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// JobTaskClusterSpec validates that job tasks have cluster spec defined +// if task requires a cluster +func JobTaskClusterSpec() bundle.ReadOnlyMutator { + return &jobTaskClusterSpec{} +} + +type jobTaskClusterSpec struct { +} + +func (v *jobTaskClusterSpec) Name() string { + return "validate:job_task_cluster_spec" +} + +func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) + + for resourceName, job := range rb.Config().Resources.Jobs { + resourcePath := jobsPath.Append(dyn.Key(resourceName)) + + for taskIndex, task := range job.Tasks { + taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) + + diags = diags.Extend(validateJobTask(rb, task, taskPath)) + } + } + + return diags +} + +func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { + diags := diag.Diagnostics{} + + var specified []string + var unspecified []string + + if task.JobClusterKey != "" { + specified = append(specified, "job_cluster_key") + } else { + unspecified = append(unspecified, "job_cluster_key") + } + + if task.EnvironmentKey != "" { + specified = append(specified, "environment_key") + } else { + unspecified = append(unspecified, "environment_key") + } + + if task.ExistingClusterId != "" { + specified = append(specified, "existing_cluster_id") + } else { + unspecified = append(unspecified, "existing_cluster_id") + } + + if task.NewCluster != nil { + specified = append(specified, "new_cluster") + } else { + unspecified = append(unspecified, "new_cluster") + } + + if task.ForEachTask != nil { + forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task")) + + diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath)) + } + + if isComputeTask(task) && len(specified) == 0 { + if task.NotebookTask != nil { + // notebook tasks without cluster spec will use notebook environment + } else { + // path might be not very helpful, adding user-specified task key clarifies the context + detail := fmt.Sprintf( + "Task %q requires a cluster or an environment to run.\nSpecify one of the following fields: %s.", + task.TaskKey, + strings.Join(unspecified, ", "), + ) + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Missing required cluster or environment settings", + Detail: detail, + Locations: rb.Config().GetLocations(taskPath.String()), + Paths: []dyn.Path{taskPath}, + }) + } + } + + return diags +} + +// isComputeTask returns true if the task runs on a cluster or serverless GC +func isComputeTask(task jobs.Task) bool { + if task.NotebookTask != nil { + // if warehouse_id is set, it's SQL notebook that doesn't need cluster or serverless GC + if task.NotebookTask.WarehouseId != "" { + return false + } else { + // task settings don't require specifying a cluster/serverless GC, but task itself can run on one + // we handle that case separately in validateJobTask + return true + } + } + + if task.PythonWheelTask != nil { + return true + } + + if task.DbtTask != nil { + return true + } + + if task.SparkJarTask != nil { + return true + } + + if task.SparkSubmitTask != nil { + return true + } + + if task.SparkPythonTask != nil { + return true + } + + if task.SqlTask != nil { + return false + } + + if task.PipelineTask != nil { + // while pipelines use clusters, pipeline tasks don't, they only trigger pipelines + return false + } + + if task.RunJobTask != nil { + return false + } + + if task.ConditionTask != nil { + return false + } + + // for each task doesn't use clusters, underlying task(s) can though + if task.ForEachTask != nil { + return false + } + + return false +} diff --git a/bundle/config/validate/job_task_cluster_spec_test.go b/bundle/config/validate/job_task_cluster_spec_test.go new file mode 100644 index 00000000..a3a7ccf2 --- /dev/null +++ b/bundle/config/validate/job_task_cluster_spec_test.go @@ -0,0 +1,203 @@ +package validate + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestJobTaskClusterSpec(t *testing.T) { + expectedSummary := "Missing required cluster or environment settings" + + type testCase struct { + name string + task jobs.Task + errorPath string + errorDetail string + errorSummary string + } + + testCases := []testCase{ + { + name: "valid notebook task", + task: jobs.Task{ + // while a cluster is needed, it will use notebook environment to create one + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (job_cluster_key)", + task: jobs.Task{ + JobClusterKey: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (new_cluster)", + task: jobs.Task{ + NewCluster: &compute.ClusterSpec{}, + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (existing_cluster_id)", + task: jobs.Task{ + ExistingClusterId: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid SQL notebook task", + task: jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + WarehouseId: "warehouse1", + }, + }, + }, + { + name: "valid python wheel task", + task: jobs.Task{ + JobClusterKey: "cluster1", + PythonWheelTask: &jobs.PythonWheelTask{}, + }, + }, + { + name: "valid python wheel task (environment_key)", + task: jobs.Task{ + EnvironmentKey: "environment1", + PythonWheelTask: &jobs.PythonWheelTask{}, + }, + }, + { + name: "valid dbt task", + task: jobs.Task{ + JobClusterKey: "cluster1", + DbtTask: &jobs.DbtTask{}, + }, + }, + { + name: "valid spark jar task", + task: jobs.Task{ + JobClusterKey: "cluster1", + SparkJarTask: &jobs.SparkJarTask{}, + }, + }, + { + name: "valid spark submit", + task: jobs.Task{ + NewCluster: &compute.ClusterSpec{}, + SparkSubmitTask: &jobs.SparkSubmitTask{}, + }, + }, + { + name: "valid spark python task", + task: jobs.Task{ + JobClusterKey: "cluster1", + SparkPythonTask: &jobs.SparkPythonTask{}, + }, + }, + { + name: "valid SQL task", + task: jobs.Task{ + SqlTask: &jobs.SqlTask{}, + }, + }, + { + name: "valid pipeline task", + task: jobs.Task{ + PipelineTask: &jobs.PipelineTask{}, + }, + }, + { + name: "valid run job task", + task: jobs.Task{ + RunJobTask: &jobs.RunJobTask{}, + }, + }, + { + name: "valid condition task", + task: jobs.Task{ + ConditionTask: &jobs.ConditionTask{}, + }, + }, + { + name: "valid for each task", + task: jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + JobClusterKey: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + }, + }, + { + name: "invalid python wheel task", + task: jobs.Task{ + PythonWheelTask: &jobs.PythonWheelTask{}, + TaskKey: "my_task", + }, + errorPath: "resources.jobs.job1.tasks[0]", + errorDetail: `Task "my_task" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`, + errorSummary: expectedSummary, + }, + { + name: "invalid for each task", + task: jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + PythonWheelTask: &jobs.PythonWheelTask{}, + TaskKey: "my_task", + }, + }, + }, + errorPath: "resources.jobs.job1.tasks[0].for_each_task.task", + errorDetail: `Task "my_task" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`, + errorSummary: expectedSummary, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + job := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{tc.task}, + }, + } + + b := createBundle(map[string]*resources.Job{"job1": job}) + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec()) + + if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" { + assert.Len(t, diags, 1) + assert.Len(t, diags[0].Paths, 1) + + diag := diags[0] + + assert.Equal(t, tc.errorPath, diag.Paths[0].String()) + assert.Equal(t, tc.errorSummary, diag.Summary) + assert.Equal(t, tc.errorDetail, diag.Detail) + } else { + assert.ElementsMatch(t, []string{}, diags) + } + }) + } +} + +func createBundle(jobs map[string]*resources.Job) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: jobs, + }, + }, + } +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index b4da0bc0..79f42bd2 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -34,6 +34,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics JobClusterKeyDefined(), FilesToSync(), ValidateSyncPatterns(), + JobTaskClusterSpec(), )) } From b3a3071086899dabbdf36f063d1cf892993090ff Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 25 Sep 2024 14:35:16 +0200 Subject: [PATCH 49/78] Fixed full variable override detection (#1787) ## Changes Fixes #1786 ## Tests All valid override combinations are added as test cases --- bundle/config/root.go | 37 +++++++++++++---- bundle/config/root_test.go | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 8 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 92d834f0..ff169e4c 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -418,22 +418,43 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } - // If the map has more than 2 keys, it is not a full variable override. - if mv.Len() > 2 { + // If the map has more than 3 keys, it is not a full variable override. + if mv.Len() > 3 { return false } - // If the map has 2 keys, one of them should be "default" and the other is "type" + // If the map has 3 keys, they should be "description", "type" and "default" or "lookup" + if mv.Len() == 3 { + if _, ok := mv.GetByString("type"); ok { + if _, ok := mv.GetByString("description"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } + } + } + + return false + } + + // If the map has 2 keys, one of them should be "default" or "lookup" and the other is "type" or "description" if mv.Len() == 2 { - if _, ok := mv.GetByString("type"); !ok { - return false + if _, ok := mv.GetByString("type"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } } - if _, ok := mv.GetByString("default"); !ok { - return false + if _, ok := mv.GetByString("description"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } + + if _, ok := mv.GetByString("lookup"); ok { + return true + } } - return true + return false } for _, keyword := range variableKeywords { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index d2c7a9b1..9e612353 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -169,3 +170,87 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { assert.Equal(t, "complex var", root.Variables["complex"].Description) } + +func TestIsFullVariableOverrideDef(t *testing.T) { + testCases := []struct { + value dyn.Value + expected bool + }{ + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + "description": dyn.V("foo var"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "lookup": dyn.V("foo"), + "description": dyn.V("foo var"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "lookup": dyn.V("foo"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "description": dyn.V("string"), + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "description": dyn.V("string"), + "lookup": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "lookup": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + "description": dyn.V("foo var"), + "lookup": dyn.V("foo"), + }), + expected: false, + }, + } + + for i, tc := range testCases { + assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i) + } + +} From a4ba0bbe9f332dbc497d6cc3be0e19436e2e9375 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:28:14 +0530 Subject: [PATCH 50/78] Add sub-extension to resource files in built-in templates (#1777) ## Changes We want to encourage a pattern of only specifying a single resource in a YAML file when an `..yml` (like `.job.yml`) is used. This convention could allow us to bijectively map a resource YAML file to it's corresponding resource in the Databricks workspace. This PR simply makes the built-in templates compliant to this format. ## Tests Existing tests. --- .../dbt-sql/template/{{.project_name}}/README.md.tmpl | 2 +- ...ect_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} | 0 .../templates/default-python/template/__preamble.tmpl | 4 ++-- .../default-python/template/{{.project_name}}/README.md.tmpl | 2 +- ...ect_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} | 2 +- ..._pipeline.yml.tmpl => {{.project_name}}.pipeline.yml.tmpl} | 0 .../template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl | 2 +- .../template/{{.project_name}}/src/notebook.ipynb.tmpl | 2 +- ...}}_sql_job.yml.tmpl => {{.project_name}}_sql.job.yml.tmpl} | 0 .../template/{{.project_name}}/src/orders_daily.sql.tmpl | 2 +- .../template/{{.project_name}}/src/orders_raw.sql.tmpl | 2 +- 11 files changed, 9 insertions(+), 9 deletions(-) rename libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{{.project_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} (100%) rename libs/template/templates/default-python/template/{{.project_name}}/resources/{{{.project_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} (97%) rename libs/template/templates/default-python/template/{{.project_name}}/resources/{{{.project_name}}_pipeline.yml.tmpl => {{.project_name}}.pipeline.yml.tmpl} (100%) rename libs/template/templates/default-sql/template/{{.project_name}}/resources/{{{.project_name}}_sql_job.yml.tmpl => {{.project_name}}_sql.job.yml.tmpl} (100%) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl index dbf8a8d8..cd4c29a7 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl @@ -121,7 +121,7 @@ You can find that job by opening your workpace and clicking on **Workflows**. You can also deploy to your production target directly from the command-line. The warehouse, catalog, and schema for that target are configured in databricks.yml. -When deploying to this target, note that the default job at resources/{{.project_name}}_job.yml +When deploying to this target, note that the default job at resources/{{.project_name}}.job.yml has a schedule set that runs every day. The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl similarity index 100% rename from libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl rename to libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl diff --git a/libs/template/templates/default-python/template/__preamble.tmpl b/libs/template/templates/default-python/template/__preamble.tmpl index a919a269..69b769cd 100644 --- a/libs/template/templates/default-python/template/__preamble.tmpl +++ b/libs/template/templates/default-python/template/__preamble.tmpl @@ -18,7 +18,7 @@ This file only template directives; it is skipped for the actual output. {{if $notDLT}} {{skip "{{.project_name}}/src/dlt_pipeline.ipynb"}} - {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline.yml"}} + {{skip "{{.project_name}}/resources/{{.project_name}}.pipeline.yml"}} {{end}} {{if $notNotebook}} @@ -26,7 +26,7 @@ This file only template directives; it is skipped for the actual output. {{end}} {{if (and $notDLT $notNotebook $notPython)}} - {{skip "{{.project_name}}/resources/{{.project_name}}_job.yml"}} + {{skip "{{.project_name}}/resources/{{.project_name}}.job.yml"}} {{else}} {{skip "{{.project_name}}/resources/.gitkeep"}} {{end}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl index 5adade0b..53847a9c 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl @@ -29,7 +29,7 @@ The '{{.project_name}}' project was generated by using the default-python templa ``` Note that the default job from the template has a schedule that runs every day - (defined in resources/{{.project_name}}_job.yml). The schedule + (defined in resources/{{.project_name}}.job.yml). The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl similarity index 97% rename from libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl rename to libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index d2100e90..5211e389 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -40,7 +40,7 @@ resources: - task_key: notebook_task {{- end}} pipeline_task: - {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}_pipeline.yml */}} + {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}.pipeline.yml */}} pipeline_id: ${resources.pipelines.{{.project_name}}_pipeline.id} {{end -}} {{- if (eq .include_python "yes") }} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl similarity index 100% rename from libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl rename to libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl index b152e9a3..253ed321 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# DLT pipeline\n", "\n", - "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}_pipeline.yml." + "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}.pipeline.yml." ] }, { diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl index a228f8d1..6782a053 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# Default notebook\n", "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}_job.yml." + "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}.job.yml." ] }, { diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql.job.yml.tmpl similarity index 100% rename from libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl rename to libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql.job.yml.tmpl diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index e5ceb77a..444ae4e0 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -1,4 +1,4 @@ --- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) +-- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql.job.yml) USE CATALOG {{"{{"}}catalog{{"}}"}}; USE IDENTIFIER({{"{{"}}schema{{"}}"}}); diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl index c73606ef..80f6773c 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl @@ -1,4 +1,4 @@ --- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) +-- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql.job.yml) -- -- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ -- See also https://docs.databricks.com/sql/language-manual/sql-ref-syntax-ddl-create-streaming-table.html From 7f1121d8d85900db0fc333ee901dfe6eb8488b3b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 25 Sep 2024 17:45:28 +0200 Subject: [PATCH 51/78] Pin Go toolchain to 1.22.7 (#1790) ## Changes Relates to https://github.com/databricks/cli/pull/1758. More information about toolchains: * https://go.dev/blog/toolchain * https://go.dev/doc/toolchain We need to specify the toolchain as we need to bump Go to 1.22.0 for the `mod` upgrade and want to use the latest toolchain on the 1.22 series. ## Tests The previous release was made with Go 1.22.7 so we should continue to use it. --- .github/workflows/push.yml | 6 +++--- .github/workflows/release-snapshot.yml | 2 +- .github/workflows/release.yml | 2 +- go.mod | 4 +++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 02bf7378..ee60da9d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 - name: Setup Python uses: actions/setup-python@v5 @@ -68,7 +68,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # No need to download cached dependencies when running gofmt. cache: false @@ -100,7 +100,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # Github repo: https://github.com/ajv-validator/ajv-cli - name: Install ajv-cli diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index defd1c53..6a601a5f 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 531fb39b..f9742a19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action diff --git a/go.mod b/go.mod index ba41ef3a..e1c7519f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/databricks/cli -go 1.22 +go 1.22.0 + +toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT From 495040e4cd2d8fbbbcc09aff6cf3b88cb4daee78 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:43:48 +0530 Subject: [PATCH 52/78] Modify SetLocation test utility to take full locations as argument (#1788) I plan to use this in https://github.com/databricks/cli/pull/1780, to set the line and column numbers as well for the locations. gopatch file used: ``` @@ var x expression var y expression var z expression @@ -bundletest.SetLocation(x, y, z) +bundletest.SetLocation(x, y, []dyn.Location{{File: z}}) ``` --- bundle/artifacts/expand_globs_test.go | 7 ++-- .../expand_pipeline_glob_paths_test.go | 5 +-- .../config/mutator/rewrite_sync_paths_test.go | 25 ++++++++------- bundle/config/mutator/sync_infer_root_test.go | 3 +- bundle/config/mutator/translate_paths_test.go | 32 +++++++++---------- bundle/deploy/metadata/compute_test.go | 7 ++-- bundle/internal/bundletest/location.go | 6 ++-- .../libraries/expand_glob_references_test.go | 7 ++-- 8 files changed, 48 insertions(+), 44 deletions(-) diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go index c9c47844..1665a480 100644 --- a/bundle/artifacts/expand_globs_test.go +++ b/bundle/artifacts/expand_globs_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestExpandGlobs_Nominal(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( @@ -77,7 +78,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( @@ -125,7 +126,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index d1671c25..07dd2021 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" @@ -105,8 +106,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) - bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", filepath.Join(dir, "relative", "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []dyn.Location{{File: filepath.Join(dir, "relative", "resource.yml")}}) m := ExpandPipelineGlobPaths() diags := bundle.Apply(context.Background(), b, m) diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index fa7f124b..a66f2763 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -33,12 +34,12 @@ func TestRewriteSyncPathsRelative(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml") - bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml") - bundletest.SetLocation(b, "sync.include[0]", "./file.yml") - bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml") - bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml") - bundletest.SetLocation(b, "sync.exclude[1]", "./a/b/c/file.yml") + bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "./file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "./a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "./a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "./a/b/c/file.yml"}}) diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) @@ -72,12 +73,12 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml") - bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml") - bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml") - bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml") - bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml") - bundletest.SetLocation(b, "sync.exclude[1]", "/tmp/dir/a/b/c/file.yml") + bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "/tmp/dir/file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "/tmp/dir/a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "/tmp/dir/a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "/tmp/dir/a/b/c/file.yml"}}) diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go index 383e5676..85e40adc 100644 --- a/bundle/config/mutator/sync_infer_root_test.go +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -184,7 +185,7 @@ func TestSyncInferRoot_Error(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths", "databricks.yml") + bundletest.SetLocation(b, "sync.paths", []dyn.Location{{File: "databricks.yml"}}) ctx := context.Background() diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 50fcd3b0..c03cee73 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -82,7 +82,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -210,7 +210,7 @@ func TestTranslatePaths(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -346,8 +346,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) - bundletest.SetLocation(b, "resources.pipelines", filepath.Join(dir, "pipeline/resource.yml")) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines", []dyn.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -408,7 +408,7 @@ func TestTranslatePathsOutsideSyncRoot(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "../resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), "is not contained in sync root path") @@ -439,7 +439,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found") @@ -470,7 +470,7 @@ func TestJobFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found") @@ -501,7 +501,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found") @@ -532,7 +532,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found") @@ -567,7 +567,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.jobs.job.tasks[0].spark_python_task.python_file" but got a notebook`) @@ -602,7 +602,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.jobs.job.tasks[0].notebook_task.notebook_path" but got a file`) @@ -637,7 +637,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.pipelines.pipeline.libraries[0].notebook.path" but got a file`) @@ -672,7 +672,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.pipelines.pipeline.libraries[0].file.path" but got a notebook`) @@ -710,7 +710,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -753,8 +753,8 @@ func TestTranslatePathWithComplexVariables(t *testing.T) { }, } - bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml")) - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + bundletest.SetLocation(b, "variables", []dyn.Location{{File: filepath.Join(dir, "variables/variables.yml")}}) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) ctx := context.Background() // Assign the variables to the dynamic configuration. diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index 6d43f845..2c2c7237 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/bundle/metadata" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,9 +56,9 @@ func TestComputeMetadataMutator(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.my-job-1", "a/b/c") - bundletest.SetLocation(b, "resources.jobs.my-job-2", "d/e/f") - bundletest.SetLocation(b, "resources.pipelines.my-pipeline", "abc") + bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}}) + bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}}) + bundletest.SetLocation(b, "resources.pipelines.my-pipeline", []dyn.Location{{File: "abc"}}) expectedMetadata := metadata.Metadata{ Version: metadata.Version, diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index 380d6e17..2ffd621b 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -8,15 +8,13 @@ import ( // SetLocation sets the location of all values in the bundle to the given path. // This is useful for testing where we need to associate configuration // with the path it is loaded from. -func SetLocation(b *bundle.Bundle, prefix string, filePath string) { +func SetLocation(b *bundle.Bundle, prefix string, locations []dyn.Location) { start := dyn.MustPathFromString(prefix) b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { // If the path has the given prefix, set the location. if p.HasPrefix(start) { - return v.WithLocations([]dyn.Location{{ - File: filePath, - }}), nil + return v.WithLocations(locations), nil } // The path is not nested under the given prefix. diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index e7f2e169..2dfbddb7 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -61,7 +62,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -146,7 +147,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -221,7 +222,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) From 875b112f801c8b04694e077cc07ed88a335db31b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:07:01 +0000 Subject: [PATCH 53/78] Bump golang.org/x/mod from 0.20.0 to 0.21.0 (#1758) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.20.0 to 0.21.0.
Commits
  • 46a3137 zip: set GIT_DIR in test when using bare repositories
  • 3afcd4e go.mod: set go version to 1.22.0
  • b1d336c go.mod: update required go version to go1.22
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.20.0&new-version=0.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester Co-authored-by: Pieter Noordhuis --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1c7519f..b6478a91 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.20.0 + golang.org/x/mod v0.21.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.24.0 diff --git a/go.sum b/go.sum index 3d4a2cdc..80fa43fd 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 94d8c3ba1e18abb82ca4dde85210cc1d2134f303 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:29:34 +0000 Subject: [PATCH 54/78] Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 (#1772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hashicorp/hc-install](https://github.com/hashicorp/hc-install) from 0.7.0 to 0.9.0.
Release notes

Sourced from github.com/hashicorp/hc-install's releases.

v0.9.0

What's Changed

Full Changelog: https://github.com/hashicorp/hc-install/compare/v0.8.1...v0.9.0

v0.8.1

What's Changed

New Contributors

Full Changelog: https://github.com/hashicorp/hc-install/compare/v0.8.0...v0.8.1

v0.8.0

ENHANCEMENTS:

BUG FIXES:

INTERNAL:

... (truncated)

Commits
  • 157a802 Merge pull request #250 from hashicorp/release-0.9.0
  • 4c734fc Prepare for v0.9.0 release
  • d78b328 Merge pull request #249 from hashicorp/d-contributing-md-update
  • 34f38b0 docs: Update release instructions
  • 6a5aa83 build(deps): bump golang.org/x/mod from 0.20.0 to 0.21.0 (#242)
  • 1784fcc Merge pull request #248 from hashicorp/revert-version-contents
  • ea2c69b Finish Release of 0.8.1 by updating VERSION
  • 4f3e00e Releasing 0.8.1
  • c6d1ced Merge pull request #246 from hashicorp/update-contributing
  • eea12f1 Update CONTRIBUTING.md to add clean up step
  • Additional commits viewable in compare view

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/hashicorp/hc-install | [>= 0.8.a, < 0.9] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/hc-install&package-manager=go_modules&previous-version=0.7.0&new-version=0.9.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 ++- go.sum | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b6478a91..0cf3ef8a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.9.0 // MPL 2.0 github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause @@ -51,6 +51,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 80fa43fd..d8866775 100644 --- a/go.sum +++ b/go.sum @@ -99,10 +99,14 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= -github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/hc-install v0.9.0 h1:2dIk8LcvANwtv3QZLckxcjyF5w8KVtiMxu6G6eLhghE= +github.com/hashicorp/hc-install v0.9.0/go.mod h1:+6vOP+mf3tuGgMApVYtmsnDoKWMDcFXeTxCACYZ8SFg= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= From 66f2ba64a8a479d45efb9b23eab096a5ffda1367 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Sep 2024 14:55:07 +0200 Subject: [PATCH 55/78] Simplified isFullVariableOverrideDef implementation (#1791) ## Changes Simplified isFullVariableOverrideDef implementation Follow up on https://github.com/databricks/cli/pull/1787 --- bundle/config/root.go | 54 ++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index ff169e4c..4b146745 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -406,7 +406,14 @@ func (r *Root) MergeTargetOverrides(name string) error { return r.updateWithDynamicValue(root) } -var variableKeywords = []string{"default", "lookup"} +var allowedVariableDefinitions = []([]string){ + {"default", "type", "description"}, + {"default", "type"}, + {"default", "description"}, + {"lookup", "description"}, + {"default"}, + {"lookup"}, +} // isFullVariableOverrideDef checks if the given value is a full syntax varaible override. // A full syntax variable override is a map with either 1 of 2 keys. @@ -423,42 +430,21 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } - // If the map has 3 keys, they should be "description", "type" and "default" or "lookup" - if mv.Len() == 3 { - if _, ok := mv.GetByString("type"); ok { - if _, ok := mv.GetByString("description"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } + for _, keys := range allowedVariableDefinitions { + if len(keys) != mv.Len() { + continue + } + + // Check if the keys are the same. + match := true + for _, key := range keys { + if _, ok := mv.GetByString(key); !ok { + match = false + break } } - return false - } - - // If the map has 2 keys, one of them should be "default" or "lookup" and the other is "type" or "description" - if mv.Len() == 2 { - if _, ok := mv.GetByString("type"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } - } - - if _, ok := mv.GetByString("description"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } - - if _, ok := mv.GetByString("lookup"); ok { - return true - } - } - - return false - } - - for _, keyword := range variableKeywords { - if _, ok := mv.GetByString(keyword); ok { + if match { return true } } From 4e8e02738081b74017bcb9d7b440e75ffa08c0d7 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:52:22 +0530 Subject: [PATCH 56/78] Sort tasks by `task_key` before generating the Terraform configuration (#1776) ## Changes Sort the tasks in the resultant `bundle.tf.json`. This is important because configuration from one task can leak into another if the tasks are not sorted. For more details see: 1. https://github.com/databricks/terraform-provider-databricks/issues/3951 2. https://github.com/databricks/terraform-provider-databricks/issues/4011 ## Tests Unit test and manually. For manual testing I used the following configuration: ``` resources: jobs: foo: tasks: - task_key: task-Z notebook_task: notebook_path: nb.py source: GIT existing_cluster_id: 0715-133738-ju0ma84z - task_key: task-1 notebook_task: notebook_path: ${workspace.file_path}/local.py source: WORKSPACE existing_cluster_id: 0715-133738-ju0ma84z depends_on: - task_key: task-Z git_source: git_provider: gitHub git_url: https://github.com/shreyas-goenka/job-source-tmp.git git_branch: main ``` Steps (1): 1. Deploy this bundle. 2. Comment out "source: GIT" 3. Deploy again Before: Deploying this bundle twice would fail. This is because the "source: GIT" would carry over to the next deployment. After: There was no error on the subsequent deployment. Steps (2): 1. Deploy once 2. Deploy again Before: Works correctly but leads to a update API call every time. After: No diff is detected by terraform. --- bundle/deploy/terraform/convert.go | 5 +++ bundle/deploy/terraform/tfdyn/convert_job.go | 33 ++++++++++++++++++- .../terraform/tfdyn/convert_job_test.go | 30 ++++++++++++++--- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 5a548e3b..b8993c03 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" @@ -82,6 +83,10 @@ func BundleToTerraform(config *config.Root) *schema.Root { conv(src, &dst) if src.JobSettings != nil { + sort.Slice(src.JobSettings.Tasks, func(i, j int) bool { + return src.JobSettings.Tasks[i].TaskKey < src.JobSettings.Tasks[j].TaskKey + }) + for _, v := range src.Tasks { var t schema.ResourceJobTask conv(v, &t) diff --git a/bundle/deploy/terraform/tfdyn/convert_job.go b/bundle/deploy/terraform/tfdyn/convert_job.go index d1e7e73e..8948e3ba 100644 --- a/bundle/deploy/terraform/tfdyn/convert_job.go +++ b/bundle/deploy/terraform/tfdyn/convert_job.go @@ -3,6 +3,7 @@ package tfdyn import ( "context" "fmt" + "sort" "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/libs/dyn" @@ -19,8 +20,38 @@ func convertJobResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary) } + // Sort the tasks of each job in the bundle by task key. Sorting + // the task keys ensures that the diff computed by terraform is correct and avoids + // recreates. For more details see the NOTE at + // https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage + // and https://github.com/databricks/terraform-provider-databricks/issues/4011 + // and https://github.com/databricks/cli/pull/1776 + vout := vin + var err error + tasks, ok := vin.Get("tasks").AsSequence() + if ok { + sort.Slice(tasks, func(i, j int) bool { + // We sort the tasks by their task key. Tasks without task keys are ordered + // before tasks with task keys. We do not error for those tasks + // since presence of a task_key is validated for in the Jobs backend. + tk1, ok := tasks[i].Get("task_key").AsString() + if !ok { + return true + } + tk2, ok := tasks[j].Get("task_key").AsString() + if !ok { + return false + } + return tk1 < tk2 + }) + vout, err = dyn.Set(vin, "tasks", dyn.V(tasks)) + if err != nil { + return dyn.InvalidValue, err + } + } + // Modify top-level keys. - vout, err := renameKeys(vin, map[string]string{ + vout, err = renameKeys(vout, map[string]string{ "tasks": "task", "job_clusters": "job_cluster", "parameters": "parameter", diff --git a/bundle/deploy/terraform/tfdyn/convert_job_test.go b/bundle/deploy/terraform/tfdyn/convert_job_test.go index b9e1f967..695b9ba2 100644 --- a/bundle/deploy/terraform/tfdyn/convert_job_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_job_test.go @@ -42,8 +42,8 @@ func TestConvertJob(t *testing.T) { }, Tasks: []jobs.Task{ { - TaskKey: "task_key", - JobClusterKey: "job_cluster_key", + TaskKey: "task_key_b", + JobClusterKey: "job_cluster_key_b", Libraries: []compute.Library{ { Pypi: &compute.PythonPyPiLibrary{ @@ -55,6 +55,17 @@ func TestConvertJob(t *testing.T) { }, }, }, + { + TaskKey: "task_key_a", + JobClusterKey: "job_cluster_key_a", + }, + { + TaskKey: "task_key_c", + JobClusterKey: "job_cluster_key_c", + }, + { + Description: "missing task key 😱", + }, }, }, Permissions: []resources.Permission{ @@ -100,8 +111,15 @@ func TestConvertJob(t *testing.T) { }, "task": []any{ map[string]any{ - "task_key": "task_key", - "job_cluster_key": "job_cluster_key", + "description": "missing task key 😱", + }, + map[string]any{ + "task_key": "task_key_a", + "job_cluster_key": "job_cluster_key_a", + }, + map[string]any{ + "task_key": "task_key_b", + "job_cluster_key": "job_cluster_key_b", "library": []any{ map[string]any{ "pypi": map[string]any{ @@ -113,6 +131,10 @@ func TestConvertJob(t *testing.T) { }, }, }, + map[string]any{ + "task_key": "task_key_c", + "job_cluster_key": "job_cluster_key_c", + }, }, }, out.Job["my_job"]) From a1dca56abfb16879d55e113d98735e10a97f558e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 11:30:39 +0200 Subject: [PATCH 57/78] Trim trailing whitespace (#1794) ## Changes Trailing whitespace is trimmed per the VS Code settings for this repository. ## Tests n/a --- bundle/config/mutator/python/python_mutator.go | 4 ++-- bundle/config/mutator/python/python_mutator_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index fbf3b7e0..3d4a502f 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -228,12 +228,12 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r return output, pythonDiagnostics } -const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, and that the wheel is installed in the Python environment: $ .venv/bin/pip install -e . -If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index bf12b249..7a419d79 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -570,12 +570,12 @@ func TestExplainProcessErr(t *testing.T) { Explanation: 'databricks-pydabs' library is not installed in the Python environment. -If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, and that the wheel is installed in the Python environment: $ .venv/bin/pip install -e . -If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: From 56cd96cb939207df3403546e998579a4cc768cf6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 11:32:54 +0200 Subject: [PATCH 58/78] Move trampoline code into trampoline package (#1793) ## Changes Doing this to make room for PyDABs under `bundle/python`. ## Tests n/a --- bundle/phases/deploy.go | 4 ++-- bundle/phases/initialize.go | 4 ++-- .../conditional_transform_test.go | 2 +- .../warning.go => trampoline/python_dbr_warning.go} | 2 +- .../python_dbr_warning_test.go} | 2 +- .../transform.go => trampoline/python_wheel.go} | 10 +++++----- .../python_wheel_test.go} | 2 +- bundle/{config/mutator => trampoline}/trampoline.go | 3 ++- .../{config/mutator => trampoline}/trampoline_test.go | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) rename bundle/{python => trampoline}/conditional_transform_test.go (99%) rename bundle/{python/warning.go => trampoline/python_dbr_warning.go} (99%) rename bundle/{python/warning_test.go => trampoline/python_dbr_warning_test.go} (99%) rename bundle/{python/transform.go => trampoline/python_wheel.go} (94%) rename bundle/{python/transform_test.go => trampoline/python_wheel_test.go} (99%) rename bundle/{config/mutator => trampoline}/trampoline.go (99%) rename bundle/{config/mutator => trampoline}/trampoline_test.go (99%) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 097c561e..cb0ecf75 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -15,8 +15,8 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/sync" terraformlib "github.com/databricks/cli/libs/terraform" @@ -157,7 +157,7 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { artifacts.CleanUp(), libraries.ExpandGlobReferences(), libraries.Upload(), - python.TransformWheelTask(), + trampoline.TransformWheelTask(), files.Upload(outputHandler), deploy.StateUpdate(), deploy.StatePush(), diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8039a4f1..93ce61b2 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -9,8 +9,8 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/trampoline" ) // The initialize phase fills in defaults and connects to the workspace. @@ -66,7 +66,7 @@ func Initialize() bundle.Mutator { mutator.ConfigureWSFS(), mutator.TranslatePaths(), - python.WrapperWarning(), + trampoline.WrapperWarning(), permissions.ApplyBundlePermissions(), permissions.FilterCurrentUser(), metadata.AnnotateJobs(), diff --git a/bundle/python/conditional_transform_test.go b/bundle/trampoline/conditional_transform_test.go similarity index 99% rename from bundle/python/conditional_transform_test.go rename to bundle/trampoline/conditional_transform_test.go index 1d397f7a..26e67154 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/trampoline/conditional_transform_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/warning.go b/bundle/trampoline/python_dbr_warning.go similarity index 99% rename from bundle/python/warning.go rename to bundle/trampoline/python_dbr_warning.go index 0e9d8bef..f62e9eab 100644 --- a/bundle/python/warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/warning_test.go b/bundle/trampoline/python_dbr_warning_test.go similarity index 99% rename from bundle/python/warning_test.go rename to bundle/trampoline/python_dbr_warning_test.go index a5ab7563..d293c947 100644 --- a/bundle/python/warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/transform.go b/bundle/trampoline/python_wheel.go similarity index 94% rename from bundle/python/transform.go rename to bundle/trampoline/python_wheel.go index 9d3b1ab6..8e309a62 100644 --- a/bundle/python/transform.go +++ b/bundle/trampoline/python_wheel.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" @@ -69,7 +69,7 @@ func TransformWheelTask() bundle.Mutator { res := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper return res, nil }, - mutator.NewTrampoline( + NewTrampoline( "python_wheel", &pythonTrampoline{}, NOTEBOOK_TEMPLATE, @@ -94,9 +94,9 @@ func (t *pythonTrampoline) CleanUp(task *jobs.Task) error { return nil } -func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { +func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []TaskWithJobKey { r := b.Config.Resources - result := make([]mutator.TaskWithJobKey, 0) + result := make([]TaskWithJobKey, 0) for k := range b.Config.Resources.Jobs { tasks := r.Jobs[k].JobSettings.Tasks for i := range tasks { @@ -110,7 +110,7 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { continue } - result = append(result, mutator.TaskWithJobKey{ + result = append(result, TaskWithJobKey{ JobKey: k, Task: task, }) diff --git a/bundle/python/transform_test.go b/bundle/trampoline/python_wheel_test.go similarity index 99% rename from bundle/python/transform_test.go rename to bundle/trampoline/python_wheel_test.go index c7bddca1..40c3b38f 100644 --- a/bundle/python/transform_test.go +++ b/bundle/trampoline/python_wheel_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/config/mutator/trampoline.go b/bundle/trampoline/trampoline.go similarity index 99% rename from bundle/config/mutator/trampoline.go rename to bundle/trampoline/trampoline.go index dcca5014..1dc1c446 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/trampoline/trampoline.go @@ -1,4 +1,4 @@ -package mutator +package trampoline import ( "context" @@ -23,6 +23,7 @@ type TrampolineFunctions interface { GetTasks(b *bundle.Bundle) []TaskWithJobKey CleanUp(task *jobs.Task) error } + type trampoline struct { name string functions TrampolineFunctions diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/trampoline/trampoline_test.go similarity index 99% rename from bundle/config/mutator/trampoline_test.go rename to bundle/trampoline/trampoline_test.go index 08d3c822..08a290f9 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/trampoline/trampoline_test.go @@ -1,4 +1,4 @@ -package mutator +package trampoline import ( "context" From 1d1aa0a4166903b20b0f2e6321134a1aff0c4204 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 12:03:05 +0200 Subject: [PATCH 59/78] Rename `RootPath` -> `BundleRootPath` (#1792) ## Changes After introducing the `SyncRootPath` field on the bundle (#1694), the previous `RootPath` became ambiguous. Does it mean the bundle root path or the sync root path? This PR renames to field to `BundleRootPath` to remove the ambiguity. ## Tests n/a --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- bundle/artifacts/expand_globs_test.go | 6 ++-- bundle/artifacts/prepare.go | 2 +- bundle/artifacts/whl/autodetect.go | 6 ++-- bundle/bundle.go | 30 +++++++++++-------- bundle/bundle_read_only.go | 2 +- bundle/bundle_test.go | 4 +-- bundle/config/loader/entry_point.go | 2 +- bundle/config/loader/entry_point_test.go | 2 +- bundle/config/loader/process_include_test.go | 4 +-- bundle/config/loader/process_root_includes.go | 6 ++-- .../loader/process_root_includes_test.go | 24 +++++++-------- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/load_git_details.go | 2 +- .../config/mutator/python/python_mutator.go | 2 +- bundle/config/mutator/rewrite_sync_paths.go | 6 ++-- .../config/mutator/rewrite_sync_paths_test.go | 8 ++--- .../config/mutator/sync_default_path_test.go | 8 ++--- bundle/config/mutator/sync_infer_root.go | 2 +- bundle/config/mutator/sync_infer_root_test.go | 12 ++++---- bundle/deploy/metadata/compute.go | 2 +- bundle/deploy/state_pull_test.go | 8 ++--- bundle/deploy/state_push_test.go | 2 +- bundle/deploy/state_update_test.go | 2 +- bundle/deploy/terraform/init_test.go | 22 +++++++------- bundle/deploy/terraform/load_test.go | 2 +- bundle/deploy/terraform/state_pull_test.go | 2 +- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/deploy/terraform/util_test.go | 4 +-- bundle/render/render_text_output.go | 2 +- bundle/scripts/scripts.go | 2 +- bundle/scripts/scripts_test.go | 2 +- .../trampoline/conditional_transform_test.go | 8 ++--- bundle/trampoline/python_wheel_test.go | 2 +- bundle/trampoline/trampoline_test.go | 4 +-- cmd/bundle/generate/generate_test.go | 4 +-- cmd/sync/sync_test.go | 8 ++--- internal/bundle/artifacts_test.go | 12 ++++---- 37 files changed, 112 insertions(+), 108 deletions(-) diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go index 1665a480..dc7c77de 100644 --- a/bundle/artifacts/expand_globs_test.go +++ b/bundle/artifacts/expand_globs_test.go @@ -24,7 +24,7 @@ func TestExpandGlobs_Nominal(t *testing.T) { testutil.Touch(t, tmpDir, "bc.txt") b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { @@ -63,7 +63,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { @@ -111,7 +111,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) { testutil.Touch(t, tmpDir, "b2.txt") b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index fb61ed9e..91e0bd09 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -47,7 +47,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics // If artifact path is not provided, use bundle root dir if artifact.Path == "" { - artifact.Path = b.RootPath + artifact.Path = b.BundleRootPath } if !filepath.IsAbs(artifact.Path) { diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index 1601767f..88dc742c 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic log.Infof(ctx, "Detecting Python wheel project...") // checking if there is setup.py in the bundle root - setupPy := filepath.Join(b.RootPath, "setup.py") + setupPy := filepath.Join(b.BundleRootPath, "setup.py") _, err := os.Stat(setupPy) if err != nil { log.Infof(ctx, "No Python wheel project found at bundle root folder") return nil } - log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath)) + log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath)) module := extractModuleName(setupPy) if b.Config.Artifacts == nil { b.Config.Artifacts = make(map[string]*config.Artifact) } - pkgPath, err := filepath.Abs(b.RootPath) + pkgPath, err := filepath.Abs(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/bundle.go b/bundle/bundle.go index 8b5ff976..85625568 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -31,22 +31,26 @@ import ( const internalFolder = ".internal" type Bundle struct { - // RootPath contains the directory path to the root of the bundle. + // BundleRootPath is the local path to the root directory of the bundle. // It is set when we instantiate a new bundle instance. - RootPath string + BundleRootPath string - // BundleRoot is a virtual filesystem path to the root of the bundle. + // BundleRoot is a virtual filesystem path to [BundleRootPath]. // Exclusively use this field for filesystem operations. BundleRoot vfs.Path - // SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace. - // It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot]. - SyncRoot vfs.Path - // SyncRootPath is the local path to the root directory of files that are synchronized to the workspace. - // It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access. + // By default, it is the same as [BundleRootPath]. + // If it is different, it must be an ancestor to [BundleRootPath]. + // That is, [SyncRootPath] must contain [BundleRootPath]. SyncRootPath string + // SyncRoot is a virtual filesystem path to [SyncRootPath]. + // Exclusively use this field for filesystem operations. + SyncRoot vfs.Path + + // Config contains the bundle configuration. + // It is loaded from the bundle configuration files and mutators may update it. Config config.Root // Metadata about the bundle deployment. This is the interface Databricks services @@ -84,14 +88,14 @@ type Bundle struct { func Load(ctx context.Context, path string) (*Bundle, error) { b := &Bundle{ - RootPath: filepath.Clean(path), - BundleRoot: vfs.MustNew(path), + BundleRootPath: filepath.Clean(path), + BundleRoot: vfs.MustNew(path), } configFile, err := config.FileNames.FindInPath(path) if err != nil { return nil, err } - log.Debugf(ctx, "Found bundle root at %s (file %s)", b.RootPath, configFile) + log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile) return b, nil } @@ -160,7 +164,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) if !exists || cacheDirName == "" { cacheDirName = filepath.Join( // Anchor at bundle root directory. - b.RootPath, + b.BundleRootPath, // Static cache directory. ".databricks", "bundle", @@ -212,7 +216,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - internalDirRel, err := filepath.Rel(b.RootPath, internalDir) + internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir) if err != nil { return nil, err } diff --git a/bundle/bundle_read_only.go b/bundle/bundle_read_only.go index 74b9d94d..ceab95c0 100644 --- a/bundle/bundle_read_only.go +++ b/bundle/bundle_read_only.go @@ -21,7 +21,7 @@ func (r ReadOnlyBundle) Config() config.Root { } func (r ReadOnlyBundle) RootPath() string { - return r.b.RootPath + return r.b.BundleRootPath } func (r ReadOnlyBundle) BundleRoot() vfs.Path { diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index a29aa024..1c310235 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -79,7 +79,7 @@ func TestBundleMustLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := MustLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath)) } func TestBundleMustLoadFailureWithEnv(t *testing.T) { @@ -98,7 +98,7 @@ func TestBundleTryLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := TryLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath)) } func TestBundleTryLoadFailureWithEnv(t *testing.T) { diff --git a/bundle/config/loader/entry_point.go b/bundle/config/loader/entry_point.go index 2c73a582..d476cb22 100644 --- a/bundle/config/loader/entry_point.go +++ b/bundle/config/loader/entry_point.go @@ -20,7 +20,7 @@ func (m *entryPoint) Name() string { } func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - path, err := config.FileNames.FindInPath(b.RootPath) + path, err := config.FileNames.FindInPath(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index 80271f0b..406b9b67 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) { func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ - RootPath: "testdata", + BundleRootPath: "testdata", } diags := bundle.Apply(context.Background(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index da4da9ff..2ccd84b3 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -14,7 +14,7 @@ import ( func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - RootPath: "testdata", + BundleRootPath: "testdata", Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -22,7 +22,7 @@ func TestProcessInclude(t *testing.T) { }, } - m := loader.ProcessInclude(filepath.Join(b.RootPath, "host.yml"), "host.yml") + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, "host.yml"), "host.yml") assert.Equal(t, "ProcessInclude(host.yml)", m.Name()) // Assert the host value prior to applying the mutator diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 25f284fd..c14fb7ce 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -47,7 +47,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } // Anchor includes to the bundle root path. - matches, err := filepath.Glob(filepath.Join(b.RootPath, entry)) + matches, err := filepath.Glob(filepath.Join(b.BundleRootPath, entry)) if err != nil { return diag.FromErr(err) } @@ -61,7 +61,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Filter matches to ones we haven't seen yet. var includes []string for _, match := range matches { - rel, err := filepath.Rel(b.RootPath, match) + rel, err := filepath.Rel(b.BundleRootPath, match) if err != nil { return diag.FromErr(err) } @@ -76,7 +76,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.RootPath, include), include)) + out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include)) } } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index 737dbbef..27ff9b05 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -15,7 +15,7 @@ import ( func TestProcessRootIncludesEmpty(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", } diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -30,7 +30,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { } b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Include: []string{ "/tmp/*.yml", @@ -44,7 +44,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { func TestProcessRootIncludesSingleGlob(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -52,9 +52,9 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "databricks.yml") - testutil.Touch(t, b.RootPath, "a.yml") - testutil.Touch(t, b.RootPath, "b.yml") + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") + testutil.Touch(t, b.BundleRootPath, "b.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -63,7 +63,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { func TestProcessRootIncludesMultiGlob(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "a*.yml", @@ -72,8 +72,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "a1.yml") - testutil.Touch(t, b.RootPath, "b1.yml") + testutil.Touch(t, b.BundleRootPath, "a1.yml") + testutil.Touch(t, b.BundleRootPath, "b1.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -82,7 +82,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { func TestProcessRootIncludesRemoveDups(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -91,7 +91,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "a.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -100,7 +100,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "notexist.yml", diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index 07dd2021..9f70b74a 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -42,7 +42,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "skip/test7.py")) b := &bundle.Bundle{ - RootPath: dir, + BundleRootPath: dir, Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 9b1c963c..00e7f54d 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } // Compute relative path of the bundle root from the Git repo root. - absBundlePath, err := filepath.Abs(b.RootPath) + absBundlePath, err := filepath.Abs(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 3d4a502f..da6c4d21 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -108,7 +108,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err) } - rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.RootPath, pythonPath, leftRoot) + rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.BundleRootPath, pythonPath, leftRoot) mutateDiags = diags if diags.HasError() { return dyn.InvalidValue, mutateDiagsHasError diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 888714ab..f9a02369 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -45,15 +45,15 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } - v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } - v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index a66f2763..0e4dfc4c 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -15,7 +15,7 @@ import ( func TestRewriteSyncPathsRelative(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -54,7 +54,7 @@ func TestRewriteSyncPathsRelative(t *testing.T) { func TestRewriteSyncPathsAbsolute(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/dir", + BundleRootPath: "/tmp/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -94,7 +94,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("no sync block", func(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", } diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) @@ -103,7 +103,7 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("empty include/exclude blocks", func(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Sync: config.Sync{ Include: []string{}, diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go index a37e913d..c82a687b 100644 --- a/bundle/config/mutator/sync_default_path_test.go +++ b/bundle/config/mutator/sync_default_path_test.go @@ -15,8 +15,8 @@ import ( func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", - Config: config.Root{}, + BundleRootPath: "/tmp/some/dir", + Config: config.Root{}, } ctx := context.Background() @@ -51,8 +51,8 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { for _, tcase := range tcases { t.Run(tcase.name, func(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", - Config: config.Root{}, + BundleRootPath: "/tmp/some/dir", + Config: config.Root{}, } diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 012acf80..512adcdb 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -57,7 +57,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno var diags diag.Diagnostics // Use the bundle root path as the starting point for inferring the sync root path. - bundleRootPath := filepath.Clean(b.RootPath) + bundleRootPath := filepath.Clean(b.BundleRootPath) // Infer the sync root path by looking at each one of the sync paths. // Every sync path must be a descendant of the final sync root path. diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go index 85e40adc..f507cbc7 100644 --- a/bundle/config/mutator/sync_infer_root_test.go +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -16,7 +16,7 @@ import ( func TestSyncInferRoot_NominalAbsolute(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -47,7 +47,7 @@ func TestSyncInferRoot_NominalAbsolute(t *testing.T) { func TestSyncInferRoot_NominalRelative(t *testing.T) { b := &bundle.Bundle{ - RootPath: "./some/dir", + BundleRootPath: "./some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -78,7 +78,7 @@ func TestSyncInferRoot_NominalRelative(t *testing.T) { func TestSyncInferRoot_ParentDirectory(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -109,7 +109,7 @@ func TestSyncInferRoot_ParentDirectory(t *testing.T) { func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir/that/is/very/deeply/nested", + BundleRootPath: "/tmp/some/dir/that/is/very/deeply/nested", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -146,7 +146,7 @@ func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { func TestSyncInferRoot_MultiplePaths(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/bundle/root", + BundleRootPath: "/tmp/some/bundle/root", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -173,7 +173,7 @@ func TestSyncInferRoot_MultiplePaths(t *testing.T) { func TestSyncInferRoot_Error(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 6ab997e2..bc8767de 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -40,7 +40,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { // Compute config file path the job is defined in, relative to the bundle // root l := b.Config.GetLocation("resources.jobs." + name) - relativePath, err := filepath.Rel(b.RootPath, l.File) + relativePath, err := filepath.Rel(b.BundleRootPath, l.File) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index f7519306..42701eb2 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -62,8 +62,8 @@ func testStatePull(t *testing.T, opts statePullOpts) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, - BundleRoot: vfs.MustNew(tmpDir), + BundleRootPath: tmpDir, + BundleRoot: vfs.MustNew(tmpDir), SyncRootPath: tmpDir, SyncRoot: vfs.MustNew(tmpDir), @@ -259,7 +259,7 @@ func TestStatePullNoState(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -447,7 +447,7 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_push_test.go b/bundle/deploy/state_push_test.go index 39e4d13a..038b7534 100644 --- a/bundle/deploy/state_push_test.go +++ b/bundle/deploy/state_push_test.go @@ -45,7 +45,7 @@ func TestStatePush(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index 72096d14..1f5010b5 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -27,7 +27,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle { require.NoError(t, err) return &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index 450e7eb6..e3621c6c 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -33,7 +33,7 @@ func TestInitEnvironmentVariables(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -60,7 +60,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -88,7 +88,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -114,7 +114,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -144,7 +144,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -174,7 +174,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { func TestSetProxyEnvVars(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -250,7 +250,7 @@ func TestSetProxyEnvVars(t *testing.T) { func TestSetUserAgentExtraEnvVar(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Experimental: &config.Experimental{ PyDABs: config.PyDABs{ @@ -333,7 +333,7 @@ func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) { ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -357,7 +357,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T) ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -380,7 +380,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) { ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/load_test.go b/bundle/deploy/terraform/load_test.go index c6221718..b7243ca1 100644 --- a/bundle/deploy/terraform/load_test.go +++ b/bundle/deploy/terraform/load_test.go @@ -17,7 +17,7 @@ func TestLoadWithNoState(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index 39937a3c..c4798c57 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -32,7 +32,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) fi func statePullTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index ac74f345..e022dee1 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -29,7 +29,7 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer { func statePushTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go index 251a7c25..74b32925 100644 --- a/bundle/deploy/terraform/util_test.go +++ b/bundle/deploy/terraform/util_test.go @@ -13,7 +13,7 @@ import ( func TestParseResourcesStateWithNoFile(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -31,7 +31,7 @@ func TestParseResourcesStateWithNoFile(t *testing.T) { func TestParseResourcesStateWithExistingStateFile(t *testing.T) { ctx := context.Background() b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index ea0b9a94..e1fad98a 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -148,7 +148,7 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) // Make location relative to bundle root if d.Locations[i].File != "" { - out, err := filepath.Rel(b.RootPath, d.Locations[i].File) + out, err := filepath.Rel(b.BundleRootPath, d.Locations[i].File) // if we can't relativize the path, just use path as-is if err == nil { d.Locations[i].File = out diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 629b3a8a..f9e1541e 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -30,7 +30,7 @@ func (m *script) Name() string { } func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - executor, err := exec.NewCommandExecutor(b.RootPath) + executor, err := exec.NewCommandExecutor(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go index 1bc216b6..0c92bc2c 100644 --- a/bundle/scripts/scripts_test.go +++ b/bundle/scripts/scripts_test.go @@ -23,7 +23,7 @@ func TestExecutesHook(t *testing.T) { }, } - executor, err := exec.NewCommandExecutor(b.RootPath) + executor, err := exec.NewCommandExecutor(b.BundleRootPath) require.NoError(t, err) _, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild) require.NoError(t, err) diff --git a/bundle/trampoline/conditional_transform_test.go b/bundle/trampoline/conditional_transform_test.go index 26e67154..57aa9aac 100644 --- a/bundle/trampoline/conditional_transform_test.go +++ b/bundle/trampoline/conditional_transform_test.go @@ -17,8 +17,8 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", @@ -66,8 +66,8 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/trampoline/python_wheel_test.go b/bundle/trampoline/python_wheel_test.go index 40c3b38f..517be35e 100644 --- a/bundle/trampoline/python_wheel_test.go +++ b/bundle/trampoline/python_wheel_test.go @@ -115,7 +115,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/trampoline/trampoline_test.go b/bundle/trampoline/trampoline_test.go index 08a290f9..4682d8fa 100644 --- a/bundle/trampoline/trampoline_test.go +++ b/bundle/trampoline/trampoline_test.go @@ -56,8 +56,8 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/Workspace/files", diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index 7de6805f..943f721c 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -24,7 +24,7 @@ func TestGeneratePipelineCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - RootPath: root, + BundleRootPath: root, } m := mocks.NewMockWorkspaceClient(t) @@ -122,7 +122,7 @@ func TestGenerateJobCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - RootPath: root, + BundleRootPath: root, } m := mocks.NewMockWorkspaceClient(t) diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index bd03eec9..8f65aedb 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -17,10 +17,10 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tempDir, - BundleRoot: vfs.MustNew(tempDir), - SyncRootPath: tempDir, - SyncRoot: vfs.MustNew(tempDir), + BundleRootPath: tempDir, + BundleRoot: vfs.MustNew(tempDir), + SyncRootPath: tempDir, + SyncRoot: vfs.MustNew(tempDir), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index fa052e22..775327e1 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -36,8 +36,8 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -101,8 +101,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -171,8 +171,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { touchEmptyFile(t, whlPath) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", From da3b4f7c72ab6c48168ca968637846f61e7899c8 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Sun, 29 Sep 2024 16:08:10 +0200 Subject: [PATCH 60/78] Fix panic in `apply_presets.go` (#1796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This fixes the user-reported panic in `apply_presets.go`. I'm still unsure how to reproduce this, since the CLI just reports `ob broken_job is not defined` when I try to use `bundle deploy` with an empty job. That said — we may as well be defensive here and I see we have lots of checks for empty job/cluster/etc. settings scattered throughout our code base so at least we're somewhat consistent. --- bundle/config/mutator/apply_presets.go | 79 ++++++++++---- bundle/config/mutator/apply_presets_test.go | 113 ++++++++++++++++++++ 2 files changed, 171 insertions(+), 21 deletions(-) diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 27af82e5..1fd49206 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -35,8 +35,10 @@ func (m *applyPresets) Name() string { } func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + if d := validatePauseStatus(b); d != nil { - return d + diags = diags.Extend(d) } r := b.Config.Resources @@ -45,7 +47,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos tags := toTagArray(t.Tags) // Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus - for _, j := range r.Jobs { + for key, j := range r.Jobs { + if j.JobSettings == nil { + diags = diags.Extend(diag.Errorf("job %s is not defined", key)) + continue + } j.Name = prefix + j.Name if j.Tags == nil { j.Tags = make(map[string]string) @@ -77,20 +83,27 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Pipelines presets: Prefix, PipelinesDevelopment - for i := range r.Pipelines { - r.Pipelines[i].Name = prefix + r.Pipelines[i].Name + for key, p := range r.Pipelines { + if p.PipelineSpec == nil { + diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) + continue + } + p.Name = prefix + p.Name if config.IsExplicitlyEnabled(t.PipelinesDevelopment) { - r.Pipelines[i].Development = true + p.Development = true } if t.TriggerPauseStatus == config.Paused { - r.Pipelines[i].Continuous = false + p.Continuous = false } - // As of 2024-06, pipelines don't yet support tags } // Models presets: Prefix, Tags - for _, m := range r.Models { + for key, m := range r.Models { + if m.Model == nil { + diags = diags.Extend(diag.Errorf("model %s is not defined", key)) + continue + } m.Name = prefix + m.Name for _, t := range tags { exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool { @@ -104,7 +117,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Experiments presets: Prefix, Tags - for _, e := range r.Experiments { + for key, e := range r.Experiments { + if e.Experiment == nil { + diags = diags.Extend(diag.Errorf("experiment %s is not defined", key)) + continue + } filepath := e.Name dir := path.Dir(filepath) base := path.Base(filepath) @@ -128,40 +145,60 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Model serving endpoint presets: Prefix - for i := range r.ModelServingEndpoints { - r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name + for key, e := range r.ModelServingEndpoints { + if e.CreateServingEndpoint == nil { + diags = diags.Extend(diag.Errorf("model serving endpoint %s is not defined", key)) + continue + } + e.Name = normalizePrefix(prefix) + e.Name // As of 2024-06, model serving endpoints don't yet support tags } // Registered models presets: Prefix - for i := range r.RegisteredModels { - r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name + for key, m := range r.RegisteredModels { + if m.CreateRegisteredModelRequest == nil { + diags = diags.Extend(diag.Errorf("registered model %s is not defined", key)) + continue + } + m.Name = normalizePrefix(prefix) + m.Name // As of 2024-06, registered models don't yet support tags } - // Quality monitors presets: Prefix + // Quality monitors presets: Schedule if t.TriggerPauseStatus == config.Paused { - for i := range r.QualityMonitors { + for key, q := range r.QualityMonitors { + if q.CreateMonitor == nil { + diags = diags.Extend(diag.Errorf("quality monitor %s is not defined", key)) + continue + } // Remove all schedules from monitors, since they don't support pausing/unpausing. // Quality monitors might support the "pause" property in the future, so at the // CLI level we do respect that property if it is set to "unpaused." - if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { - r.QualityMonitors[i].Schedule = nil + if q.Schedule != nil && q.Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + q.Schedule = nil } } } // Schemas: Prefix - for i := range r.Schemas { - r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name + for key, s := range r.Schemas { + if s.CreateSchema == nil { + diags = diags.Extend(diag.Errorf("schema %s is not defined", key)) + continue + } + s.Name = normalizePrefix(prefix) + s.Name // HTTP API for schemas doesn't yet support tags. It's only supported in // the Databricks UI and via the SQL API. } // Clusters: Prefix, Tags - for _, c := range r.Clusters { + for key, c := range r.Clusters { + if c.ClusterSpec == nil { + diags = diags.Extend(diag.Errorf("cluster %s is not defined", key)) + continue + } c.ClusterName = prefix + c.ClusterName if c.CustomTags == nil { c.CustomTags = make(map[string]string) @@ -175,7 +212,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } - return nil + return diags } func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index ab2478ae..24295da4 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -251,3 +251,116 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) { }) } } + +func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + Presets: config.Presets{ + NamePrefix: "prefix-", + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), "job job1 is not defined") +} + +func TestApplyPresetsResourceNotDefined(t *testing.T) { + tests := []struct { + resources config.Resources + error string + }{ + { + resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + error: "job job1 is not defined", + }, + { + resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline1": {}, // no pipelinespec inside + }, + }, + error: "pipeline pipeline1 is not defined", + }, + { + resources: config.Resources{ + Models: map[string]*resources.MlflowModel{ + "model1": {}, // no model inside + }, + }, + error: "model model1 is not defined", + }, + { + resources: config.Resources{ + Experiments: map[string]*resources.MlflowExperiment{ + "experiment1": {}, // no experiment inside + }, + }, + error: "experiment experiment1 is not defined", + }, + { + resources: config.Resources{ + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint1": {}, // no CreateServingEndpoint inside + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "model1": {}, // no CreateRegisteredModelRequest inside + }, + }, + error: "model serving endpoint endpoint1 is not defined", + }, + { + resources: config.Resources{ + QualityMonitors: map[string]*resources.QualityMonitor{ + "monitor1": {}, // no CreateMonitor inside + }, + }, + error: "quality monitor monitor1 is not defined", + }, + { + resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "schema1": {}, // no CreateSchema inside + }, + }, + error: "schema schema1 is not defined", + }, + { + resources: config.Resources{ + Clusters: map[string]*resources.Cluster{ + "cluster1": {}, // no ClusterSpec inside + }, + }, + error: "cluster cluster1 is not defined", + }, + } + + for _, tt := range tests { + t.Run(tt.error, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: tt.resources, + Presets: config.Presets{ + TriggerPauseStatus: config.Paused, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), tt.error) + }) + } +} From 84fc1ed13162fafd54b6cebc3df991202acc431e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 1 Oct 2024 12:44:47 +0200 Subject: [PATCH 61/78] Upgrade to Go SDK 0.47.0 (#1799) ## Changes Upgrade to Go SDK 0.47.0 ## Tests --- .codegen/_openapi_sha | 2 +- .gitattributes | 3 + bundle/schema/jsonschema.json | 152 +++++++++++- .../disable-legacy-features.go | 215 +++++++++++++++++ cmd/account/settings/settings.go | 2 + cmd/workspace/apps/apps.go | 83 ++++--- cmd/workspace/clusters/clusters.go | 10 +- cmd/workspace/cmd.go | 2 + .../disable-legacy-access.go | 217 ++++++++++++++++++ cmd/workspace/pipelines/pipelines.go | 1 + .../serving-endpoints/serving-endpoints.go | 80 ++++++- cmd/workspace/settings/settings.go | 2 + cmd/workspace/tables/tables.go | 3 + .../temporary-table-credentials.go | 122 ++++++++++ go.mod | 2 +- go.sum | 4 +- 16 files changed, 851 insertions(+), 49 deletions(-) create mode 100755 cmd/account/disable-legacy-features/disable-legacy-features.go create mode 100755 cmd/workspace/disable-legacy-access/disable-legacy-access.go create mode 100755 cmd/workspace/temporary-table-credentials/temporary-table-credentials.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 4ceeab3d..ffd6f58d 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -d05898328669a3f8ab0c2ecee37db2673d3ea3f7 \ No newline at end of file +6f6b1371e640f2dfeba72d365ac566368656f6b6 \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index f35c4f81..2470eb33 100755 --- a/.gitattributes +++ b/.gitattributes @@ -6,6 +6,7 @@ cmd/account/cmd.go linguist-generated=true cmd/account/credentials/credentials.go linguist-generated=true cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true +cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true cmd/account/encryption-keys/encryption-keys.go linguist-generated=true cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true cmd/account/groups/groups.go linguist-generated=true @@ -52,6 +53,7 @@ cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true cmd/workspace/dashboards/dashboards.go linguist-generated=true cmd/workspace/data-sources/data-sources.go linguist-generated=true cmd/workspace/default-namespace/default-namespace.go linguist-generated=true +cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true @@ -108,6 +110,7 @@ cmd/workspace/storage-credentials/storage-credentials.go linguist-generated=true cmd/workspace/system-schemas/system-schemas.go linguist-generated=true cmd/workspace/table-constraints/table-constraints.go linguist-generated=true cmd/workspace/tables/tables.go linguist-generated=true +cmd/workspace/temporary-table-credentials/temporary-table-credentials.go linguist-generated=true cmd/workspace/token-management/token-management.go linguist-generated=true cmd/workspace/tokens/tokens.go linguist-generated=true cmd/workspace/users/users.go linguist-generated=true diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2db1a5ab..afdf9fb9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -59,6 +59,127 @@ "cli": { "bundle": { "config": { + "resources.Cluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Grant": { "anyOf": [ { @@ -109,7 +230,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" }, "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", + "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, "format": { @@ -293,7 +414,7 @@ "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", + "description": "Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" }, "route_optimized": { @@ -747,6 +868,9 @@ { "type": "object", "properties": { + "cluster_id": { + "$ref": "#/$defs/string" + }, "compute_id": { "$ref": "#/$defs/string" }, @@ -923,6 +1047,9 @@ { "type": "object", "properties": { + "clusters": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster" + }, "experiments": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" }, @@ -990,6 +1117,9 @@ "bundle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" }, + "cluster_id": { + "$ref": "#/$defs/string" + }, "compute_id": { "$ref": "#/$defs/string" }, @@ -2028,7 +2158,7 @@ }, "compute.RuntimeEngine": { "type": "string", - "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", + "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n", "enum": [ "NULL", "STANDARD", @@ -2610,7 +2740,7 @@ "anyOf": [ { "type": "object", - "description": "Write-only setting, available only in Create/Update/Reset and Submit calls. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "description": "Write-only setting. Specifies the user, service principal or group that the job/pipeline runs as. If not specified, the job/pipeline runs as the user who created the job/pipeline.\n\nExactly one of `user_name`, `service_principal_name`, `group_name` should be specified. If not, an error is thrown.", "properties": { "service_principal_name": { "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", @@ -4904,6 +5034,20 @@ "cli": { "bundle": { "config": { + "resources.Cluster": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Cluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "anyOf": [ { diff --git a/cmd/account/disable-legacy-features/disable-legacy-features.go b/cmd/account/disable-legacy-features/disable-legacy-features.go new file mode 100755 index 00000000..6d25b943 --- /dev/null +++ b/cmd/account/disable-legacy-features/disable-legacy-features.go @@ -0,0 +1,215 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package disable_legacy_features + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "disable-legacy-features", + Short: `Disable legacy features for new Databricks workspaces.`, + Long: `Disable legacy features for new Databricks workspaces. + + For newly created workspaces: 1. Disables the use of DBFS root and mounts. 2. + Hive Metastore will not be provisioned. 3. Disables the use of ‘No-isolation + clusters’. 4. Disables Databricks Runtime versions prior to 13.3LTS.`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDisableLegacyFeaturesRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDisableLegacyFeaturesRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the disable legacy features setting.` + cmd.Long = `Delete the disable legacy features setting. + + Deletes the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.DisableLegacyFeatures().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDisableLegacyFeaturesRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDisableLegacyFeaturesRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the disable legacy features setting.` + cmd.Long = `Get the disable legacy features setting. + + Gets the value of the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.DisableLegacyFeatures().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDisableLegacyFeaturesRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDisableLegacyFeaturesRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the disable legacy features setting.` + cmd.Long = `Update the disable legacy features setting. + + Updates the value of the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := a.Settings.DisableLegacyFeatures().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DisableLegacyFeatures diff --git a/cmd/account/settings/settings.go b/cmd/account/settings/settings.go index a750e81e..9a9cd44b 100755 --- a/cmd/account/settings/settings.go +++ b/cmd/account/settings/settings.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" csp_enablement_account "github.com/databricks/cli/cmd/account/csp-enablement-account" + disable_legacy_features "github.com/databricks/cli/cmd/account/disable-legacy-features" esm_enablement_account "github.com/databricks/cli/cmd/account/esm-enablement-account" personal_compute "github.com/databricks/cli/cmd/account/personal-compute" ) @@ -27,6 +28,7 @@ func New() *cobra.Command { // Add subservices cmd.AddCommand(csp_enablement_account.New()) + cmd.AddCommand(disable_legacy_features.New()) cmd.AddCommand(esm_enablement_account.New()) cmd.AddCommand(personal_compute.New()) diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index bc3fbe92..baec6d03 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -75,8 +75,8 @@ func newCreate() *cobra.Command { var createSkipWait bool var createTimeout time.Duration - cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach IDLE state`) - cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach IDLE state`) + cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach ACTIVE state`) + cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach ACTIVE state`) // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) @@ -130,13 +130,13 @@ func newCreate() *cobra.Command { } spinner := cmdio.Spinner(ctx) info, err := wait.OnProgress(func(i *apps.App) { - if i.Status == nil { + if i.ComputeStatus == nil { return } - status := i.Status.State + status := i.ComputeStatus.State statusMessage := fmt.Sprintf("current status: %s", status) - if i.Status != nil { - statusMessage = i.Status.Message + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message } spinner <- statusMessage }).GetWithTimeout(createTimeout) @@ -198,11 +198,11 @@ func newDelete() *cobra.Command { deleteReq.Name = args[0] - err = w.Apps.Delete(ctx, deleteReq) + response, err := w.Apps.Delete(ctx, deleteReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. @@ -240,35 +240,23 @@ func newDeploy() *cobra.Command { // TODO: short flags cmd.Flags().Var(&deployJson, "json", `either inline JSON string or @path/to/file.json with request body`) + cmd.Flags().StringVar(&deployReq.DeploymentId, "deployment-id", deployReq.DeploymentId, `The unique id of the deployment.`) cmd.Flags().Var(&deployReq.Mode, "mode", `The mode of which the deployment will manage the source code. Supported values: [AUTO_SYNC, SNAPSHOT]`) + cmd.Flags().StringVar(&deployReq.SourceCodePath, "source-code-path", deployReq.SourceCodePath, `The workspace file system path of the source code used to create the app deployment.`) - cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH" + cmd.Use = "deploy APP_NAME" cmd.Short = `Create an app deployment.` cmd.Long = `Create an app deployment. Creates an app deployment for the app with the supplied name. Arguments: - APP_NAME: The name of the app. - SOURCE_CODE_PATH: The workspace file system path of the source code used to create the app - deployment. This is different from - deployment_artifacts.source_code_path, which is the path used by the - deployed app. The former refers to the original source code location of - the app in the workspace during deployment creation, whereas the latter - provides a system generated stable snapshotted source code path used by - the deployment.` + APP_NAME: The name of the app.` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(1)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input") - } - return nil - } - check := root.ExactArgs(2) + check := root.ExactArgs(1) return check(cmd, args) } @@ -284,9 +272,6 @@ func newDeploy() *cobra.Command { } } deployReq.AppName = args[0] - if !cmd.Flags().Changed("json") { - deployReq.SourceCodePath = args[1] - } wait, err := w.Apps.Deploy(ctx, deployReq) if err != nil { @@ -759,8 +744,8 @@ func newStart() *cobra.Command { var startSkipWait bool var startTimeout time.Duration - cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach SUCCEEDED state`) - cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`) + cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach ACTIVE state`) + cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach ACTIVE state`) // TODO: short flags cmd.Use = "start NAME" @@ -794,14 +779,14 @@ func newStart() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *apps.AppDeployment) { - if i.Status == nil { + info, err := wait.OnProgress(func(i *apps.App) { + if i.ComputeStatus == nil { return } - status := i.Status.State + status := i.ComputeStatus.State statusMessage := fmt.Sprintf("current status: %s", status) - if i.Status != nil { - statusMessage = i.Status.Message + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message } spinner <- statusMessage }).GetWithTimeout(startTimeout) @@ -838,6 +823,11 @@ func newStop() *cobra.Command { var stopReq apps.StopAppRequest + var stopSkipWait bool + var stopTimeout time.Duration + + cmd.Flags().BoolVar(&stopSkipWait, "no-wait", stopSkipWait, `do not wait to reach STOPPED state`) + cmd.Flags().DurationVar(&stopTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach STOPPED state`) // TODO: short flags cmd.Use = "stop NAME" @@ -863,11 +853,30 @@ func newStop() *cobra.Command { stopReq.Name = args[0] - err = w.Apps.Stop(ctx, stopReq) + wait, err := w.Apps.Stop(ctx, stopReq) if err != nil { return err } - return nil + if stopSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *apps.App) { + if i.ComputeStatus == nil { + return + } + status := i.ComputeStatus.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message + } + spinner <- statusMessage + }).GetWithTimeout(stopTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index a64a6ab7..b36102d9 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -217,7 +217,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.NodeTypeId, "node-type-id", createReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&createReq.NumWorkers, "num-workers", createReq.NumWorkers, `Number of worker nodes that this cluster should have.`) cmd.Flags().StringVar(&createReq.PolicyId, "policy-id", createReq.PolicyId, `The ID of the cluster policy used to create the cluster if applicable.`) - cmd.Flags().Var(&createReq.RuntimeEngine, "runtime-engine", `Decides which runtime engine to be use, e.g. Supported values: [NULL, PHOTON, STANDARD]`) + cmd.Flags().Var(&createReq.RuntimeEngine, "runtime-engine", `Determines the cluster's runtime engine, either standard or Photon. Supported values: [NULL, PHOTON, STANDARD]`) cmd.Flags().StringVar(&createReq.SingleUserName, "single-user-name", createReq.SingleUserName, `Single user name if data_security_mode is SINGLE_USER.`) // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars @@ -236,6 +236,12 @@ func newCreate() *cobra.Command { If Databricks acquires at least 85% of the requested on-demand nodes, cluster creation will succeed. Otherwise the cluster will terminate with an informative error message. + + Rather than authoring the cluster's JSON definition from scratch, Databricks + recommends filling out the [create compute UI] and then copying the generated + JSON definition from the UI. + + [create compute UI]: https://docs.databricks.com/compute/configure.html Arguments: SPARK_VERSION: The Spark version of the cluster, e.g. 3.3.x-scala2.11. A list of @@ -463,7 +469,7 @@ func newEdit() *cobra.Command { cmd.Flags().StringVar(&editReq.NodeTypeId, "node-type-id", editReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&editReq.NumWorkers, "num-workers", editReq.NumWorkers, `Number of worker nodes that this cluster should have.`) cmd.Flags().StringVar(&editReq.PolicyId, "policy-id", editReq.PolicyId, `The ID of the cluster policy used to create the cluster if applicable.`) - cmd.Flags().Var(&editReq.RuntimeEngine, "runtime-engine", `Decides which runtime engine to be use, e.g. Supported values: [NULL, PHOTON, STANDARD]`) + cmd.Flags().Var(&editReq.RuntimeEngine, "runtime-engine", `Determines the cluster's runtime engine, either standard or Photon. Supported values: [NULL, PHOTON, STANDARD]`) cmd.Flags().StringVar(&editReq.SingleUserName, "single-user-name", editReq.SingleUserName, `Single user name if data_security_mode is SINGLE_USER.`) // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 11be8077..3fe5b268 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -76,6 +76,7 @@ import ( system_schemas "github.com/databricks/cli/cmd/workspace/system-schemas" table_constraints "github.com/databricks/cli/cmd/workspace/table-constraints" tables "github.com/databricks/cli/cmd/workspace/tables" + temporary_table_credentials "github.com/databricks/cli/cmd/workspace/temporary-table-credentials" token_management "github.com/databricks/cli/cmd/workspace/token-management" tokens "github.com/databricks/cli/cmd/workspace/tokens" users "github.com/databricks/cli/cmd/workspace/users" @@ -165,6 +166,7 @@ func All() []*cobra.Command { out = append(out, system_schemas.New()) out = append(out, table_constraints.New()) out = append(out, tables.New()) + out = append(out, temporary_table_credentials.New()) out = append(out, token_management.New()) out = append(out, tokens.New()) out = append(out, users.New()) diff --git a/cmd/workspace/disable-legacy-access/disable-legacy-access.go b/cmd/workspace/disable-legacy-access/disable-legacy-access.go new file mode 100755 index 00000000..fea2b3c4 --- /dev/null +++ b/cmd/workspace/disable-legacy-access/disable-legacy-access.go @@ -0,0 +1,217 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package disable_legacy_access + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "disable-legacy-access", + Short: `'Disabling legacy access' has the following impacts: 1.`, + Long: `'Disabling legacy access' has the following impacts: + + 1. Disables direct access to the Hive Metastore. However, you can still access + Hive Metastore through HMS Federation. 2. Disables Fallback Mode (docs link) + on any External Location access from the workspace. 3. Alters DBFS path access + to use External Location permissions in place of legacy credentials. 4. + Enforces Unity Catalog access on all path based access.`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDisableLegacyAccessRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDisableLegacyAccessRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete Legacy Access Disablement Status.` + cmd.Long = `Delete Legacy Access Disablement Status. + + Deletes legacy access disablement status.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyAccess().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDisableLegacyAccessRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDisableLegacyAccessRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Retrieve Legacy Access Disablement Status.` + cmd.Long = `Retrieve Legacy Access Disablement Status. + + Retrieves legacy access disablement Status.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyAccess().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDisableLegacyAccessRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDisableLegacyAccessRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update Legacy Access Disablement Status.` + cmd.Long = `Update Legacy Access Disablement Status. + + Updates legacy access disablement status.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.Settings.DisableLegacyAccess().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DisableLegacyAccess diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index f1cc4e3f..5b4d9645 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -935,6 +935,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().BoolVar(&updateReq.AllowDuplicateNames, "allow-duplicate-names", updateReq.AllowDuplicateNames, `If false, deployment will fail if name has changed and conflicts the name of another pipeline.`) + cmd.Flags().StringVar(&updateReq.BudgetPolicyId, "budget-policy-id", updateReq.BudgetPolicyId, `Budget policy of this pipeline.`) cmd.Flags().StringVar(&updateReq.Catalog, "catalog", updateReq.Catalog, `A catalog in Unity Catalog to publish data from this pipeline to.`) cmd.Flags().StringVar(&updateReq.Channel, "channel", updateReq.Channel, `DLT Release Channel that specifies which version to use.`) // TODO: array: clusters diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index b92f824d..0837652d 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -53,6 +53,7 @@ func New() *cobra.Command { cmd.AddCommand(newLogs()) cmd.AddCommand(newPatch()) cmd.AddCommand(newPut()) + cmd.AddCommand(newPutAiGateway()) cmd.AddCommand(newQuery()) cmd.AddCommand(newSetPermissions()) cmd.AddCommand(newUpdateConfig()) @@ -151,6 +152,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: ai_gateway // TODO: array: rate_limits cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`) // TODO: array: tags @@ -754,8 +756,9 @@ func newPut() *cobra.Command { cmd.Short = `Update rate limits of a serving endpoint.` cmd.Long = `Update rate limits of a serving endpoint. - Used to update the rate limits of a serving endpoint. NOTE: only external and - foundation model endpoints are supported as of now. + Used to update the rate limits of a serving endpoint. NOTE: Only foundation + model endpoints are currently supported. For external models, use AI Gateway + to manage rate limits. Arguments: NAME: The name of the serving endpoint whose rate limits are being updated. This @@ -800,6 +803,79 @@ func newPut() *cobra.Command { return cmd } +// start put-ai-gateway command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var putAiGatewayOverrides []func( + *cobra.Command, + *serving.PutAiGatewayRequest, +) + +func newPutAiGateway() *cobra.Command { + cmd := &cobra.Command{} + + var putAiGatewayReq serving.PutAiGatewayRequest + var putAiGatewayJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&putAiGatewayJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: guardrails + // TODO: complex arg: inference_table_config + // TODO: array: rate_limits + // TODO: complex arg: usage_tracking_config + + cmd.Use = "put-ai-gateway NAME" + cmd.Short = `Update AI Gateway of a serving endpoint.` + cmd.Long = `Update AI Gateway of a serving endpoint. + + Used to update the AI Gateway of a serving endpoint. NOTE: Only external model + endpoints are currently supported. + + Arguments: + NAME: The name of the serving endpoint whose AI Gateway is being updated. This + field is required.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = putAiGatewayJson.Unmarshal(&putAiGatewayReq) + if err != nil { + return err + } + } + putAiGatewayReq.Name = args[0] + + response, err := w.ServingEndpoints.PutAiGateway(ctx, putAiGatewayReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range putAiGatewayOverrides { + fn(cmd, &putAiGatewayReq) + } + + return cmd +} + // start query command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go index 214986c7..aaeecf41 100755 --- a/cmd/workspace/settings/settings.go +++ b/cmd/workspace/settings/settings.go @@ -8,6 +8,7 @@ import ( automatic_cluster_update "github.com/databricks/cli/cmd/workspace/automatic-cluster-update" compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile" default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace" + disable_legacy_access "github.com/databricks/cli/cmd/workspace/disable-legacy-access" enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring" restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins" ) @@ -31,6 +32,7 @@ func New() *cobra.Command { cmd.AddCommand(automatic_cluster_update.New()) cmd.AddCommand(compliance_security_profile.New()) cmd.AddCommand(default_namespace.New()) + cmd.AddCommand(disable_legacy_access.New()) cmd.AddCommand(enhanced_security_monitoring.New()) cmd.AddCommand(restrict_workspace_admins.New()) diff --git a/cmd/workspace/tables/tables.go b/cmd/workspace/tables/tables.go index 4564b4fe..ec297f29 100755 --- a/cmd/workspace/tables/tables.go +++ b/cmd/workspace/tables/tables.go @@ -220,6 +220,7 @@ func newGet() *cobra.Command { cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include tables in the response for which the principal can only access selective metadata for.`) cmd.Flags().BoolVar(&getReq.IncludeDeltaMetadata, "include-delta-metadata", getReq.IncludeDeltaMetadata, `Whether delta metadata should be included in the response.`) + cmd.Flags().BoolVar(&getReq.IncludeManifestCapabilities, "include-manifest-capabilities", getReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Use = "get FULL_NAME" cmd.Short = `Get a table.` @@ -299,6 +300,7 @@ func newList() *cobra.Command { cmd.Flags().BoolVar(&listReq.IncludeBrowse, "include-browse", listReq.IncludeBrowse, `Whether to include tables in the response for which the principal can only access selective metadata for.`) cmd.Flags().BoolVar(&listReq.IncludeDeltaMetadata, "include-delta-metadata", listReq.IncludeDeltaMetadata, `Whether delta metadata should be included in the response.`) + cmd.Flags().BoolVar(&listReq.IncludeManifestCapabilities, "include-manifest-capabilities", listReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of tables to return.`) cmd.Flags().BoolVar(&listReq.OmitColumns, "omit-columns", listReq.OmitColumns, `Whether to omit the columns of the table from the response or not.`) cmd.Flags().BoolVar(&listReq.OmitProperties, "omit-properties", listReq.OmitProperties, `Whether to omit the properties of the table from the response or not.`) @@ -366,6 +368,7 @@ func newListSummaries() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&listSummariesReq.IncludeManifestCapabilities, "include-manifest-capabilities", listSummariesReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Flags().IntVar(&listSummariesReq.MaxResults, "max-results", listSummariesReq.MaxResults, `Maximum number of summaries for tables to return.`) cmd.Flags().StringVar(&listSummariesReq.PageToken, "page-token", listSummariesReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Flags().StringVar(&listSummariesReq.SchemaNamePattern, "schema-name-pattern", listSummariesReq.SchemaNamePattern, `A sql LIKE pattern (% and _) for schema names.`) diff --git a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go new file mode 100755 index 00000000..8718f7ba --- /dev/null +++ b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go @@ -0,0 +1,122 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package temporary_table_credentials + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "temporary-table-credentials", + Short: `Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locationswhere table data is stored in Databricks.`, + Long: `Temporary Table Credentials refer to short-lived, downscoped credentials used + to access cloud storage locationswhere table data is stored in Databricks. + These credentials are employed to provide secure and time-limitedaccess to + data in cloud environments such as AWS, Azure, and Google Cloud. Each cloud + provider has its own typeof credentials: AWS uses temporary session tokens via + AWS Security Token Service (STS), Azure utilizesShared Access Signatures (SAS) + for its data storage services, and Google Cloud supports temporary + credentialsthrough OAuth 2.0.Temporary table credentials ensure that data + access is limited in scope and duration, reducing the risk ofunauthorized + access or misuse. To use the temporary table credentials API, a metastore + admin needs to enable the external_access_enabled flag (off by default) at the + metastore level, and user needs to be granted the EXTERNAL USE SCHEMA + permission at the schema level by catalog admin. Note that EXTERNAL USE SCHEMA + is a schema level permission that can only be granted by catalog admin + explicitly and is not included in schema ownership or ALL PRIVILEGES on the + schema for security reason.`, + GroupID: "catalog", + Annotations: map[string]string{ + "package": "catalog", + }, + } + + // Add methods + cmd.AddCommand(newGenerateTemporaryTableCredentials()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start generate-temporary-table-credentials command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var generateTemporaryTableCredentialsOverrides []func( + *cobra.Command, + *catalog.GenerateTemporaryTableCredentialRequest, +) + +func newGenerateTemporaryTableCredentials() *cobra.Command { + cmd := &cobra.Command{} + + var generateTemporaryTableCredentialsReq catalog.GenerateTemporaryTableCredentialRequest + var generateTemporaryTableCredentialsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&generateTemporaryTableCredentialsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().Var(&generateTemporaryTableCredentialsReq.Operation, "operation", `The operation performed against the table data, either READ or READ_WRITE. Supported values: [READ, READ_WRITE]`) + cmd.Flags().StringVar(&generateTemporaryTableCredentialsReq.TableId, "table-id", generateTemporaryTableCredentialsReq.TableId, `UUID of the table to read or write.`) + + cmd.Use = "generate-temporary-table-credentials" + cmd.Short = `Generate a temporary table credential.` + cmd.Long = `Generate a temporary table credential. + + Get a short-lived credential for directly accessing the table data on cloud + storage. The metastore must have external_access_enabled flag set to true + (default false). The caller must have EXTERNAL_USE_SCHEMA privilege on the + parent schema and this privilege can only be granted by catalog owners.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) + if err != nil { + return err + } + } + + response, err := w.TemporaryTableCredentials.GenerateTemporaryTableCredentials(ctx, generateTemporaryTableCredentialsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range generateTemporaryTableCredentialsOverrides { + fn(cmd, &generateTemporaryTableCredentialsReq) + } + + return cmd +} + +// end service TemporaryTableCredentials diff --git a/go.mod b/go.mod index 0cf3ef8a..9141274c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.46.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index d8866775..177707a5 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.46.0 h1:D0TxmtSVAOsdnfzH4OGtAmcq+8TyA7Z6fA6JEYhupeY= -github.com/databricks/databricks-sdk-go v0.46.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ= +github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From c28d64f2dc3885becbddc3b44b1bdc5893ba6b7c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 1 Oct 2024 15:20:17 +0200 Subject: [PATCH 62/78] [Release] Release v0.229.0 (#1801) Bundles: * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)). * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)). * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)). * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)). * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)). * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)). * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)). Internal: * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)). * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)). * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)). * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)). * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)). * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)). * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)). * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)). * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)). * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)). API Changes: * Changed `databricks apps delete` command to return . * Changed `databricks apps deploy` command with new required argument order. * Changed `databricks apps start` command to return . * Changed `databricks apps stop` command to return . * Added `databricks temporary-table-credentials` command group. * Added `databricks serving-endpoints put-ai-gateway` command. * Added `databricks disable-legacy-access` command group. * Added `databricks account disable-legacy-features` command group. OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19) Dependency updates: * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)). * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)). * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)). * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)). --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a7e5cf..4f5f68ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Version changelog +## [Release] Release v0.229.0 + +Bundles: + * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)). + * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)). + * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)). + * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)). + * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)). + * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)). + * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)). + +Internal: + * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)). + * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)). + * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)). + * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)). + * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)). + * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)). + * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)). + * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)). + * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)). + * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)). + +API Changes: + * Changed `databricks apps delete` command to return . + * Changed `databricks apps deploy` command with new required argument order. + * Changed `databricks apps start` command to return . + * Changed `databricks apps stop` command to return . + * Added `databricks temporary-table-credentials` command group. + * Added `databricks serving-endpoints put-ai-gateway` command. + * Added `databricks disable-legacy-access` command group. + * Added `databricks account disable-legacy-features` command group. + +OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19) +Dependency updates: + * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)). + * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)). + * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)). + * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)). + ## [Release] Release v0.228.1 Bundles: From 044a00c7f9fc2f2d35794062b63a54d01081ffa4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 15:53:24 +0200 Subject: [PATCH 63/78] Add an error if state files grow bigger than the export limit (#1795) ## Changes Currently API limits on exporting files from workspaces are set at 10 MBs while uploading to is 500 MBs. We want to prevent users running into deadlock when they won't be able to pull state file anymore so we prevent from uploading large state files (over 10 MBs) to Databricks workspace. --- bundle/deploy/state_push.go | 13 +++++++++++ bundle/deploy/terraform/state_push.go | 11 +++++++++ bundle/deploy/terraform/state_push_test.go | 27 ++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/bundle/deploy/state_push.go b/bundle/deploy/state_push.go index 176a907c..6912414c 100644 --- a/bundle/deploy/state_push.go +++ b/bundle/deploy/state_push.go @@ -10,6 +10,8 @@ import ( "github.com/databricks/cli/libs/log" ) +const MaxStateFileSize = 10 * 1024 * 1024 // 10MB + type statePush struct { filerFactory FilerFactory } @@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > MaxStateFileSize { + return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize) + } + } + log.Infof(ctx, "Writing local deployment state file to remote state directory") err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index 6cdde137..84d8e767 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > deploy.MaxStateFileSize { + return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize) + } + } + // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") log.Infof(ctx, "Writing local state file to remote state directory") diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index e022dee1..4cc52b7a 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -3,6 +3,7 @@ package terraform import ( "context" "encoding/json" + "fmt" "io" "testing" @@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) { diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } + +func TestStatePushLargeState(t *testing.T) { + mock := mockfiler.NewMockFiler(t) + m := &statePush{ + identityFiler(mock), + } + + ctx := context.Background() + b := statePushTestBundle(t) + + largeState := map[string]any{} + for i := 0; i < 1000000; i++ { + largeState[fmt.Sprintf("field_%d", i)] = i + } + + // Write a stale local state file. + writeLocalState(t, ctx, b, largeState) + diags := bundle.Apply(ctx, b, m) + assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag") + + // Force the write. + b = statePushTestBundle(t) + b.Config.Bundle.Force = true + diags = bundle.Apply(ctx, b, m) + assert.NoError(t, diags.Error()) +} From 80d55f454066a9c1666cb53387f9e735494fe72a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 2 Oct 2024 15:55:40 +0200 Subject: [PATCH 64/78] Add resource path field to bundle workspace configuration (#1800) ## Changes Default workspace path for resources with a presence in the workspace tree. Note: this path is **not** created automatically (yet). We need this only for dashboards (so far), so can take care of creation if one or more dashboards are part of a deployment. This saves an API call for deployments where this is not necessary. ## Tests Expanded existing tests. --- bundle/config/mutator/default_workspace_paths.go | 4 ++++ bundle/config/mutator/default_workspace_paths_test.go | 3 +++ bundle/config/mutator/process_target_mode.go | 9 ++++++--- bundle/config/workspace.go | 5 +++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 71e562b5..02a1ddb3 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -29,6 +29,10 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl b.Config.Workspace.FilePath = path.Join(root, "files") } + if b.Config.Workspace.ResourcePath == "" { + b.Config.Workspace.ResourcePath = path.Join(root, "resources") + } + if b.Config.Workspace.ArtifactPath == "" { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/default_workspace_paths_test.go b/bundle/config/mutator/default_workspace_paths_test.go index 0ba20ea2..6779c373 100644 --- a/bundle/config/mutator/default_workspace_paths_test.go +++ b/bundle/config/mutator/default_workspace_paths_test.go @@ -22,6 +22,7 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/files", b.Config.Workspace.FilePath) + assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath) assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/state", b.Config.Workspace.StatePath) } @@ -32,6 +33,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { Workspace: config.Workspace{ RootPath: "/", FilePath: "/foo/bar", + ResourcePath: "/foo/bar", ArtifactPath: "/foo/bar", StatePath: "/foo/bar", }, @@ -40,6 +42,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath) + assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath) assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath) } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 70382f05..9944f6ff 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -118,15 +118,18 @@ func findNonUserPath(b *bundle.Bundle) string { if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) { return "root_path" } - if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { - return "state_path" - } if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) { return "file_path" } + if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) { + return "resource_path" + } if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) { return "artifact_path" } + if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { + return "state_path" + } return "" } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index efc5caa6..878d0783 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -54,6 +54,11 @@ type Workspace struct { // This defaults to "${workspace.root}/files". FilePath string `json:"file_path,omitempty"` + // Remote workspace path for resources with a presence in the workspace. + // These are kept outside [FilePath] to avoid potential naming collisions. + // This defaults to "${workspace.root}/resources". + ResourcePath string `json:"resource_path,omitempty"` + // Remote workspace path for build artifacts. // This defaults to "${workspace.root}/artifacts". ArtifactPath string `json:"artifact_path,omitempty"` From a8cff48c0b4d747cb769d09724fecdf3c5a3f25b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 17:34:00 +0200 Subject: [PATCH 65/78] Always prepend bundle remote paths with /Workspace (#1724) ## Changes Due to platform changes, all libraries, notebooks and etc. paths used in Databricks must be started with either /Workspace or /Volumes prefix. This PR makes sure that all bundle paths are correctly prefixed. Note: this change is a breaking change if user previously configured and used `/Workspace/Workspace` folder in their workspace file system or having `/Workspace/${workspace.root_path}...` pattern configured anywhere in their bundle config Fixes: #1751 AI: - [x] Scan DABs config and error out on `/Workspace/${workspace.root_path}...` pattern usage ## Tests Added unit tests --------- Co-authored-by: Pieter Noordhuis --- .../config/mutator/expand_workspace_root.go | 2 +- .../mutator/expand_workspace_root_test.go | 2 +- .../mutator/prepend_workspace_prefix.go | 67 +++++++++++++++ .../mutator/prepend_workspace_prefix_test.go | 79 +++++++++++++++++ .../mutator/rewrite_workspace_prefix.go | 72 ++++++++++++++++ .../mutator/rewrite_workspace_prefix_test.go | 85 +++++++++++++++++++ bundle/config/workspace.go | 2 +- bundle/phases/initialize.go | 7 ++ bundle/tests/pipeline_glob_paths_test.go | 2 +- .../tests/relative_path_translation_test.go | 8 +- internal/bundle/deploy_test.go | 2 +- internal/bundle/helpers.go | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}.pipeline.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- 16 files changed, 327 insertions(+), 17 deletions(-) create mode 100644 bundle/config/mutator/prepend_workspace_prefix.go create mode 100644 bundle/config/mutator/prepend_workspace_prefix_test.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix_test.go diff --git a/bundle/config/mutator/expand_workspace_root.go b/bundle/config/mutator/expand_workspace_root.go index 8954abd4..3f0547de 100644 --- a/bundle/config/mutator/expand_workspace_root.go +++ b/bundle/config/mutator/expand_workspace_root.go @@ -33,7 +33,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag. } if strings.HasPrefix(root, "~/") { - home := fmt.Sprintf("/Users/%s", currentUser.UserName) + home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName) b.Config.Workspace.RootPath = path.Join(home, root[2:]) } diff --git a/bundle/config/mutator/expand_workspace_root_test.go b/bundle/config/mutator/expand_workspace_root_test.go index e6260dbd..40bf35ca 100644 --- a/bundle/config/mutator/expand_workspace_root_test.go +++ b/bundle/config/mutator/expand_workspace_root_test.go @@ -27,7 +27,7 @@ func TestExpandWorkspaceRoot(t *testing.T) { } diags := bundle.Apply(context.Background(), b, mutator.ExpandWorkspaceRoot()) require.NoError(t, diags.Error()) - assert.Equal(t, "/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) + assert.Equal(t, "/Workspace/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) } func TestExpandWorkspaceRootDoesNothing(t *testing.T) { diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go new file mode 100644 index 00000000..dd467344 --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -0,0 +1,67 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type prependWorkspacePrefix struct{} + +// PrependWorkspacePrefix prepends the workspace root path to all paths in the bundle. +func PrependWorkspacePrefix() bundle.Mutator { + return &prependWorkspacePrefix{} +} + +func (m *prependWorkspacePrefix) Name() string { + return "PrependWorkspacePrefix" +} + +var skipPrefixes = []string{ + "/Workspace/", + "/Volumes/", +} + +func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), + } + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { + path, ok := pv.AsString() + if !ok { + return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind()) + } + + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return pv, nil + } + } + + return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + return v, nil + }) + + if err != nil { + return diag.FromErr(err) + } + + return nil +} diff --git a/bundle/config/mutator/prepend_workspace_prefix_test.go b/bundle/config/mutator/prepend_workspace_prefix_test.go new file mode 100644 index 00000000..287c694d --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix_test.go @@ -0,0 +1,79 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestPrependWorkspacePrefix(t *testing.T) { + testCases := []struct { + path string + expected string + }{ + { + path: "/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Shared/test", + expected: "/Workspace/Shared/test", + }, + { + path: "/Workspace/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Volumes/Users/test", + expected: "/Volumes/Users/test", + }, + } + + for _, tc := range testCases { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: tc.path, + ArtifactPath: tc.path, + FilePath: tc.path, + StatePath: tc.path, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, PrependWorkspacePrefix()) + require.Empty(t, diags) + require.Equal(t, tc.expected, b.Config.Workspace.RootPath) + require.Equal(t, tc.expected, b.Config.Workspace.ArtifactPath) + require.Equal(t, tc.expected, b.Config.Workspace.FilePath) + require.Equal(t, tc.expected, b.Config.Workspace.StatePath) + } +} + +func TestPrependWorkspaceForDefaultConfig(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Name: "test", + Target: "dev", + }, + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{ + UserName: "jane@doe.com", + }, + }, + }, + }, + } + diags := bundle.Apply(context.Background(), b, bundle.Seq(DefineDefaultWorkspaceRoot(), ExpandWorkspaceRoot(), DefineDefaultWorkspacePaths(), PrependWorkspacePrefix())) + require.Empty(t, diags) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev", b.Config.Workspace.RootPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/artifacts", b.Config.Workspace.ArtifactPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/files", b.Config.Workspace.FilePath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/state", b.Config.Workspace.StatePath) +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go new file mode 100644 index 00000000..8a39ee8a --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -0,0 +1,72 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type rewriteWorkspacePrefix struct{} + +// RewriteWorkspacePrefix finds any strings in bundle configration that have +// workspace prefix plus workspace path variable used and removes workspace prefix from it. +func RewriteWorkspacePrefix() bundle.Mutator { + return &rewriteWorkspacePrefix{} +} + +func (m *rewriteWorkspacePrefix) Name() string { + return "RewriteWorkspacePrefix" +} + +func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + paths := map[string]string{ + "/Workspace/${workspace.root_path}": "${workspace.root_path}", + "/Workspace${workspace.root_path}": "${workspace.root_path}", + "/Workspace/${workspace.file_path}": "${workspace.file_path}", + "/Workspace${workspace.file_path}": "${workspace.file_path}", + "/Workspace/${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace/${workspace.state_path}": "${workspace.state_path}", + "/Workspace${workspace.state_path}": "${workspace.state_path}", + } + + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + // Walk through the bundle configuration, check all the string leafs and + // see if any of the prefixes are used in the remote path. + return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + vv, ok := v.AsString() + if !ok { + return v, nil + } + + for path, replacePath := range paths { + if strings.Contains(vv, path) { + newPath := strings.Replace(vv, path, replacePath, 1) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", path, vv, newPath), + Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", + Locations: v.Locations(), + Paths: []dyn.Path{p}, + }) + + // Remove the workspace prefix from the string. + return dyn.NewValue(newPath, v.Locations()), nil + } + } + + return v, nil + }) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go new file mode 100644 index 00000000..d75ec89d --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go @@ -0,0 +1,85 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestNoWorkspacePrefixUsed(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/test", + ArtifactPath: "/Workspace/Users/test/artifacts", + FilePath: "/Workspace/Users/test/files", + StatePath: "/Workspace/Users/test/state", + }, + + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "/Workspace/${workspace.root_path}/file1.py", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "/Workspace${workspace.file_path}/notebook1", + }, + Libraries: []compute.Library{ + { + Jar: "/Workspace/${workspace.artifact_path}/jar1.jar", + }, + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "${workspace.file_path}/notebook2", + }, + Libraries: []compute.Library{ + { + Jar: "${workspace.artifact_path}/jar2.jar", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, RewriteWorkspacePrefix()) + require.Len(t, diags, 3) + + expectedErrors := map[string]bool{ + `substring "/Workspace/${workspace.root_path}" found in "/Workspace/${workspace.root_path}/file1.py". Please update this to "${workspace.root_path}/file1.py".`: true, + `substring "/Workspace${workspace.file_path}" found in "/Workspace${workspace.file_path}/notebook1". Please update this to "${workspace.file_path}/notebook1".`: true, + `substring "/Workspace/${workspace.artifact_path}" found in "/Workspace/${workspace.artifact_path}/jar1.jar". Please update this to "${workspace.artifact_path}/jar1.jar".`: true, + } + + for _, d := range diags { + require.Equal(t, d.Severity, diag.Warning) + require.Contains(t, expectedErrors, d.Summary) + delete(expectedErrors, d.Summary) + } + + require.Equal(t, "${workspace.root_path}/file1.py", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].SparkPythonTask.PythonFile) + require.Equal(t, "${workspace.file_path}/notebook1", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar) + require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar) + +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index 878d0783..e64808ab 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -47,7 +47,7 @@ type Workspace struct { // Remote workspace base path for deployment state, for artifacts, as synchronization target. // This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to - // the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`). + // the current user's home directory in the workspace (e.g. `/Workspace/Users/jane@doe.com`). RootPath string `json:"root_path,omitempty"` // Remote workspace path to synchronize local files to. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 93ce61b2..a41819c7 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -39,9 +39,16 @@ func Initialize() bundle.Mutator { mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), mutator.PopulateCurrentUser(), + mutator.DefineDefaultWorkspaceRoot(), mutator.ExpandWorkspaceRoot(), mutator.DefineDefaultWorkspacePaths(), + mutator.PrependWorkspacePrefix(), + + // This mutator needs to be run before variable interpolation because it + // searches for strings with variable references in them. + mutator.RewriteWorkspacePrefix(), + mutator.SetVariables(), // Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences, // ResolveVariableReferencesInComplexVariables and ResolveVariableReferences. diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go index c1c62cfb..d17f0404 100644 --- a/bundle/tests/pipeline_glob_paths_test.go +++ b/bundle/tests/pipeline_glob_paths_test.go @@ -11,7 +11,7 @@ func TestExpandPipelineGlobPaths(t *testing.T) { require.NoError(t, diags.Error()) require.Equal( t, - "/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", + "/Workspace/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Libraries[0].Notebook.Path, ) } diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go index 199871d2..0f553ac3 100644 --- a/bundle/tests/relative_path_translation_test.go +++ b/bundle/tests/relative_path_translation_test.go @@ -12,9 +12,9 @@ func TestRelativePathTranslationDefault(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file1.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file1.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t1.SparkPythonTask.PythonFile) } func TestRelativePathTranslationOverride(t *testing.T) { @@ -22,7 +22,7 @@ func TestRelativePathTranslationOverride(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file2.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file2.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t1.SparkPythonTask.PythonFile) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 736c880d..88543585 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -236,7 +236,7 @@ func TestAccDeployBasicBundleLogs(t *testing.T) { stdout, stderr := blackBoxRun(t, root, "bundle", "deploy") assert.Equal(t, strings.Join([]string{ - fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), + fmt.Sprintf("Uploading bundle files to /Workspace/Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), "Deploying resources...", "Updating deployment state...", "Deployment complete!\n", diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 3547c175..b8c81a8d 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -114,7 +114,7 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique // Compute root path for the bundle deployment me, err := w.CurrentUser.Me(context.Background()) require.NoError(t, err) - root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId) + root := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s", me.UserName, uniqueId) return root } diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index f96ce4fe..5594749a 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -24,8 +24,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl index 8544dc83..c42b822a 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl @@ -21,8 +21,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index bf469046..50e5ad97 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -15,4 +15,4 @@ resources: path: ../src/dlt_pipeline.ipynb configuration: - bundle.sourcePath: /Workspace/${workspace.file_path}/src + bundle.sourcePath: ${workspace.file_path}/src diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 55c1aae4..51d03e99 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -41,8 +41,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}} catalog: {{.default_catalog}} From bca9c2eda4cf66d1baaaf78a11abf67d5e587503 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:46:20 +0530 Subject: [PATCH 66/78] Add validation for files with a `.(resource-name).yml` extension (#1780) ## Changes We want to encourage a pattern of specifying only a single resource in a YAML file when the `.(resource-type).yml` extension is used (for example, `.job.yml`). This convention could allow us to bijectively map a resource YAML file to its corresponding resource in the Databricks workspace. This PR: 1. Emits a recommendation diagnostic when we detect this convention is being violated. We can promote this to a warning when we want to encourage this pattern more strongly. 2. Visualises the recommendation diagnostics in the `bundle validate` command. **NOTE:** While this PR also shows the recommendation for `.yaml` files, we do not encourage users to use this extension. We only support it here since it's part of the YAML standard and some existing users might already be using `.yaml`. ## Tests Unit tests and manually. Here's what an example output looks like: ``` Recommendation: define a single job in a file with the .job.yml extension. at resources.jobs.bar resources.jobs.foo in foo.job.yml:13:7 foo.job.yml:5:7 The following resources are defined or configured in this file: - bar (job) - foo (job) ``` --------- Co-authored-by: Lennart Kats (databricks) --- bundle/config/loader/entry_point_test.go | 2 +- bundle/config/loader/process_include.go | 130 ++++++++++++ bundle/config/loader/process_include_test.go | 185 +++++++++++++++++- .../testdata/{ => basic}/databricks.yml | 0 .../loader/testdata/{ => basic}/host.yml | 0 .../format_match/job_and_pipeline.yml | 11 ++ .../format_match/multiple_resources.yml | 43 ++++ .../testdata/format_match/one_job.job.yml | 11 ++ .../format_match/one_pipeline.pipeline.yaml | 4 + .../loader/testdata/format_match/two_job.yml | 7 + .../job_and_pipeline.experiment.yml | 11 ++ .../format_not_match/job_and_pipeline.job.yml | 11 ++ ...tiple_resources.model_serving_endpoint.yml | 43 ++++ .../second_job_in_target.job.yml | 11 ++ .../format_not_match/single_job.pipeline.yaml | 11 ++ .../format_not_match/two_jobs.job.yml | 7 + .../two_jobs_in_target.job.yml | 8 + bundle/config/resources.go | 19 ++ bundle/config/resources_test.go | 16 ++ bundle/render/render_text_output.go | 34 +++- bundle/render/render_text_output_test.go | 164 +++++++++++++++- libs/diag/severity.go | 1 + 22 files changed, 721 insertions(+), 8 deletions(-) rename bundle/config/loader/testdata/{ => basic}/databricks.yml (100%) rename bundle/config/loader/testdata/{ => basic}/host.yml (100%) create mode 100644 bundle/config/loader/testdata/format_match/job_and_pipeline.yml create mode 100644 bundle/config/loader/testdata/format_match/multiple_resources.yml create mode 100644 bundle/config/loader/testdata/format_match/one_job.job.yml create mode 100644 bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml create mode 100644 bundle/config/loader/testdata/format_match/two_job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml create mode 100644 bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml create mode 100644 bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml create mode 100644 bundle/config/loader/testdata/format_not_match/two_jobs.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index 406b9b67..0723c056 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) { func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", } diags := bundle.Apply(context.Background(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 7cf9a17d..f82f5db1 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -3,12 +3,135 @@ package loader import ( "context" "fmt" + "slices" + "sort" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) +func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { + for _, resourceDescription := range config.SupportedResources() { + singularName := resourceDescription.SingularName + + for _, yamlExt := range []string{"yml", "yaml"} { + ext := fmt.Sprintf(".%s.%s", singularName, yamlExt) + if strings.HasSuffix(filePath, ext) { + return validateSingleResourceDefined(configRoot, ext, singularName) + } + } + } + + return nil +} + +func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { + type resource struct { + path dyn.Path + value dyn.Value + typ string + key string + } + + resources := []resource{} + supportedResources := config.SupportedResources() + + // Gather all resources defined in the resources block. + _, err := dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[2].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[1].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + // Gather all resources defined in a target block. + _, err = dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[4].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[3].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + typeMatch := true + seenKeys := map[string]struct{}{} + for _, rr := range resources { + // case: The resource is not of the correct type. + if rr.typ != typ { + typeMatch = false + break + } + + seenKeys[rr.key] = struct{}{} + } + + // Format matches. There's at most one resource defined in the file. + // The resource is also of the correct type. + if typeMatch && len(seenKeys) <= 1 { + return nil + } + + detail := strings.Builder{} + detail.WriteString("The following resources are defined or configured in this file:\n") + lines := []string{} + for _, r := range resources { + lines = append(lines, fmt.Sprintf(" - %s (%s)\n", r.key, r.typ)) + } + // Sort the lines to print to make the output deterministic. + sort.Strings(lines) + // Compact the lines before writing them to the message to remove any duplicate lines. + // This is needed because we do not dedup earlier when gathering the resources + // and it's valid to define the same resource in both the resources and targets block. + lines = slices.Compact(lines) + for _, l := range lines { + detail.WriteString(l) + } + + locations := []dyn.Location{} + paths := []dyn.Path{} + for _, rr := range resources { + locations = append(locations, rr.value.Locations()...) + paths = append(paths, rr.path) + } + // Sort the locations and paths to make the output deterministic. + sort.Slice(locations, func(i, j int) bool { + return locations[i].String() < locations[j].String() + }) + sort.Slice(paths, func(i, j int) bool { + return paths[i].String() < paths[j].String() + }) + + return diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext), + Detail: detail.String(), + Locations: locations, + Paths: paths, + }, + } +} + type processInclude struct { fullPath string relPath string @@ -31,6 +154,13 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos if diags.HasError() { return diags } + + // Add any diagnostics associated with the file format. + diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + if diags.HasError() { + return diags + } + err := b.Config.Merge(this) if err != nil { diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index 2ccd84b3..66c695e1 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -8,13 +8,15 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -33,3 +35,184 @@ func TestProcessInclude(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Workspace.Host) } + +func TestProcessIncludeFormatMatch(t *testing.T) { + for _, fileName := range []string{ + "one_job.job.yml", + "one_pipeline.pipeline.yaml", + "two_job.yml", + "job_and_pipeline.yml", + "multiple_resources.yml", + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + assert.Empty(t, diags) + }) + } +} + +func TestProcessIncludeFormatNotMatch(t *testing.T) { + for fileName, expectedDiags := range map[string]diag.Diagnostics{ + "single_job.pipeline.yaml": { + { + Severity: diag.Recommendation, + Summary: "define a single pipeline in a file with the .pipeline.yaml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.experiment.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single experiment in a file with the .experiment.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "two_jobs.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 7, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + }, + }, + }, + "second_job_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "two_jobs_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 6, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 8, Column: 11}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "multiple_resources.model_serving_endpoint.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single model serving endpoint in a file with the .model_serving_endpoint.yml extension.", + Detail: `The following resources are defined or configured in this file: + - experiment1 (experiment) + - job1 (job) + - job2 (job) + - job3 (job) + - model1 (model) + - model_serving_endpoint1 (model_serving_endpoint) + - pipeline1 (pipeline) + - pipeline2 (pipeline) + - quality_monitor1 (quality_monitor) + - registered_model1 (registered_model) + - schema1 (schema) +`, + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 12, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 14, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 18, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 22, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 24, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 28, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 35, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 39, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 43, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.experiments.experiment1"), + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"), + dyn.MustPathFromString("resources.models.model1"), + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("resources.pipelines.pipeline2"), + dyn.MustPathFromString("resources.schemas.schema1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job3"), + dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"), + dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"), + }, + }, + }, + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_not_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + require.Len(t, diags, 1) + assert.Equal(t, expectedDiags, diags) + }) + } +} diff --git a/bundle/config/loader/testdata/databricks.yml b/bundle/config/loader/testdata/basic/databricks.yml similarity index 100% rename from bundle/config/loader/testdata/databricks.yml rename to bundle/config/loader/testdata/basic/databricks.yml diff --git a/bundle/config/loader/testdata/host.yml b/bundle/config/loader/testdata/basic/host.yml similarity index 100% rename from bundle/config/loader/testdata/host.yml rename to bundle/config/loader/testdata/basic/host.yml diff --git a/bundle/config/loader/testdata/format_match/job_and_pipeline.yml b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml new file mode 100644 index 00000000..0867fcae --- /dev/null +++ b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_match/multiple_resources.yml b/bundle/config/loader/testdata/format_match/multiple_resources.yml new file mode 100644 index 00000000..dc8e837c --- /dev/null +++ b/bundle/config/loader/testdata/format_match/multiple_resources.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_match/one_job.job.yml b/bundle/config/loader/testdata/format_match/one_job.job.yml new file mode 100644 index 00000000..91af87cd --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_job.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml new file mode 100644 index 00000000..85cb0d7f --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml @@ -0,0 +1,4 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 diff --git a/bundle/config/loader/testdata/format_match/two_job.yml b/bundle/config/loader/testdata/format_match/two_job.yml new file mode 100644 index 00000000..81ff90a7 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/two_job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml new file mode 100644 index 00000000..0867fcae --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml new file mode 100644 index 00000000..0867fcae --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml new file mode 100644 index 00000000..dc8e837c --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml new file mode 100644 index 00000000..628b9879 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml new file mode 100644 index 00000000..91af87cd --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml new file mode 100644 index 00000000..81ff90a7 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml new file mode 100644 index 00000000..3b489c1f --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml @@ -0,0 +1,8 @@ +targets: + target1: + resources: + jobs: + job1: + description: job1 + job2: + description: job2 diff --git a/bundle/config/resources.go b/bundle/config/resources.go index a3afb7fc..dc51a7ca 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -59,3 +59,22 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) return found[0], nil } + +type ResourceDescription struct { + SingularName string +} + +// The keys of the map corresponds to the resource key in the bundle configuration. +func SupportedResources() map[string]ResourceDescription { + return map[string]ResourceDescription{ + "jobs": {SingularName: "job"}, + "pipelines": {SingularName: "pipeline"}, + "models": {SingularName: "model"}, + "experiments": {SingularName: "experiment"}, + "model_serving_endpoints": {SingularName: "model_serving_endpoint"}, + "registered_models": {SingularName: "registered_model"}, + "quality_monitors": {SingularName: "quality_monitor"}, + "schemas": {SingularName: "schema"}, + "clusters": {SingularName: "cluster"}, + } +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 6860d73d..c1b76118 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -61,3 +62,18 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { }, "Resource %s does not have a custom unmarshaller", field.Name) } } + +func TestSupportedResources(t *testing.T) { + expected := map[string]ResourceDescription{} + typ := reflect.TypeOf(Resources{}) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + jsonTags := strings.Split(field.Tag.Get("json"), ",") + singularName := strings.TrimSuffix(jsonTags[0], "s") + expected[jsonTags[0]] = ResourceDescription{SingularName: singularName} + } + + // Please add your resource to the SupportedResources() function in resources.go + // if you are adding a new resource. + assert.Equal(t, expected, SupportedResources()) +} diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index e1fad98a..56387c38 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -56,6 +56,20 @@ const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} ` +const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + const summaryTemplate = `{{- if .Name -}} Name: {{ .Name | bold }} {{- if .Target }} @@ -94,9 +108,20 @@ func buildTrailer(diags diag.Diagnostics) string { if warnings := len(diags.Filter(diag.Warning)); warnings > 0 { parts = append(parts, color.YellowString(pluralize(warnings, "warning", "warnings"))) } - if len(parts) > 0 { - return fmt.Sprintf("Found %s", strings.Join(parts, " and ")) - } else { + if recommendations := len(diags.Filter(diag.Recommendation)); recommendations > 0 { + parts = append(parts, color.BlueString(pluralize(recommendations, "recommendation", "recommendations"))) + } + switch { + case len(parts) >= 3: + first := strings.Join(parts[:len(parts)-1], ", ") + last := parts[len(parts)-1] + return fmt.Sprintf("Found %s, and %s", first, last) + case len(parts) == 2: + return fmt.Sprintf("Found %s and %s", parts[0], parts[1]) + case len(parts) == 1: + return fmt.Sprintf("Found %s", parts[0]) + default: + // No diagnostics to print. return color.GreenString("Validation OK!") } } @@ -130,6 +155,7 @@ func renderSummaryTemplate(out io.Writer, b *bundle.Bundle, diags diag.Diagnosti func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) + recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate)) // Print errors and warnings. for _, d := range diags { @@ -139,6 +165,8 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) t = errorT case diag.Warning: t = warningT + case diag.Recommendation: + t = recommendationT } for i := range d.Locations { diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 976f86e7..1a41fa01 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -45,6 +45,19 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "Found 1 error\n", }, + { + name: "nil bundle and 1 recommendation", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "recommendation", + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Recommendation: recommendation\n" + + "\n" + + "Found 1 recommendation\n", + }, { name: "bundle during 'load' and 1 error", bundle: loadingBundle, @@ -84,7 +97,7 @@ func TestRenderTextOutput(t *testing.T) { "Found 2 warnings\n", }, { - name: "bundle during 'load' and 2 errors, 1 warning with details", + name: "bundle during 'load' and 2 errors, 1 warning and 1 recommendation with details", bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -105,6 +118,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (3)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: error (1)\n" + @@ -122,10 +141,114 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "detail (3)\n" + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + "Name: test-bundle\n" + "Target: test-target\n" + "\n" + - "Found 2 errors and 1 warning\n", + "Found 2 errors, 1 warning, and 1 recommendation\n", + }, + { + name: "bundle during 'load' and 1 error and 1 warning", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error and 1 warning\n", + }, + { + name: "bundle during 'load' and 1 errors, 2 warning and 2 recommendations with details", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (5)", + Detail: "detail (5)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Warning: warning (3)\n" + + " in foo.py:3:1\n" + + "\n" + + "detail (3)\n" + + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + + "Recommendation: recommendation (5)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (5)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error, 2 warnings, and 2 recommendations\n", }, { name: "bundle during 'init'", @@ -158,7 +281,7 @@ func TestRenderTextOutput(t *testing.T) { "Validation OK!\n", }, { - name: "nil bundle without summary with 1 error and 1 warning", + name: "nil bundle without summary with 1 error, 1 warning and 1 recommendation", bundle: nil, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -173,6 +296,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (2)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: false}, expected: "Error: error (1)\n" + @@ -184,6 +313,11 @@ func TestRenderTextOutput(t *testing.T) { " in foo.py:3:1\n" + "\n" + "detail (2)\n" + + "\n" + + "Recommendation: recommendation (3)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (3)\n" + "\n", }, } @@ -304,6 +438,30 @@ func TestRenderDiagnostics(t *testing.T) { "\n" + "'name' is required\n\n", }, + { + name: "recommendation with multiple paths and locations", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "summary", + Detail: "detail", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + }, + Locations: []dyn.Location{ + {File: "foo.yaml", Line: 1, Column: 2}, + {File: "bar.yaml", Line: 3, Column: 4}, + }, + }, + }, + expected: "Recommendation: summary\n" + + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "detail\n\n", + }, } for _, tc := range testCases { diff --git a/libs/diag/severity.go b/libs/diag/severity.go index d25c1280..0e88085f 100644 --- a/libs/diag/severity.go +++ b/libs/diag/severity.go @@ -6,4 +6,5 @@ const ( Error Severity = iota Warning Info + Recommendation ) From 589e5ff1a44b3632d69fa5d2b992530a54ab7247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:58:25 +0200 Subject: [PATCH 67/78] Bump golang.org/x/term from 0.24.0 to 0.25.0 (#1811) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.24.0 to 0.25.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.24.0&new-version=0.25.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9141274c..ae56b513 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( golang.org/x/mod v0.21.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.24.0 + golang.org/x/term v0.25.0 golang.org/x/text v0.18.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -64,7 +64,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 177707a5..e797ac54 100644 --- a/go.sum +++ b/go.sum @@ -212,10 +212,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= From 02bfd397eff71380b3b9e466f6a821eee6794fb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:14:02 +0000 Subject: [PATCH 68/78] Bump golang.org/x/text from 0.18.0 to 0.19.0 (#1812) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.18.0 to 0.19.0.
Commits
  • 3043346 x/text: Correct examples in number/doc
  • 38a95c2 all: fix some comments
  • 20097e4 all: fix printf(var) mistakes detected by latest printf checker
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.18.0&new-version=0.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ae56b513..c029e5da 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.25.0 - golang.org/x/text v0.18.0 + golang.org/x/text v0.19.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index e797ac54..9f05b145 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,8 @@ golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 7a2141fc5ba1af695ba55aeac524bf80644d7606 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:21:05 +0200 Subject: [PATCH 69/78] Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 (#1810) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.47.0 to 0.48.0.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.48.0

Internal Changes

  • Update SDK to latest OpenAPI spec (#1057).

Note: This release contains breaking changes, please see the API changes below for more details.

API Changes:

OpenAPI SHA: 0c86ea6dbd9a730c24ff0d4e509603e476955ac5, Date: 2024-10-02

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.48.0

Internal Changes

  • Update SDK to latest OpenAPI spec (#1057).

Note: This release contains breaking changes, please see the API changes below for more details.

API Changes:

OpenAPI SHA: 0c86ea6dbd9a730c24ff0d4e509603e476955ac5, Date: 2024-10-02

Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.47.0&new-version=0.48.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- bundle/schema/jsonschema.json | 235 +++++++++++++++++- cmd/workspace/apps/apps.go | 2 + .../git-credentials/git-credentials.go | 62 ++--- cmd/workspace/pipelines/pipelines.go | 1 + cmd/workspace/repos/overrides.go | 4 +- cmd/workspace/repos/repos.go | 35 +-- go.mod | 2 +- go.sum | 4 +- internal/helpers.go | 2 +- internal/locker_test.go | 2 +- internal/repos_test.go | 2 +- internal/sync_test.go | 2 +- 13 files changed, 296 insertions(+), 59 deletions(-) diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index ffd6f58d..303c7855 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -6f6b1371e640f2dfeba72d365ac566368656f6b6 \ No newline at end of file +0c86ea6dbd9a730c24ff0d4e509603e476955ac5 \ No newline at end of file diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index afdf9fb9..ae209c01 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -402,6 +402,10 @@ { "type": "object", "properties": { + "ai_gateway": { + "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig" + }, "config": { "description": "The core config of the serving endpoint.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" @@ -472,6 +476,10 @@ { "type": "object", "properties": { + "budget_policy_id": { + "description": "Budget policy of this pipeline.", + "$ref": "#/$defs/string" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string" @@ -539,6 +547,10 @@ "description": "Whether Photon is enabled for this pipeline.", "$ref": "#/$defs/bool" }, + "schema": { + "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", + "$ref": "#/$defs/string" + }, "serverless": { "description": "Whether serverless compute is enabled for this pipeline.", "$ref": "#/$defs/bool" @@ -1206,6 +1218,9 @@ "profile": { "$ref": "#/$defs/string" }, + "resource_path": { + "$ref": "#/$defs/string" + }, "root_path": { "$ref": "#/$defs/string" }, @@ -2632,7 +2647,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -3073,6 +3088,7 @@ "$ref": "#/$defs/map/string" }, "pipeline_params": { + "description": "Controls whether the pipeline should perform a full refresh", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" }, "python_named_params": { @@ -3547,7 +3563,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -4365,6 +4381,207 @@ } ] }, + "serving.AiGatewayConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "guardrails": { + "description": "Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails" + }, + "inference_table_config": { + "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig" + }, + "rate_limits": { + "description": "Configuration for rate limits which can be set to limit endpoint traffic.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + }, + "usage_tracking_config": { + "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailParameters": { + "anyOf": [ + { + "type": "object", + "properties": { + "invalid_keywords": { + "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", + "$ref": "#/$defs/slice/string" + }, + "pii": { + "description": "Configuration for guardrail PII filter.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior" + }, + "safety": { + "description": "Indicates whether the safety filter is enabled.", + "$ref": "#/$defs/bool" + }, + "valid_topics": { + "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehavior": { + "anyOf": [ + { + "type": "object", + "properties": { + "behavior": { + "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior", + "enum": [ + "NONE", + "BLOCK" + ] + } + }, + "additionalProperties": false, + "required": [ + "behavior" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehaviorBehavior": { + "type": "string" + }, + "serving.AiGatewayGuardrails": { + "anyOf": [ + { + "type": "object", + "properties": { + "input": { + "description": "Configuration for input guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + }, + "output": { + "description": "Configuration for output guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayInferenceTableConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int" + }, + "key": { + "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey", + "enum": [ + "user", + "endpoint" + ] + }, + "renewal_period": { + "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod", + "enum": [ + "minute" + ] + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimitKey": { + "type": "string" + }, + "serving.AiGatewayRateLimitRenewalPeriod": { + "type": "string" + }, + "serving.AiGatewayUsageTrackingConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to enable usage tracking.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.AmazonBedrockConfig": { "anyOf": [ { @@ -5569,6 +5786,20 @@ } ] }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.EndpointTag": { "anyOf": [ { diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index baec6d03..780f5594 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -81,6 +81,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "create NAME" cmd.Short = `Create an app.` @@ -910,6 +911,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "update NAME" cmd.Short = `Update an app.` diff --git a/cmd/workspace/git-credentials/git-credentials.go b/cmd/workspace/git-credentials/git-credentials.go index 2e8cc2cd..b5082d31 100755 --- a/cmd/workspace/git-credentials/git-credentials.go +++ b/cmd/workspace/git-credentials/git-credentials.go @@ -53,13 +53,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateCredentials, + *workspace.CreateCredentialsRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateCredentials + var createReq workspace.CreateCredentialsRequest var createJson flags.JsonFlag // TODO: short flags @@ -79,8 +79,9 @@ func newCreate() *cobra.Command { Arguments: GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -136,13 +137,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *workspace.DeleteGitCredentialRequest, + *workspace.DeleteCredentialsRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq workspace.DeleteGitCredentialRequest + var deleteReq workspace.DeleteCredentialsRequest // TODO: short flags @@ -209,13 +210,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *workspace.GetGitCredentialRequest, + *workspace.GetCredentialsRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq workspace.GetGitCredentialRequest + var getReq workspace.GetCredentialsRequest // TODO: short flags @@ -322,33 +323,48 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateCredentials, + *workspace.UpdateCredentialsRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateCredentials + var updateReq workspace.UpdateCredentialsRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateReq.GitProvider, "git-provider", updateReq.GitProvider, `Git provider.`) cmd.Flags().StringVar(&updateReq.GitUsername, "git-username", updateReq.GitUsername, `The username or email provided with your Git provider account, depending on which provider you are using.`) cmd.Flags().StringVar(&updateReq.PersonalAccessToken, "personal-access-token", updateReq.PersonalAccessToken, `The personal access token used to authenticate to the corresponding Git provider.`) - cmd.Use = "update CREDENTIAL_ID" + cmd.Use = "update CREDENTIAL_ID GIT_PROVIDER" cmd.Short = `Update a credential.` cmd.Long = `Update a credential. Updates the specified Git credential. Arguments: - CREDENTIAL_ID: The ID for the corresponding credential to access.` + CREDENTIAL_ID: The ID for the corresponding credential to access. + GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only CREDENTIAL_ID as positional arguments. Provide 'git_provider' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -360,27 +376,13 @@ func newUpdate() *cobra.Command { return err } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No CREDENTIAL_ID argument specified. Loading names for Git Credentials drop-down." - names, err := w.GitCredentials.CredentialInfoGitProviderToCredentialIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Git Credentials drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding credential to access") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding credential to access") - } _, err = fmt.Sscan(args[0], &updateReq.CredentialId) if err != nil { return fmt.Errorf("invalid CREDENTIAL_ID: %s", args[0]) } + if !cmd.Flags().Changed("json") { + updateReq.GitProvider = args[1] + } err = w.GitCredentials.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 5b4d9645..ac361e31 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -954,6 +954,7 @@ func newUpdate() *cobra.Command { // TODO: array: notifications cmd.Flags().BoolVar(&updateReq.Photon, "photon", updateReq.Photon, `Whether Photon is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.PipelineId, "pipeline-id", updateReq.PipelineId, `Unique identifier for this pipeline.`) + cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`) cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`) cmd.Flags().StringVar(&updateReq.Target, "target", updateReq.Target, `Target schema (database) to add tables in this pipeline to.`) diff --git a/cmd/workspace/repos/overrides.go b/cmd/workspace/repos/overrides.go index 96d645ef..9546d1c1 100644 --- a/cmd/workspace/repos/overrides.go +++ b/cmd/workspace/repos/overrides.go @@ -19,7 +19,7 @@ func listOverride(listCmd *cobra.Command, listReq *workspace.ListReposRequest) { {{end}}`) } -func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepo) { +func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepoRequest) { createCmd.Use = "create URL [PROVIDER]" createCmd.Args = func(cmd *cobra.Command, args []string) error { // If the provider argument is not specified, we try to detect it from the URL. @@ -95,7 +95,7 @@ func getOverride(getCmd *cobra.Command, getReq *workspace.GetRepoRequest) { } } -func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepo) { +func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepoRequest) { updateCmd.Use = "update REPO_ID_OR_PATH" updateJson := updateCmd.Flag("json").Value.(*flags.JsonFlag) diff --git a/cmd/workspace/repos/repos.go b/cmd/workspace/repos/repos.go index fb3d51b0..f11dd3ac 100755 --- a/cmd/workspace/repos/repos.go +++ b/cmd/workspace/repos/repos.go @@ -61,13 +61,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateRepo, + *workspace.CreateRepoRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateRepo + var createReq workspace.CreateRepoRequest var createJson flags.JsonFlag // TODO: short flags @@ -87,8 +87,9 @@ func newCreate() *cobra.Command { Arguments: URL: URL of the Git repository to be linked. PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -164,7 +165,7 @@ func newDelete() *cobra.Command { Deletes the specified repo. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -181,14 +182,14 @@ func newDelete() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &deleteReq.RepoId) if err != nil { @@ -237,7 +238,7 @@ func newGet() *cobra.Command { Returns the repo with the given repo ID. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -254,14 +255,14 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &getReq.RepoId) if err != nil { @@ -451,8 +452,8 @@ func newList() *cobra.Command { cmd.Short = `Get repos.` cmd.Long = `Get repos. - Returns repos that the calling user has Manage permissions on. Results are - paginated with each page containing twenty repos.` + Returns repos that the calling user has Manage permissions on. Use + next_page_token to iterate through additional pages.` cmd.Annotations = make(map[string]string) @@ -569,13 +570,13 @@ func newSetPermissions() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateRepo, + *workspace.UpdateRepoRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateRepo + var updateReq workspace.UpdateRepoRequest var updateJson flags.JsonFlag // TODO: short flags @@ -593,7 +594,7 @@ func newUpdate() *cobra.Command { latest commit on the same branch. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -616,14 +617,14 @@ func newUpdate() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &updateReq.RepoId) if err != nil { diff --git a/go.mod b/go.mod index c029e5da..697205f3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.48.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index 9f05b145..03698b20 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ= -github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.48.0 h1:46KtsnRo+FGhC3izUXbpL0PXBNomvsdignYDhJZlm9s= +github.com/databricks/databricks-sdk-go v0.48.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/helpers.go b/internal/helpers.go index 419fa419..9387706b 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -519,7 +519,7 @@ func TemporaryRepo(t *testing.T, w *databricks.WorkspaceClient) string { repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("integration-test-repo-")) t.Logf("Creating repo:%s", repoPath) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Url: "https://github.com/databricks/cli", Provider: "github", Path: repoPath, diff --git a/internal/locker_test.go b/internal/locker_test.go index 21e08f73..3ae783d1 100644 --- a/internal/locker_test.go +++ b/internal/locker_test.go @@ -29,7 +29,7 @@ func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databr assert.NoError(t, err) remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix)) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: remoteProjectRoot, Url: EmptyRepoUrl, Provider: "gitHub", diff --git a/internal/repos_test.go b/internal/repos_test.go index de0d926a..1ad0e877 100644 --- a/internal/repos_test.go +++ b/internal/repos_test.go @@ -34,7 +34,7 @@ func synthesizeTemporaryRepoPath(t *testing.T, w *databricks.WorkspaceClient, ct func createTemporaryRepo(t *testing.T, w *databricks.WorkspaceClient, ctx context.Context) (int64, string) { repoPath := synthesizeTemporaryRepoPath(t, w, ctx) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", diff --git a/internal/sync_test.go b/internal/sync_test.go index 4021e649..6f8b1827 100644 --- a/internal/sync_test.go +++ b/internal/sync_test.go @@ -38,7 +38,7 @@ func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Contex require.NoError(t, err) repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-")) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", From 88318d384acd29ed58143e0bfbbb0c382fa785c8 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:43:42 +0530 Subject: [PATCH 70/78] Remove deprecated or readonly fields from the bundle schema (#1809) ## Changes We do not need the user to specify these fields in their bundle configuration, so we remove them from the JSON schema. ## Tests Manually and end to end tests. The JSON schema has also been regenerated after these changes. --- bundle/internal/schema/main.go | 28 +++++++++++++++++++ .../fail/deprecated_job_field_format.yml | 4 +++ .../fail/hidden_job_field_deployment.yml | 6 ++++ .../fail/hidden_job_field_edit_mode.yml | 6 ++++ .../fail/readonly_job_field_git_snapshot.yml | 8 ++++++ .../fail/readonly_job_field_job_source.yml | 9 ++++++ bundle/internal/schema/testdata/pass/job.yml | 5 ++-- bundle/schema/embed_test.go | 4 +-- bundle/schema/jsonschema.json | 19 ------------- 9 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml create mode 100644 bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml create mode 100644 bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml create mode 100644 bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml create mode 100644 bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 4a237147..ddeffe2f 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -8,8 +8,10 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/databricks-sdk-go/service/jobs" ) func interpolationPattern(s string) string { @@ -66,6 +68,31 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } +func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Job{}): + // This field has been deprecated in jobs API v2.1 and is always set to + // "MULTI_TASK" in the backend. We should not expose it to the user. + delete(s.Properties, "format") + + // These fields are only meant to be set by the DABs client (ie the CLI) + // and thus should not be exposed to the user. These are used to annotate + // jobs that were created by DABs. + delete(s.Properties, "deployment") + delete(s.Properties, "edit_mode") + + case reflect.TypeOf(jobs.GitSource{}): + // These fields are readonly and are not meant to be set by the user. + delete(s.Properties, "job_source") + delete(s.Properties, "git_snapshot") + + default: + // Do nothing + } + + return s +} + func main() { if len(os.Args) != 2 { fmt.Println("Usage: go run main.go ") @@ -90,6 +117,7 @@ func main() { s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ p.addDescriptions, p.addEnums, + removeJobsFields, addInterpolationPatterns, }) if err != nil { diff --git a/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml new file mode 100644 index 00000000..62e490b0 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + format: SINGLE_TASK diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml new file mode 100644 index 00000000..705ce951 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml @@ -0,0 +1,6 @@ +resources: + jobs: + foo: + deployment: + kind: BUNDLE + metadata_file_path: /a/b/c diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml new file mode 100644 index 00000000..9cbe95f0 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml @@ -0,0 +1,6 @@ +targets: + foo: + resources: + jobs: + bar: + edit_mode: whatever diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml new file mode 100644 index 00000000..c57a560a --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + git_snapshot: + used_commit: abcdef diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml new file mode 100644 index 00000000..9973e3bd --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml @@ -0,0 +1,9 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + job_source: + import_from_git_branch: master + job_config_path: def diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml index d9b0e832..e13a52c0 100644 --- a/bundle/internal/schema/testdata/pass/job.yml +++ b/bundle/internal/schema/testdata/pass/job.yml @@ -32,7 +32,6 @@ resources: name: myjob continuous: pause_status: PAUSED - edit_mode: EDITABLE max_concurrent_runs: 10 description: "my job description" email_notifications: @@ -43,10 +42,12 @@ resources: dependencies: - python=3.7 client: "myclient" - format: MULTI_TASK tags: foo: bar bar: baz + git_source: + git_provider: gitHub + git_url: www.github.com/a/b tasks: - task_key: mytask notebook_task: diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go index ee0b5a61..dcb381b8 100644 --- a/bundle/schema/embed_test.go +++ b/bundle/schema/embed_test.go @@ -39,7 +39,7 @@ func TestJsonSchema(t *testing.T) { // Assert job fields have their descriptions loaded. resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job") - fields := []string{"name", "continuous", "deployment", "tasks", "trigger"} + fields := []string{"name", "continuous", "tasks", "trigger"} for _, field := range fields { assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description) } @@ -53,7 +53,7 @@ func TestJsonSchema(t *testing.T) { // Assert descriptions are loaded for pipelines pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline") - fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"} + fields = []string{"name", "catalog", "clusters", "channel", "continuous", "development"} for _, field := range fields { assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index ae209c01..06b9cc15 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -213,18 +213,10 @@ "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment" - }, "description": { "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", "$ref": "#/$defs/string" }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode" - }, "email_notifications": { "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" @@ -233,10 +225,6 @@ "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format" - }, "git_source": { "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" @@ -2547,9 +2535,6 @@ "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" }, - "git_snapshot": { - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot" - }, "git_tag": { "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", "$ref": "#/$defs/string" @@ -2557,10 +2542,6 @@ "git_url": { "description": "URL of the repository to be cloned by this job.", "$ref": "#/$defs/string" - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource" } }, "additionalProperties": false, From ae568743c5ccbfb8ae9d3298c7aeee96d0dcf1b4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 8 Oct 2024 13:41:32 +0200 Subject: [PATCH 71/78] Upgrade TF provider to 1.53.0 (#1815) ## Changes See https://github.com/databricks/terraform-provider-databricks/releases/tag/v1.53.0 ## Tests Integration tests pass. --- bundle/internal/tf/codegen/schema/version.go | 2 +- .../schema/data_source_current_metastore.go | 1 + .../tf/schema/data_source_metastore.go | 1 + .../tf/schema/data_source_mlflow_models.go | 8 +++ bundle/internal/tf/schema/data_sources.go | 2 + bundle/internal/tf/schema/resource_budget.go | 49 +++++++++++++++++ .../tf/schema/resource_model_serving.go | 52 +++++++++++++++++++ .../tf/schema/resource_permissions.go | 2 +- .../internal/tf/schema/resource_pipeline.go | 2 + .../internal/tf/schema/resource_sql_table.go | 2 + bundle/internal/tf/schema/resources.go | 2 + bundle/internal/tf/schema/root.go | 2 +- 12 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 bundle/internal/tf/schema/data_source_mlflow_models.go create mode 100644 bundle/internal/tf/schema/resource_budget.go diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index b71ea7d1..49e48a6e 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.53.0" diff --git a/bundle/internal/tf/schema/data_source_current_metastore.go b/bundle/internal/tf/schema/data_source_current_metastore.go index 11e647fd..4f8c135a 100644 --- a/bundle/internal/tf/schema/data_source_current_metastore.go +++ b/bundle/internal/tf/schema/data_source_current_metastore.go @@ -10,6 +10,7 @@ type DataSourceCurrentMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_metastore.go b/bundle/internal/tf/schema/data_source_metastore.go index ce206479..4244febc 100644 --- a/bundle/internal/tf/schema/data_source_metastore.go +++ b/bundle/internal/tf/schema/data_source_metastore.go @@ -10,6 +10,7 @@ type DataSourceMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_mlflow_models.go b/bundle/internal/tf/schema/data_source_mlflow_models.go new file mode 100644 index 00000000..360924e5 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_mlflow_models.go @@ -0,0 +1,8 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceMlflowModels struct { + Id string `json:"id,omitempty"` + Names []string `json:"names,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 4ac78613..10829b99 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -30,6 +30,7 @@ type DataSources struct { Metastores map[string]any `json:"databricks_metastores,omitempty"` MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MlflowModels map[string]any `json:"databricks_mlflow_models,omitempty"` MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` NodeType map[string]any `json:"databricks_node_type,omitempty"` @@ -85,6 +86,7 @@ func NewDataSources() *DataSources { Metastores: make(map[string]any), MlflowExperiment: make(map[string]any), MlflowModel: make(map[string]any), + MlflowModels: make(map[string]any), MwsCredentials: make(map[string]any), MwsWorkspaces: make(map[string]any), NodeType: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_budget.go b/bundle/internal/tf/schema/resource_budget.go new file mode 100644 index 00000000..5566eb93 --- /dev/null +++ b/bundle/internal/tf/schema/resource_budget.go @@ -0,0 +1,49 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceBudgetAlertConfigurationsActionConfigurations struct { + ActionConfigurationId string `json:"action_configuration_id,omitempty"` + ActionType string `json:"action_type,omitempty"` + Target string `json:"target,omitempty"` +} + +type ResourceBudgetAlertConfigurations struct { + AlertConfigurationId string `json:"alert_configuration_id,omitempty"` + QuantityThreshold string `json:"quantity_threshold,omitempty"` + QuantityType string `json:"quantity_type,omitempty"` + TimePeriod string `json:"time_period,omitempty"` + TriggerType string `json:"trigger_type,omitempty"` + ActionConfigurations []ResourceBudgetAlertConfigurationsActionConfigurations `json:"action_configurations,omitempty"` +} + +type ResourceBudgetFilterTagsValue struct { + Operator string `json:"operator,omitempty"` + Values []string `json:"values,omitempty"` +} + +type ResourceBudgetFilterTags struct { + Key string `json:"key,omitempty"` + Value *ResourceBudgetFilterTagsValue `json:"value,omitempty"` +} + +type ResourceBudgetFilterWorkspaceId struct { + Operator string `json:"operator,omitempty"` + Values []int `json:"values,omitempty"` +} + +type ResourceBudgetFilter struct { + Tags []ResourceBudgetFilterTags `json:"tags,omitempty"` + WorkspaceId *ResourceBudgetFilterWorkspaceId `json:"workspace_id,omitempty"` +} + +type ResourceBudget struct { + AccountId string `json:"account_id,omitempty"` + BudgetConfigurationId string `json:"budget_configuration_id,omitempty"` + CreateTime int `json:"create_time,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Id string `json:"id,omitempty"` + UpdateTime int `json:"update_time,omitempty"` + AlertConfigurations []ResourceBudgetAlertConfigurations `json:"alert_configurations,omitempty"` + Filter *ResourceBudgetFilter `json:"filter,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 29d55cd5..71cf8925 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -2,6 +2,57 @@ package schema +type ResourceModelServingAiGatewayGuardrailsInputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsInput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsInputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrailsOutputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsOutput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsOutputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrails struct { + Input *ResourceModelServingAiGatewayGuardrailsInput `json:"input,omitempty"` + Output *ResourceModelServingAiGatewayGuardrailsOutput `json:"output,omitempty"` +} + +type ResourceModelServingAiGatewayInferenceTableConfig struct { + CatalogName string `json:"catalog_name,omitempty"` + Enabled bool `json:"enabled,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + TableNamePrefix string `json:"table_name_prefix,omitempty"` +} + +type ResourceModelServingAiGatewayRateLimits struct { + Calls int `json:"calls"` + Key string `json:"key,omitempty"` + RenewalPeriod string `json:"renewal_period"` +} + +type ResourceModelServingAiGatewayUsageTrackingConfig struct { + Enabled bool `json:"enabled,omitempty"` +} + +type ResourceModelServingAiGateway struct { + Guardrails *ResourceModelServingAiGatewayGuardrails `json:"guardrails,omitempty"` + InferenceTableConfig *ResourceModelServingAiGatewayInferenceTableConfig `json:"inference_table_config,omitempty"` + RateLimits []ResourceModelServingAiGatewayRateLimits `json:"rate_limits,omitempty"` + UsageTrackingConfig *ResourceModelServingAiGatewayUsageTrackingConfig `json:"usage_tracking_config,omitempty"` +} + type ResourceModelServingConfigAutoCaptureConfig struct { CatalogName string `json:"catalog_name,omitempty"` Enabled bool `json:"enabled,omitempty"` @@ -139,6 +190,7 @@ type ResourceModelServing struct { Name string `json:"name"` RouteOptimized bool `json:"route_optimized,omitempty"` ServingEndpointId string `json:"serving_endpoint_id,omitempty"` + AiGateway *ResourceModelServingAiGateway `json:"ai_gateway,omitempty"` Config *ResourceModelServingConfig `json:"config,omitempty"` RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"` Tags []ResourceModelServingTags `json:"tags,omitempty"` diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index ee94a1a8..0c3b90ed 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -4,7 +4,7 @@ package schema type ResourcePermissionsAccessControl struct { GroupName string `json:"group_name,omitempty"` - PermissionLevel string `json:"permission_level"` + PermissionLevel string `json:"permission_level,omitempty"` ServicePrincipalName string `json:"service_principal_name,omitempty"` UserName string `json:"user_name,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 15468646..1bed91fc 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -238,6 +238,7 @@ type ResourcePipelineTrigger struct { type ResourcePipeline struct { AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` Catalog string `json:"catalog,omitempty"` Cause string `json:"cause,omitempty"` Channel string `json:"channel,omitempty"` @@ -254,6 +255,7 @@ type ResourcePipeline struct { Name string `json:"name,omitempty"` Photon bool `json:"photon,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` + Schema string `json:"schema,omitempty"` Serverless bool `json:"serverless,omitempty"` State string `json:"state,omitempty"` Storage string `json:"storage,omitempty"` diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 4f305c52..bcf2a8e8 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -4,9 +4,11 @@ package schema type ResourceSqlTableColumn struct { Comment string `json:"comment,omitempty"` + Identity string `json:"identity,omitempty"` Name string `json:"name"` Nullable bool `json:"nullable,omitempty"` Type string `json:"type,omitempty"` + TypeJson string `json:"type_json,omitempty"` } type ResourceSqlTable struct { diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 737b77a2..53f558df 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -10,6 +10,7 @@ type Resources struct { AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` + Budget map[string]any `json:"databricks_budget,omitempty"` Catalog map[string]any `json:"databricks_catalog,omitempty"` CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` Cluster map[string]any `json:"databricks_cluster,omitempty"` @@ -112,6 +113,7 @@ func NewResources() *Resources { AzureAdlsGen1Mount: make(map[string]any), AzureAdlsGen2Mount: make(map[string]any), AzureBlobMount: make(map[string]any), + Budget: make(map[string]any), Catalog: make(map[string]any), CatalogWorkspaceBinding: make(map[string]any), Cluster: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 5fc34d6b..7a0cc01f 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.53.0" func NewRoot() *Root { return &Root{ From c92c67afc9755e0d3ae58f85c1ea427a2253d195 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 9 Oct 2024 15:06:57 +0200 Subject: [PATCH 72/78] [Release] Release v0.230.0 (#1817) Notable changes for Databricks Asset Bundles: Workspace paths are automatically prefixed with `/Workspace`. In addition, all usage of path strings such as `/Workspace/${workspace.root_path}/...` in bundle configuration is automatically replaced with `${workspace.root_path}/...` and generates a warning as part of bundle validate. More details can be find here: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths-will-be-automatically-prefixed Bundles: * Add an error if state files grow bigger than the export limit ([#1795](https://github.com/databricks/cli/pull/1795)). * Always prepend bundle remote paths with /Workspace ([#1724](https://github.com/databricks/cli/pull/1724)). * Add resource path field to bundle workspace configuration ([#1800](https://github.com/databricks/cli/pull/1800)). * Add validation for files with a `.(resource-name).yml` extension ([#1780](https://github.com/databricks/cli/pull/1780)). Internal: * Remove deprecated or readonly fields from the bundle schema ([#1809](https://github.com/databricks/cli/pull/1809)). API Changes: * Changed `databricks git-credentials create`, `databricks git-credentials delete`, `databricks git-credentials get`, `databricks git-credentials list`, `databricks git-credentials update` commands . * Changed `databricks repos create`, `databricks repos delete`, `databricks repos get`, `databricks repos update` command . OpenAPI commit 0c86ea6dbd9a730c24ff0d4e509603e476955ac5 (2024-10-02) Dependency updates: * Upgrade TF provider to 1.53.0 ([#1815](https://github.com/databricks/cli/pull/1815)). * Bump golang.org/x/term from 0.24.0 to 0.25.0 ([#1811](https://github.com/databricks/cli/pull/1811)). * Bump golang.org/x/text from 0.18.0 to 0.19.0 ([#1812](https://github.com/databricks/cli/pull/1812)). * Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 ([#1810](https://github.com/databricks/cli/pull/1810)). --------- Co-authored-by: Pieter Noordhuis --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5f68ac..f31bb10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Version changelog +## [Release] Release v0.230.0 + +Notable changes for Databricks Asset Bundles: + +Workspace paths are automatically prefixed with `/Workspace`. In addition, all usage of path strings such as `/Workspace/${workspace.root_path}/...` in bundle configuration is automatically replaced with `${workspace.root_path}/...` and generates a warning as part of bundle validate. + +More details can be found here: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths + +Bundles: + * Add an error if state files grow bigger than the export limit ([#1795](https://github.com/databricks/cli/pull/1795)). + * Always prepend bundle remote paths with /Workspace ([#1724](https://github.com/databricks/cli/pull/1724)). + * Add resource path field to bundle workspace configuration ([#1800](https://github.com/databricks/cli/pull/1800)). + * Add validation for files with a `.(resource-name).yml` extension ([#1780](https://github.com/databricks/cli/pull/1780)). + +Internal: + * Remove deprecated or readonly fields from the bundle schema ([#1809](https://github.com/databricks/cli/pull/1809)). + +API Changes: + * Changed `databricks git-credentials create`, `databricks git-credentials delete`, `databricks git-credentials get`, `databricks git-credentials list`, `databricks git-credentials update` commands . + * Changed `databricks repos create`, `databricks repos delete`, `databricks repos get`, `databricks repos update` command . + +OpenAPI commit 0c86ea6dbd9a730c24ff0d4e509603e476955ac5 (2024-10-02) +Dependency updates: + * Upgrade TF provider to 1.53.0 ([#1815](https://github.com/databricks/cli/pull/1815)). + * Bump golang.org/x/term from 0.24.0 to 0.25.0 ([#1811](https://github.com/databricks/cli/pull/1811)). + * Bump golang.org/x/text from 0.18.0 to 0.19.0 ([#1812](https://github.com/databricks/cli/pull/1812)). + * Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 ([#1810](https://github.com/databricks/cli/pull/1810)). + ## [Release] Release v0.229.0 Bundles: From e885794722793a53ae2cd05ee87659cde702a8fb Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Thu, 10 Oct 2024 13:18:23 +0200 Subject: [PATCH 73/78] Show actionable errors for collaborative deployment scenarios (#1386) ## Changes This adds diagnostics for collaborative (production) deployment scenarios, including: - Bob deploys a bundle that is normally deployed by Alice, but this fails because Bob can't write to `/Users/Alice/.bundle`. - Charlie deploys a bundle that is normally deployed by Alice, but this fails because he can't create a new pipeline where Alice would be the owner. - Alice deploys a bundle where she didn't list herself as one of the CAN_MANAGE users in permissions. That can work, but is probably a mistake. ## Tests Unit tests, manual testing. --- bundle/config/mutator/run_as.go | 101 ++++++++-------- bundle/config/mutator/run_as_test.go | 9 +- bundle/deploy/files/upload.go | 6 + bundle/deploy/lock/acquire.go | 9 +- bundle/deploy/terraform/apply.go | 5 + bundle/permissions/mutator.go | 1 + bundle/permissions/permission_diagnostics.go | 110 ++++++++++++++++++ .../permission_diagnostics_test.go | 52 +++++++++ bundle/permissions/permission_report.go | 51 ++++++++ bundle/permissions/permission_report_test.go | 76 ++++++++++++ bundle/permissions/terraform_errors.go | 47 ++++++++ bundle/permissions/terraform_errors_test.go | 97 +++++++++++++++ bundle/phases/initialize.go | 2 + bundle/tests/run_as_test.go | 25 ++-- libs/diag/diagnostic.go | 15 ++- libs/diag/id.go | 16 +++ libs/filer/filer.go | 12 ++ libs/filer/workspace_files_client.go | 12 +- libs/set/set.go | 10 ++ 19 files changed, 579 insertions(+), 77 deletions(-) create mode 100644 bundle/permissions/permission_diagnostics.go create mode 100644 bundle/permissions/permission_diagnostics_test.go create mode 100644 bundle/permissions/permission_report.go create mode 100644 bundle/permissions/permission_report_test.go create mode 100644 bundle/permissions/terraform_errors.go create mode 100644 bundle/permissions/terraform_errors_test.go create mode 100644 libs/diag/id.go diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index 423bc38e..6b3069d4 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -30,50 +30,44 @@ func (m *setRunAs) Name() string { return "SetRunAs" } -type errUnsupportedResourceTypeForRunAs struct { - resourceType string - resourceLocation dyn.Location - currentUser string - runAsUser string +func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser string, runAsUser string) diag.Diagnostics { + return diag.Diagnostics{{ + Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: %s. Run as identity: %s.\n"+ + "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser), + Locations: []dyn.Location{location}, + Severity: diag.Error, + }} } -func (e errUnsupportedResourceTypeForRunAs) Error() string { - return fmt.Sprintf("%s are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s. Current identity: %s. Run as identity: %s", e.resourceType, e.resourceLocation, e.currentUser, e.runAsUser) -} +func validateRunAs(b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} -type errBothSpAndUserSpecified struct { - spName string - spLoc dyn.Location - userName string - userLoc dyn.Location -} + neitherSpecifiedErr := diag.Diagnostics{{ + Summary: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, + }} -func (e errBothSpAndUserSpecified) Error() string { - return fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name %q is specified at %s. A user_name %q is defined at %s", e.spName, e.spLoc, e.userName, e.userLoc) -} - -func validateRunAs(b *bundle.Bundle) error { - neitherSpecifiedErr := fmt.Errorf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s", b.Config.GetLocation("run_as")) - // Error if neither service_principal_name nor user_name are specified, but the + // Fail fast if neither service_principal_name nor user_name are specified, but the // run_as section is present. if b.Config.Value().Get("run_as").Kind() == dyn.KindNil { return neitherSpecifiedErr } - // Error if one or both of service_principal_name and user_name are specified, + + // Fail fast if one or both of service_principal_name and user_name are specified, // but with empty values. - if b.Config.RunAs.ServicePrincipalName == "" && b.Config.RunAs.UserName == "" { + runAs := b.Config.RunAs + if runAs.ServicePrincipalName == "" && runAs.UserName == "" { return neitherSpecifiedErr } - // Error if both service_principal_name and user_name are specified - runAs := b.Config.RunAs if runAs.UserName != "" && runAs.ServicePrincipalName != "" { - return errBothSpAndUserSpecified{ - spName: runAs.ServicePrincipalName, - userName: runAs.UserName, - spLoc: b.Config.GetLocation("run_as.service_principal_name"), - userLoc: b.Config.GetLocation("run_as.user_name"), - } + diags = diags.Extend(diag.Diagnostics{{ + Summary: "run_as section cannot specify both user_name and service_principal_name", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, + }}) } identity := runAs.ServicePrincipalName @@ -83,40 +77,40 @@ func validateRunAs(b *bundle.Bundle) error { // All resources are supported if the run_as identity is the same as the current deployment identity. if identity == b.Config.Workspace.CurrentUser.UserName { - return nil + return diags } // DLT pipelines do not support run_as in the API. if len(b.Config.Resources.Pipelines) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "pipelines", - resourceLocation: b.Config.GetLocation("resources.pipelines"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "pipelines", + b.Config.GetLocation("resources.pipelines"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } // Model serving endpoints do not support run_as in the API. if len(b.Config.Resources.ModelServingEndpoints) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "model_serving_endpoints", - resourceLocation: b.Config.GetLocation("resources.model_serving_endpoints"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "model_serving_endpoints", + b.Config.GetLocation("resources.model_serving_endpoints"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } // Monitors do not support run_as in the API. if len(b.Config.Resources.QualityMonitors) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "quality_monitors", - resourceLocation: b.Config.GetLocation("resources.quality_monitors"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "quality_monitors", + b.Config.GetLocation("resources.quality_monitors"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } - return nil + return diags } func setRunAsForJobs(b *bundle.Bundle) { @@ -187,8 +181,9 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { } // Assert the run_as configuration is valid in the context of the bundle - if err := validateRunAs(b); err != nil { - return diag.FromErr(err) + diags := validateRunAs(b) + if diags.HasError() { + return diags } setRunAsForJobs(b) diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index abeea45d..8076b82f 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -188,11 +188,8 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { Config: *r, } diags := bundle.Apply(context.Background(), b, SetRunAs()) - assert.Equal(t, diags.Error().Error(), errUnsupportedResourceTypeForRunAs{ - resourceType: rt, - resourceLocation: dyn.Location{}, - currentUser: "alice", - runAsUser: "bob", - }.Error(), "expected run_as with a different identity than the current deployment user to not supported for resources of type: %s", rt) + assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: alice. Run as identity: bob.\n"+ + "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt) } } diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index 77b83611..bab4e176 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -2,9 +2,12 @@ package files import ( "context" + "errors" "fmt" + "io/fs" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -35,6 +38,9 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { b.Files, err = sync.RunOnce(ctx) if err != nil { + if errors.Is(err, fs.ErrPermission) { + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.FilePath) + } return diag.FromErr(err) } diff --git a/bundle/deploy/lock/acquire.go b/bundle/deploy/lock/acquire.go index 7d3d0eca..ab1f1cba 100644 --- a/bundle/deploy/lock/acquire.go +++ b/bundle/deploy/lock/acquire.go @@ -3,8 +3,10 @@ package lock import ( "context" "errors" + "io/fs" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/locker" @@ -51,12 +53,17 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics if err != nil { log.Errorf(ctx, "Failed to acquire deployment lock: %v", err) + if errors.Is(err, fs.ErrPermission) { + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath) + } + notExistsError := filer.NoSuchDirectoryError{} if errors.As(err, ¬ExistsError) { // If we get a "doesn't exist" error from the API this indicates // we either don't have permissions or the path is invalid. - return diag.Errorf("cannot write to deployment root (this can indicate a previous deploy was done with a different identity): %s", b.Config.Workspace.RootPath) + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath) } + return diag.FromErr(err) } diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index e52d0ca8..5ea2effa 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -4,6 +4,7 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" @@ -34,6 +35,10 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Apply terraform according to the computed plan err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { + diags := permissions.TryExtendTerraformPermissionError(ctx, b, err) + if diags != nil { + return diags + } return diag.Errorf("terraform apply: %v", err) } diff --git a/bundle/permissions/mutator.go b/bundle/permissions/mutator.go index 7787bc04..247ffcda 100644 --- a/bundle/permissions/mutator.go +++ b/bundle/permissions/mutator.go @@ -13,6 +13,7 @@ import ( const CAN_MANAGE = "CAN_MANAGE" const CAN_VIEW = "CAN_VIEW" const CAN_RUN = "CAN_RUN" +const IS_OWNER = "IS_OWNER" var allowedLevels = []string{CAN_MANAGE, CAN_VIEW, CAN_RUN} var levelsMap = map[string](map[string]string){ diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go new file mode 100644 index 00000000..d2c24fa0 --- /dev/null +++ b/bundle/permissions/permission_diagnostics.go @@ -0,0 +1,110 @@ +package permissions + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/set" +) + +type permissionDiagnostics struct{} + +func PermissionDiagnostics() bundle.Mutator { + return &permissionDiagnostics{} +} + +func (m *permissionDiagnostics) Name() string { + return "CheckPermissions" +} + +func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if len(b.Config.Permissions) == 0 { + // Only warn if there is an explicit top-level permissions section + return nil + } + + canManageBundle, _ := analyzeBundlePermissions(b) + if canManageBundle { + return nil + } + + return diag.Diagnostics{{ + Severity: diag.Warning, + Summary: fmt.Sprintf("permissions section should include %s or one of their groups with CAN_MANAGE permissions", b.Config.Workspace.CurrentUser.UserName), + Locations: []dyn.Location{b.Config.GetLocation("permissions")}, + ID: diag.PermissionNotIncluded, + }} +} + +// analyzeBundlePermissions analyzes the top-level permissions of the bundle. +// This permission set is important since it determines the permissions of the +// target workspace folder. +// +// Returns: +// - isManager: true if the current user is can manage the bundle resources. +// - assistance: advice on who to contact as to manage this project +func analyzeBundlePermissions(b *bundle.Bundle) (bool, string) { + canManageBundle := false + otherManagers := set.NewSet[string]() + if b.Config.RunAs != nil && b.Config.RunAs.UserName != "" && b.Config.RunAs.UserName != b.Config.Workspace.CurrentUser.UserName { + // The run_as user is another human that could be contacted + // about this bundle. + otherManagers.Add(b.Config.RunAs.UserName) + } + + currentUser := b.Config.Workspace.CurrentUser.UserName + targetPermissions := b.Config.Permissions + for _, p := range targetPermissions { + if p.Level != CAN_MANAGE { + continue + } + + if p.UserName == currentUser || p.ServicePrincipalName == currentUser { + canManageBundle = true + continue + } + + if isGroupOfCurrentUser(b, p.GroupName) { + canManageBundle = true + continue + } + + // Permission doesn't apply to current user; add to otherManagers + otherManager := p.UserName + if otherManager == "" { + otherManager = p.GroupName + } + if otherManager == "" { + // Skip service principals + continue + } + otherManagers.Add(otherManager) + } + + assistance := "For assistance, contact the owners of this project." + if otherManagers.Size() > 0 { + list := otherManagers.Values() + sort.Strings(list) + assistance = fmt.Sprintf( + "For assistance, users or groups with appropriate permissions may include: %s.", + strings.Join(list, ", "), + ) + } + return canManageBundle, assistance +} + +func isGroupOfCurrentUser(b *bundle.Bundle, groupName string) bool { + currentUserGroups := b.Config.Workspace.CurrentUser.User.Groups + + for _, g := range currentUserGroups { + if g.Display == groupName { + return true + } + } + return false +} diff --git a/bundle/permissions/permission_diagnostics_test.go b/bundle/permissions/permission_diagnostics_test.go new file mode 100644 index 00000000..7b0afefa --- /dev/null +++ b/bundle/permissions/permission_diagnostics_test.go @@ -0,0 +1,52 @@ +package permissions_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestPermissionDiagnosticsApplySuccess(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) + require.NoError(t, diags.Error()) +} + +func TestPermissionDiagnosticsApplyFail(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_VIEW", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) + require.Equal(t, diags[0].Severity, diag.Warning) + require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions") +} + +func mockBundle(permissions []resources.Permission) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{ + UserName: "testuser@databricks.com", + DisplayName: "Test User", + Groups: []iam.ComplexValue{ + {Display: "testgroup"}, + }, + }, + }, + }, + Permissions: permissions, + }, + } +} diff --git a/bundle/permissions/permission_report.go b/bundle/permissions/permission_report.go new file mode 100644 index 00000000..a9468edb --- /dev/null +++ b/bundle/permissions/permission_report.go @@ -0,0 +1,51 @@ +package permissions + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" +) + +// ReportPossiblePermissionDenied generates a diagnostic message when a permission denied error is encountered. +// +// Note that since the workspace API doesn't always distinguish between permission denied and path errors, +// we must treat this as a "possible permission error". See acquire.go for more about this. +func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) diag.Diagnostics { + log.Errorf(ctx, "Failed to update, encountered possible permission error: %v", path) + + user := b.Config.Workspace.CurrentUser.UserName + if auth.IsServicePrincipal(user) { + user = b.Config.Workspace.CurrentUser.DisplayName + } + canManageBundle, assistance := analyzeBundlePermissions(b) + + if !canManageBundle { + return diag.Diagnostics{{ + Summary: fmt.Sprintf("unable to deploy to %s as %s.\n"+ + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n"+ + "%s\n"+ + "They may need to redeploy the bundle to apply the new permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + path, user, assistance), + Severity: diag.Error, + ID: diag.PathPermissionDenied, + }} + } + + // According databricks.yml, the current user has the right permissions. + // But we're still seeing permission errors. So someone else will need + // to redeploy the bundle with the right set of permissions. + return diag.Diagnostics{{ + Summary: fmt.Sprintf("unable to deploy to %s as %s. Cannot apply local deployment permissions.\n"+ + "%s\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + path, user, assistance), + Severity: diag.Error, + ID: diag.CannotChangePathPermissions, + }} +} diff --git a/bundle/permissions/permission_report_test.go b/bundle/permissions/permission_report_test.go new file mode 100644 index 00000000..61592f7e --- /dev/null +++ b/bundle/permissions/permission_report_test.go @@ -0,0 +1,76 @@ +package permissions_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/stretchr/testify/require" +) + +func TestPermissionsReportPermissionDeniedWithGroup(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "testgroup"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n" + + "For assistance, contact the owners of this project.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedWithOtherGroup(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "othergroup"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, users or groups with appropriate permissions may include: othergroup.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedWithoutPermission(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_VIEW", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, contact the owners of this project.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedNilPermission(t *testing.T) { + b := mockBundle(nil) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, contact the owners of this project.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportFindOtherOwners(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "testgroup"}, + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + require.ErrorContains(t, diags.Error(), "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n"+ + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.") +} diff --git a/bundle/permissions/terraform_errors.go b/bundle/permissions/terraform_errors.go new file mode 100644 index 00000000..cc4c9f61 --- /dev/null +++ b/bundle/permissions/terraform_errors.go @@ -0,0 +1,47 @@ +package permissions + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" +) + +func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) diag.Diagnostics { + _, assistance := analyzeBundlePermissions(b) + + // In a best-effort attempt to provide actionable error messages, we match + // against a few specific error messages that come from the Jobs and Pipelines API. + // For matching errors we provide a more specific error message that includes + // details on how to resolve the issue. + if !strings.Contains(err.Error(), "cannot update permissions") && + !strings.Contains(err.Error(), "permissions on pipeline") && + !strings.Contains(err.Error(), "cannot read permissions") && + !strings.Contains(err.Error(), "cannot set run_as to user") { + return nil + } + + log.Errorf(ctx, "Terraform error during deployment: %v", err.Error()) + + // Best-effort attempt to extract the resource name from the error message. + re := regexp.MustCompile(`databricks_(\w*)\.(\w*)`) + match := re.FindStringSubmatch(err.Error()) + resource := "resource" + if len(match) > 1 { + resource = match[2] + } + + return diag.Diagnostics{{ + Summary: fmt.Sprintf("permission denied creating or updating %s.\n"+ + "%s\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + resource, assistance), + Severity: diag.Error, + ID: diag.ResourcePermissionDenied, + }} +} diff --git a/bundle/permissions/terraform_errors_test.go b/bundle/permissions/terraform_errors_test.go new file mode 100644 index 00000000..c9b5b178 --- /dev/null +++ b/bundle/permissions/terraform_errors_test.go @@ -0,0 +1,97 @@ +package permissions_test + +import ( + "context" + "errors" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestTryExtendTerraformPermissionError1(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot update permissions: ...\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" + + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionError2(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + {Level: "CAN_MANAGE", UserName: "bob@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com, bob@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionError3(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "testuser@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read permissions: 1706906c-c0a2-4c25-9f57-3a7aa3cb8b90 does not have Owner permissions on Job with ID: ElasticJobId(28263044278868). Please contact the owner or an administrator for access.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, contact the owners of this project.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionErrorNotOwner(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "data_team@databricks.com"}, + }) + b.Config.RunAs = &jobs.JobRunAs{ + UserName: "testuser@databricks.com", + } + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: data_team@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a41819c7..da5b2eff 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -62,6 +62,8 @@ func Initialize() bundle.Mutator { "workspace", "variables", ), + // Provide permission config errors & warnings after initializing all variables + permissions.PermissionDiagnostics(), mutator.SetRunAs(), mutator.OverrideCompute(), mutator.ProcessTargetMode(), diff --git a/bundle/tests/run_as_test.go b/bundle/tests/run_as_test.go index 6c07cc53..92057714 100644 --- a/bundle/tests/run_as_test.go +++ b/bundle/tests/run_as_test.go @@ -3,7 +3,6 @@ package config_tests import ( "context" "fmt" - "path/filepath" "testing" "github.com/databricks/cli/bundle" @@ -113,8 +112,9 @@ func TestRunAsErrorForPipelines(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/pipelines/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("pipelines are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath)) + assert.ErrorContains(t, err, "pipelines do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+ + "See https://docs") } func TestRunAsNoErrorForPipelines(t *testing.T) { @@ -152,8 +152,9 @@ func TestRunAsErrorForModelServing(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/model_serving/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("model_serving_endpoints are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath)) + assert.ErrorContains(t, err, "model_serving_endpoints do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+ + "See https://docs") } func TestRunAsNoErrorForModelServingEndpoints(t *testing.T) { @@ -191,8 +192,7 @@ func TestRunAsErrorWhenBothUserAndSpSpecified(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/both_sp_and_user/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name \"my_service_principal\" is specified at %s:6:27. A user_name \"my_user_name\" is defined at %s:7:14", configPath, configPath)) + assert.ErrorContains(t, err, "run_as section cannot specify both user_name and service_principal_name") } func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) { @@ -202,19 +202,19 @@ func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) { }{ { name: "empty_run_as", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:8", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_run_as/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_sp", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_sp/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_user", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_user_and_sp", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user_and_sp/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, } @@ -257,8 +257,7 @@ func TestRunAsErrorNeitherUserOrSpSpecifiedAtTargetOverride(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/override/override.yml") - assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:12", configPath)) + assert.EqualError(t, err, "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified") } func TestLegacyRunAs(t *testing.T) { diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index 93334c06..254ecbd7 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -1,6 +1,7 @@ package diag import ( + "errors" "fmt" "github.com/databricks/cli/libs/dyn" @@ -24,6 +25,9 @@ type Diagnostic struct { // Paths are paths to the values in the configuration tree that the diagnostic is associated with. // It may be nil if there are no associated paths. Paths []dyn.Path + + // A diagnostic ID. Only used for select diagnostic messages. + ID ID } // Errorf creates a new error diagnostic. @@ -69,7 +73,7 @@ func Infof(format string, args ...any) Diagnostics { } } -// Diagsnostics holds zero or more instances of [Diagnostic]. +// Diagnostics holds zero or more instances of [Diagnostic]. type Diagnostics []Diagnostic // Append adds a new diagnostic to the end of the list. @@ -96,7 +100,14 @@ func (ds Diagnostics) HasError() bool { func (ds Diagnostics) Error() error { for _, d := range ds { if d.Severity == Error { - return fmt.Errorf(d.Summary) + message := d.Detail + if message == "" { + message = d.Summary + } + if d.ID != "" { + message = string(d.ID) + ": " + message + } + return errors.New(message) } } return nil diff --git a/libs/diag/id.go b/libs/diag/id.go new file mode 100644 index 00000000..44af3c93 --- /dev/null +++ b/libs/diag/id.go @@ -0,0 +1,16 @@ +package diag + +type ID string + +// For select diagnostic messages we use IDs to identify them +// for support or tooling purposes. +// It is a non-goal to have an exhaustive list of IDs. +const ( + // We have many subtly different permission errors. + // These are numbered for easy reference and tooling support. + PathPermissionDenied ID = "EPERM1" + ResourcePermissionDenied ID = "EPERM2" + CannotChangePathPermissions ID = "EPERM3" + RunAsDenied ID = "EPERM4" + PermissionNotIncluded ID = "EPERM5" +) diff --git a/libs/filer/filer.go b/libs/filer/filer.go index c1c747c5..fcfbcea0 100644 --- a/libs/filer/filer.go +++ b/libs/filer/filer.go @@ -103,6 +103,18 @@ func (err CannotDeleteRootError) Is(other error) bool { return other == fs.ErrInvalid } +type PermissionError struct { + path string +} + +func (err PermissionError) Error() string { + return fmt.Sprintf("access denied: %s", err.path) +} + +func (err PermissionError) Is(other error) bool { + return other == fs.ErrPermission +} + // Filer is used to access files in a workspace. // It has implementations for accessing files in WSFS and in DBFS. type Filer interface { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index d8ab5a6b..4bb03aea 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -178,6 +178,9 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { + if errors.As(err, &aerr) && aerr.StatusCode == http.StatusForbidden { + return PermissionError{absPath} + } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -203,6 +206,11 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io return FileAlreadyExistsError{absPath} } + // This API returns StatusForbidden when you have read access but don't have write access to a file + if aerr.StatusCode == http.StatusForbidden { + return PermissionError{absPath} + } + return err } @@ -295,11 +303,11 @@ func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D return nil, err } - // This API returns a 404 if the specified path does not exist. + // NOTE: This API returns a 404 if the specified path does not exist, + // but can also do so if we don't have read access. if aerr.StatusCode == http.StatusNotFound { return nil, NoSuchDirectoryError{path.Dir(absPath)} } - return nil, err } diff --git a/libs/set/set.go b/libs/set/set.go index 4798ed09..4b6bc876 100644 --- a/libs/set/set.go +++ b/libs/set/set.go @@ -14,6 +14,11 @@ type Set[T any] struct { data map[string]T } +// Values returns a slice of the set's values +func (s *Set[T]) Values() []T { + return maps.Values(s.data) +} + // NewSetFromF initialise a new set with initial values and a hash function // to define uniqueness of value func NewSetFromF[T any](values []T, f hashFunc[T]) *Set[T] { @@ -69,6 +74,11 @@ func (s *Set[T]) Has(item T) bool { return ok } +// Size returns the number of elements in the set +func (s *Set[T]) Size() int { + return len(s.data) +} + // Returns an iterable slice of values from set func (s *Set[T]) Iter() []T { return maps.Values(s.data) From 3270afaff43e4ad7e7998a67060d9ce8539e7e98 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 10 Oct 2024 15:02:25 +0200 Subject: [PATCH 74/78] Move utility functions dealing with IAM to libs/iamutil (#1820) ## Changes The two functions `GetShortUserName` and `IsServicePrincipal` are unrelated to auth or the purpose of the auth package. This change moves them into their own package and updates `IsServicePrincipal` to take an `*iam.User` argument instead of a string username. ## Tests Tests pass. --- bundle/config/mutator/populate_current_user.go | 4 ++-- bundle/config/mutator/process_target_mode.go | 4 ++-- bundle/permissions/permission_report.go | 13 +++++++------ internal/init_test.go | 6 +++--- libs/{auth => iamutil}/service_principal.go | 9 +++++---- libs/{auth => iamutil}/service_principal_test.go | 15 ++++++++++----- libs/{auth => iamutil}/user.go | 4 ++-- libs/{auth => iamutil}/user_test.go | 2 +- libs/template/helpers.go | 6 +++--- 9 files changed, 35 insertions(+), 28 deletions(-) rename libs/{auth => iamutil}/service_principal.go (61%) rename libs/{auth => iamutil}/service_principal_test.go (57%) rename libs/{auth => iamutil}/user.go (87%) rename libs/{auth => iamutil}/user_test.go (99%) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 1e99b327..cab5db1b 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -5,8 +5,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" ) @@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } b.Config.Workspace.CurrentUser = &config.User{ - ShortName: auth.GetShortUserName(me), + ShortName: iamutil.GetShortUserName(me), User: me, } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 9944f6ff..44b53681 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -6,9 +6,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/log" ) @@ -174,7 +174,7 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di transformDevelopmentMode(ctx, b) return diags case config.Production: - isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName) + isPrincipal := iamutil.IsServicePrincipal(b.Config.Workspace.CurrentUser.User) return validateProductionMode(ctx, b, isPrincipal) case "": // No action diff --git a/bundle/permissions/permission_report.go b/bundle/permissions/permission_report.go index a9468edb..36526eee 100644 --- a/bundle/permissions/permission_report.go +++ b/bundle/permissions/permission_report.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/log" ) @@ -17,9 +17,10 @@ import ( func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) diag.Diagnostics { log.Errorf(ctx, "Failed to update, encountered possible permission error: %v", path) - user := b.Config.Workspace.CurrentUser.UserName - if auth.IsServicePrincipal(user) { - user = b.Config.Workspace.CurrentUser.DisplayName + me := b.Config.Workspace.CurrentUser.User + userName := me.UserName + if iamutil.IsServicePrincipal(me) { + userName = me.DisplayName } canManageBundle, assistance := analyzeBundlePermissions(b) @@ -30,7 +31,7 @@ func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path "%s\n"+ "They may need to redeploy the bundle to apply the new permissions.\n"+ "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", - path, user, assistance), + path, userName, assistance), Severity: diag.Error, ID: diag.PathPermissionDenied, }} @@ -44,7 +45,7 @@ func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path "%s\n"+ "They can redeploy the project to apply the latest set of permissions.\n"+ "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", - path, user, assistance), + path, userName, assistance), Severity: diag.Error, ID: diag.CannotChangePathPermissions, }} diff --git a/internal/init_test.go b/internal/init_test.go index d1a89f7b..a6241d62 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -12,7 +12,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -126,7 +126,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }{ { funcName: "{{short_name}}", - expected: auth.GetShortUserName(me), + expected: iamutil.GetShortUserName(me), }, { funcName: "{{user_name}}", @@ -138,7 +138,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }, { funcName: "{{is_service_principal}}", - expected: strconv.FormatBool(auth.IsServicePrincipal(me.UserName)), + expected: strconv.FormatBool(iamutil.IsServicePrincipal(me)), }, { funcName: "{{smallest_node_type}}", diff --git a/libs/auth/service_principal.go b/libs/iamutil/service_principal.go similarity index 61% rename from libs/auth/service_principal.go rename to libs/iamutil/service_principal.go index 5f1854e3..7b65f1f5 100644 --- a/libs/auth/service_principal.go +++ b/libs/iamutil/service_principal.go @@ -1,15 +1,16 @@ -package auth +package iamutil import ( + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/google/uuid" ) -// Determines whether a given user name is a service principal. +// Determines whether a given user is a service principal. // This function uses a heuristic: if the user name is a UUID, then we assume // it's a service principal. Unfortunately, the service principal listing API is too // slow for our purposes. And the "users" and "service principals get" APIs // only allow access by workspace admins. -func IsServicePrincipal(userName string) bool { - _, err := uuid.Parse(userName) +func IsServicePrincipal(user *iam.User) bool { + _, err := uuid.Parse(user.UserName) return err == nil } diff --git a/libs/auth/service_principal_test.go b/libs/iamutil/service_principal_test.go similarity index 57% rename from libs/auth/service_principal_test.go rename to libs/iamutil/service_principal_test.go index 95e8ab5c..07e07669 100644 --- a/libs/auth/service_principal_test.go +++ b/libs/iamutil/service_principal_test.go @@ -1,19 +1,24 @@ -package auth +package iamutil import ( "testing" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) func TestIsServicePrincipal_ValidUUID(t *testing.T) { - userId := "8b948b2e-d2b5-4b9e-8274-11b596f3b652" - isSP := IsServicePrincipal(userId) + user := &iam.User{ + UserName: "8b948b2e-d2b5-4b9e-8274-11b596f3b652", + } + isSP := IsServicePrincipal(user) assert.True(t, isSP, "Expected user ID to be recognized as a service principal") } func TestIsServicePrincipal_InvalidUUID(t *testing.T) { - userId := "invalid" - isSP := IsServicePrincipal(userId) + user := &iam.User{ + UserName: "invalid", + } + isSP := IsServicePrincipal(user) assert.False(t, isSP, "Expected user ID to not be recognized as a service principal") } diff --git a/libs/auth/user.go b/libs/iamutil/user.go similarity index 87% rename from libs/auth/user.go rename to libs/iamutil/user.go index c6aa974f..53704dab 100644 --- a/libs/auth/user.go +++ b/libs/iamutil/user.go @@ -1,4 +1,4 @@ -package auth +package iamutil import ( "strings" @@ -12,7 +12,7 @@ import ( // including dots, which are not supported in e.g. experiment names. func GetShortUserName(user *iam.User) string { name := user.UserName - if IsServicePrincipal(user.UserName) && user.DisplayName != "" { + if IsServicePrincipal(user) && user.DisplayName != "" { name = user.DisplayName } local, _, _ := strings.Cut(name, "@") diff --git a/libs/auth/user_test.go b/libs/iamutil/user_test.go similarity index 99% rename from libs/auth/user_test.go rename to libs/iamutil/user_test.go index 24b61464..8aa863e6 100644 --- a/libs/auth/user_test.go +++ b/libs/iamutil/user_test.go @@ -1,4 +1,4 @@ -package auth +package iamutil import ( "testing" diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 88c73cc4..d00d75ed 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -11,7 +11,7 @@ import ( "text/template" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/iam" @@ -119,7 +119,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return "", err } } - return auth.GetShortUserName(cachedUser), nil + return iamutil.GetShortUserName(cachedUser), nil }, // Get the default workspace catalog. If there is no default, or if // Unity Catalog is not enabled, return an empty string. @@ -151,7 +151,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return false, err } } - result := auth.IsServicePrincipal(cachedUser.UserName) + result := iamutil.IsServicePrincipal(cachedUser) cachedIsServicePrincipal = &result return result, nil }, From b9085de53388cffc12c325653f05eeb16c7dc526 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 10 Oct 2024 15:43:21 +0200 Subject: [PATCH 75/78] Remove unused `IS_OWNER` constant (#1823) ## Changes Leftover from #1386. ## Tests All tests pass (indicating it really wasn't used). --- bundle/permissions/mutator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/bundle/permissions/mutator.go b/bundle/permissions/mutator.go index 247ffcda..7787bc04 100644 --- a/bundle/permissions/mutator.go +++ b/bundle/permissions/mutator.go @@ -13,7 +13,6 @@ import ( const CAN_MANAGE = "CAN_MANAGE" const CAN_VIEW = "CAN_VIEW" const CAN_RUN = "CAN_RUN" -const IS_OWNER = "IS_OWNER" var allowedLevels = []string{CAN_MANAGE, CAN_VIEW, CAN_RUN} var levelsMap = map[string](map[string]string){ From 845d23ac21b7a945714150ef68d48e6629a8bc6f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 10 Oct 2024 16:10:16 +0200 Subject: [PATCH 76/78] Fixed typo in converting cluster permissions (#1826) ## Changes Fixed typo in converting cluster permissions --- bundle/deploy/terraform/tfdyn/convert_cluster.go | 2 +- bundle/deploy/terraform/tfdyn/convert_cluster_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index f25f09ea..18819c00 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -40,7 +40,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, // Configure permissions for this resource. if permissions := convertPermissionsResource(ctx, vin); permissions != nil { - permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key) + permissions.ClusterId = fmt.Sprintf("${databricks_cluster.%s.id}", key) out.Permissions["cluster_"+key] = permissions } diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go index e7d2542f..e6d2620c 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go @@ -81,7 +81,7 @@ func TestConvertCluster(t *testing.T) { // Assert equality on the permissions assert.Equal(t, &schema.ResourcePermissions{ - JobId: "${databricks_cluster.my_cluster.id}", + ClusterId: "${databricks_cluster.my_cluster.id}", AccessControl: []schema.ResourcePermissionsAccessControl{ { PermissionLevel: "CAN_RUN", From 08a0d083c3bafb6cd5f87f4d39f0863417c71e2b Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Fri, 11 Oct 2024 14:28:56 +0200 Subject: [PATCH 77/78] Ignore metastore permission error during template generation (#1819) ## Changes This extends the `{{default_catalog}}` helper in templates to ignore any `PERMISSION_DENIED` error. We're still reviewing when exactly this error occurs, but if it does, it should not break templates. We should fall back to assuming there's no default catalog (and no UC) instead. ## Testing I have not been able to reproduce this issue, but there is a customer report about "access denied to clusters that don't have unity catalog enabled" being returned on a non-UC workspace. The error code in this PR corresponds to that message. ## Next steps We'll work together with the UC team to review if this error even makes sense for this API. If that discussion leads to a behavior change in the API we can update the CLI code again. --- libs/template/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/template/helpers.go b/libs/template/helpers.go index d00d75ed..f25cbee4 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -128,8 +128,8 @@ func loadHelpers(ctx context.Context) template.FuncMap { metastore, err := w.Metastores.Current(ctx) if err != nil { var aerr *apierr.APIError - if errors.As(err, &aerr) && aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" { - // Workspace doesn't have a metastore assigned, ignore error + if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST") { + // Ignore: access denied or workspace doesn't have a metastore assigned empty_default := "" cachedCatalog = &empty_default return "", nil From f0e2981596488af1af3fbbdbd9572a41bfaae4e8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 11 Oct 2024 16:39:53 +0200 Subject: [PATCH 78/78] Added JSON input validation for CLI commands (#1771) ## Changes Added JSON input validation for CLI commands. Now when invalid JSON passed as a payload to CLI commands, CLI performs input normalisation and detects if there are any mismatches such as incorrect types, unknown fields and etc. This diagnostic information is printed in standard error output and does not block command execution, so the change is backward compatible. Fixes #1769 #1764 #1625 #1560 ## Tests Added unit tests ``` andrew.nester@HFW9Y94129 ~ % databricks jobs create --json '{"seeti}' Error: error decoding JSON at (inline):1:2: unexpected EOF andrew.nester@HFW9Y94129 ~ % databricks jobs create --json '{"seeti": true}' Warning: unknown field: seeti in (inline):1:9 Error: Job settings must be specified. ``` --------- Co-authored-by: Pieter Noordhuis --- .codegen/service.go.tmpl | 13 +- bundle/render/render_text_output.go | 66 +---- cmd/account/access-control/access-control.go | 12 +- cmd/account/budgets/budgets.go | 24 +- cmd/account/credentials/credentials.go | 12 +- .../csp-enablement-account.go | 12 +- .../custom-app-integration.go | 24 +- .../disable-legacy-features.go | 12 +- .../encryption-keys/encryption-keys.go | 12 +- .../esm-enablement-account.go | 12 +- cmd/account/groups/groups.go | 36 ++- .../ip-access-lists/ip-access-lists.go | 36 ++- cmd/account/log-delivery/log-delivery.go | 24 +- .../metastore-assignments.go | 24 +- cmd/account/metastores/metastores.go | 24 +- .../network-connectivity.go | 24 +- cmd/account/networks/networks.go | 12 +- .../personal-compute/personal-compute.go | 12 +- cmd/account/private-access/private-access.go | 24 +- .../published-app-integration.go | 24 +- .../service-principals/service-principals.go | 36 ++- .../storage-credentials.go | 24 +- cmd/account/storage/storage.go | 12 +- .../usage-dashboards/usage-dashboards.go | 12 +- cmd/account/users/users.go | 36 ++- cmd/account/vpc-endpoints/vpc-endpoints.go | 12 +- .../workspace-assignment.go | 12 +- cmd/account/workspaces/workspaces.go | 24 +- cmd/api/api.go | 6 +- cmd/workspace/alerts-legacy/alerts-legacy.go | 24 +- cmd/workspace/alerts/alerts.go | 24 +- cmd/workspace/apps/apps.go | 60 +++- .../artifact-allowlists.go | 12 +- .../automatic-cluster-update.go | 12 +- cmd/workspace/catalogs/catalogs.go | 24 +- cmd/workspace/clean-rooms/clean-rooms.go | 24 +- .../cluster-policies/cluster-policies.go | 60 +++- cmd/workspace/clusters/clusters.go | 168 ++++++++--- .../compliance-security-profile.go | 12 +- cmd/workspace/connections/connections.go | 24 +- .../consumer-installations.go | 24 +- .../consumer-personalization-requests.go | 12 +- .../credentials-manager.go | 12 +- .../dashboard-widgets/dashboard-widgets.go | 24 +- cmd/workspace/dashboards/dashboards.go | 24 +- .../default-namespace/default-namespace.go | 12 +- .../disable-legacy-access.go | 12 +- .../enhanced-security-monitoring.go | 12 +- cmd/workspace/experiments/experiments.go | 264 +++++++++++++----- .../external-locations/external-locations.go | 24 +- cmd/workspace/functions/functions.go | 24 +- cmd/workspace/genie/genie.go | 24 +- .../git-credentials/git-credentials.go | 24 +- .../global-init-scripts.go | 24 +- cmd/workspace/grants/grants.go | 12 +- cmd/workspace/groups/groups.go | 36 ++- .../instance-pools/instance-pools.go | 60 +++- .../instance-profiles/instance-profiles.go | 36 ++- .../ip-access-lists/ip-access-lists.go | 36 ++- cmd/workspace/jobs/jobs.go | 144 +++++++--- cmd/workspace/lakeview/lakeview.go | 84 ++++-- cmd/workspace/libraries/libraries.go | 24 +- cmd/workspace/metastores/metastores.go | 48 +++- .../model-registry/model-registry.go | 228 +++++++++++---- .../model-versions/model-versions.go | 12 +- .../notification-destinations.go | 24 +- cmd/workspace/online-tables/online-tables.go | 12 +- .../permission-migration.go | 12 +- cmd/workspace/permissions/permissions.go | 24 +- cmd/workspace/pipelines/pipelines.go | 60 +++- .../policy-compliance-for-clusters.go | 12 +- .../policy-compliance-for-jobs.go | 12 +- .../provider-exchange-filters.go | 24 +- .../provider-exchanges/provider-exchanges.go | 36 ++- .../provider-files/provider-files.go | 24 +- .../provider-listings/provider-listings.go | 24 +- .../provider-personalization-requests.go | 12 +- .../provider-provider-analytics-dashboards.go | 12 +- .../provider-providers/provider-providers.go | 24 +- cmd/workspace/providers/providers.go | 24 +- .../quality-monitors/quality-monitors.go | 36 ++- .../queries-legacy/queries-legacy.go | 24 +- cmd/workspace/queries/queries.go | 24 +- .../query-visualizations-legacy.go | 24 +- .../query-visualizations.go | 24 +- cmd/workspace/recipients/recipients.go | 36 ++- .../registered-models/registered-models.go | 36 ++- cmd/workspace/repos/overrides.go | 26 +- cmd/workspace/repos/repos.go | 48 +++- .../restrict-workspace-admins.go | 12 +- cmd/workspace/schemas/schemas.go | 24 +- cmd/workspace/secrets/put_secret.go | 14 +- cmd/workspace/secrets/secrets.go | 60 +++- .../service-principals/service-principals.go | 36 ++- .../serving-endpoints/serving-endpoints.go | 96 +++++-- cmd/workspace/shares/shares.go | 36 ++- .../storage-credentials.go | 36 ++- .../table-constraints/table-constraints.go | 12 +- cmd/workspace/tables/tables.go | 12 +- .../temporary-table-credentials.go | 12 +- .../token-management/token-management.go | 36 ++- cmd/workspace/tokens/tokens.go | 24 +- cmd/workspace/users/users.go | 60 +++- .../vector-search-endpoints.go | 12 +- .../vector-search-indexes.go | 72 +++-- cmd/workspace/volumes/volumes.go | 24 +- cmd/workspace/warehouses/warehouses.go | 60 +++- .../workspace-bindings/workspace-bindings.go | 24 +- .../workspace-conf/workspace-conf.go | 12 +- cmd/workspace/workspace/workspace.go | 60 +++- libs/cmdio/render.go | 201 +++++++++---- libs/dyn/convert/to_typed.go | 17 +- libs/dyn/convert/to_typed_test.go | 17 +- libs/dyn/jsonloader/json.go | 107 +++++++ libs/dyn/jsonloader/json_test.go | 93 ++++++ libs/dyn/jsonloader/locations.go | 53 ++++ libs/flags/json_flag.go | 47 +++- libs/flags/json_flag_test.go | 244 +++++++++++++++- 118 files changed, 3378 insertions(+), 1042 deletions(-) create mode 100644 libs/dyn/jsonloader/json.go create mode 100644 libs/dyn/jsonloader/json_test.go create mode 100644 libs/dyn/jsonloader/locations.go diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index 281dfd6e..b489a0b0 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -5,6 +5,7 @@ package {{(.TrimPrefix "account").SnakeName}} import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/cmd/root" "github.com/databricks/databricks-sdk-go/service/{{.Package.Name}}" "github.com/spf13/cobra" @@ -231,9 +232,15 @@ func new{{.PascalName}}() *cobra.Command { {{- if .Request }} {{ if .CanUseJson }} if cmd.Flags().Changed("json") { - err = {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req) - if err != nil { - return err + diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } }{{end}}{{ if .MustUseJson }}else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 56387c38..3e52d5f1 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -8,6 +8,7 @@ import ( "text/template" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/fatih/color" @@ -28,48 +29,6 @@ var renderFuncMap = template.FuncMap{ }, } -const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} -{{- range $index, $element := .Paths }} - {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} -{{- end }} -{{- range $index, $element := .Locations }} - {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} -{{- end }} -{{- if .Detail }} - -{{ .Detail }} -{{- end }} - -` - -const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} -{{- range $index, $element := .Paths }} - {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} -{{- end }} -{{- range $index, $element := .Locations }} - {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} -{{- end }} -{{- if .Detail }} - -{{ .Detail }} -{{- end }} - -` - -const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }} -{{- range $index, $element := .Paths }} - {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} -{{- end }} -{{- range $index, $element := .Locations }} - {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} -{{- end }} -{{- if .Detail }} - -{{ .Detail }} -{{- end }} - -` - const summaryTemplate = `{{- if .Name -}} Name: {{ .Name | bold }} {{- if .Target }} @@ -153,22 +112,7 @@ func renderSummaryTemplate(out io.Writer, b *bundle.Bundle, diags diag.Diagnosti } func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { - errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) - warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) - recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate)) - - // Print errors and warnings. for _, d := range diags { - var t *template.Template - switch d.Severity { - case diag.Error: - t = errorT - case diag.Warning: - t = warningT - case diag.Recommendation: - t = recommendationT - } - for i := range d.Locations { if b == nil { break @@ -183,15 +127,9 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) } } } - - // Render the diagnostic with the appropriate template. - err := t.Execute(out, d) - if err != nil { - return fmt.Errorf("failed to render template: %w", err) - } } - return nil + return cmdio.RenderDiagnostics(out, diags) } // RenderOptions contains options for rendering diagnostics. diff --git a/cmd/account/access-control/access-control.go b/cmd/account/access-control/access-control.go index f6761a1b..07a7fce9 100755 --- a/cmd/account/access-control/access-control.go +++ b/cmd/account/access-control/access-control.go @@ -205,9 +205,15 @@ func newUpdateRuleSet() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateRuleSetJson.Unmarshal(&updateRuleSetReq) - if err != nil { - return err + diags := updateRuleSetJson.Unmarshal(&updateRuleSetReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/budgets/budgets.go b/cmd/account/budgets/budgets.go index 6b47bb32..87ed41d5 100755 --- a/cmd/account/budgets/budgets.go +++ b/cmd/account/budgets/budgets.go @@ -78,9 +78,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -316,9 +322,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/credentials/credentials.go b/cmd/account/credentials/credentials.go index ed071cda..4ab70ef0 100755 --- a/cmd/account/credentials/credentials.go +++ b/cmd/account/credentials/credentials.go @@ -90,9 +90,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/csp-enablement-account/csp-enablement-account.go b/cmd/account/csp-enablement-account/csp-enablement-account.go index d6fce953..9fb94a61 100755 --- a/cmd/account/csp-enablement-account/csp-enablement-account.go +++ b/cmd/account/csp-enablement-account/csp-enablement-account.go @@ -129,9 +129,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index 5cdf422d..9d16a44d 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -88,9 +88,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -320,9 +326,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.IntegrationId = args[0] diff --git a/cmd/account/disable-legacy-features/disable-legacy-features.go b/cmd/account/disable-legacy-features/disable-legacy-features.go index 6d25b943..5e732b87 100755 --- a/cmd/account/disable-legacy-features/disable-legacy-features.go +++ b/cmd/account/disable-legacy-features/disable-legacy-features.go @@ -185,9 +185,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/encryption-keys/encryption-keys.go b/cmd/account/encryption-keys/encryption-keys.go index 44545ccf..3f7082cd 100755 --- a/cmd/account/encryption-keys/encryption-keys.go +++ b/cmd/account/encryption-keys/encryption-keys.go @@ -107,9 +107,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go index 71149e5a..1f820e3a 100755 --- a/cmd/account/esm-enablement-account/esm-enablement-account.go +++ b/cmd/account/esm-enablement-account/esm-enablement-account.go @@ -127,9 +127,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/groups/groups.go b/cmd/account/groups/groups.go index a7e1ac43..7c2e8331 100755 --- a/cmd/account/groups/groups.go +++ b/cmd/account/groups/groups.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -446,9 +458,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/ip-access-lists/ip-access-lists.go b/cmd/account/ip-access-lists/ip-access-lists.go index 5c6d27dd..61fa944a 100755 --- a/cmd/account/ip-access-lists/ip-access-lists.go +++ b/cmd/account/ip-access-lists/ip-access-lists.go @@ -132,9 +132,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -411,9 +417,15 @@ func newReplace() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.IpAccessListId = args[0] @@ -505,9 +517,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/log-delivery/log-delivery.go b/cmd/account/log-delivery/log-delivery.go index 4584f4d2..cb083f3b 100755 --- a/cmd/account/log-delivery/log-delivery.go +++ b/cmd/account/log-delivery/log-delivery.go @@ -162,9 +162,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -369,9 +375,15 @@ func newPatchStatus() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchStatusJson.Unmarshal(&patchStatusReq) - if err != nil { - return err + diags := patchStatusJson.Unmarshal(&patchStatusReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } patchStatusReq.LogDeliveryConfigurationId = args[0] diff --git a/cmd/account/metastore-assignments/metastore-assignments.go b/cmd/account/metastore-assignments/metastore-assignments.go index d7f32ccb..9d249075 100755 --- a/cmd/account/metastore-assignments/metastore-assignments.go +++ b/cmd/account/metastore-assignments/metastore-assignments.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &createReq.WorkspaceId) @@ -343,9 +349,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) diff --git a/cmd/account/metastores/metastores.go b/cmd/account/metastores/metastores.go index 7c8e3f2c..eb575578 100755 --- a/cmd/account/metastores/metastores.go +++ b/cmd/account/metastores/metastores.go @@ -80,9 +80,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -304,9 +310,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.MetastoreId = args[0] diff --git a/cmd/account/network-connectivity/network-connectivity.go b/cmd/account/network-connectivity/network-connectivity.go index cd8da290..168de9c1 100755 --- a/cmd/account/network-connectivity/network-connectivity.go +++ b/cmd/account/network-connectivity/network-connectivity.go @@ -96,9 +96,15 @@ func newCreateNetworkConnectivityConfiguration() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createNetworkConnectivityConfigurationJson.Unmarshal(&createNetworkConnectivityConfigurationReq) - if err != nil { - return err + diags := createNetworkConnectivityConfigurationJson.Unmarshal(&createNetworkConnectivityConfigurationReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -187,9 +193,15 @@ func newCreatePrivateEndpointRule() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createPrivateEndpointRuleJson.Unmarshal(&createPrivateEndpointRuleReq) - if err != nil { - return err + diags := createPrivateEndpointRuleJson.Unmarshal(&createPrivateEndpointRuleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createPrivateEndpointRuleReq.NetworkConnectivityConfigId = args[0] diff --git a/cmd/account/networks/networks.go b/cmd/account/networks/networks.go index 05ef0c81..086d24be 100755 --- a/cmd/account/networks/networks.go +++ b/cmd/account/networks/networks.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/account/personal-compute/personal-compute.go b/cmd/account/personal-compute/personal-compute.go index 2a14b0b3..dac7e2e3 100755 --- a/cmd/account/personal-compute/personal-compute.go +++ b/cmd/account/personal-compute/personal-compute.go @@ -189,9 +189,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/private-access/private-access.go b/cmd/account/private-access/private-access.go index d527fa64..312f6d02 100755 --- a/cmd/account/private-access/private-access.go +++ b/cmd/account/private-access/private-access.go @@ -109,9 +109,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -411,9 +417,15 @@ func newReplace() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.PrivateAccessSettingsId = args[0] diff --git a/cmd/account/published-app-integration/published-app-integration.go b/cmd/account/published-app-integration/published-app-integration.go index 5143d53c..c97bcfc6 100755 --- a/cmd/account/published-app-integration/published-app-integration.go +++ b/cmd/account/published-app-integration/published-app-integration.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -315,9 +321,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.IntegrationId = args[0] diff --git a/cmd/account/service-principals/service-principals.go b/cmd/account/service-principals/service-principals.go index c86810f1..f6ec1fa5 100755 --- a/cmd/account/service-principals/service-principals.go +++ b/cmd/account/service-principals/service-principals.go @@ -95,9 +95,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -448,9 +460,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/storage-credentials/storage-credentials.go b/cmd/account/storage-credentials/storage-credentials.go index 4280ae8c..b3b8ad36 100755 --- a/cmd/account/storage-credentials/storage-credentials.go +++ b/cmd/account/storage-credentials/storage-credentials.go @@ -88,9 +88,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.MetastoreId = args[0] @@ -340,9 +346,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.MetastoreId = args[0] diff --git a/cmd/account/storage/storage.go b/cmd/account/storage/storage.go index 50460ed0..24bb6d80 100755 --- a/cmd/account/storage/storage.go +++ b/cmd/account/storage/storage.go @@ -87,9 +87,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/usage-dashboards/usage-dashboards.go b/cmd/account/usage-dashboards/usage-dashboards.go index 8a1c3247..a482b466 100755 --- a/cmd/account/usage-dashboards/usage-dashboards.go +++ b/cmd/account/usage-dashboards/usage-dashboards.go @@ -80,9 +80,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/account/users/users.go b/cmd/account/users/users.go index 289d2972..10990084 100755 --- a/cmd/account/users/users.go +++ b/cmd/account/users/users.go @@ -103,9 +103,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -374,9 +380,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -465,9 +477,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/vpc-endpoints/vpc-endpoints.go b/cmd/account/vpc-endpoints/vpc-endpoints.go index e6c6c126..c1aab0d2 100755 --- a/cmd/account/vpc-endpoints/vpc-endpoints.go +++ b/cmd/account/vpc-endpoints/vpc-endpoints.go @@ -104,9 +104,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/account/workspace-assignment/workspace-assignment.go b/cmd/account/workspace-assignment/workspace-assignment.go index 58468d09..e09095d3 100755 --- a/cmd/account/workspace-assignment/workspace-assignment.go +++ b/cmd/account/workspace-assignment/workspace-assignment.go @@ -273,9 +273,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) diff --git a/cmd/account/workspaces/workspaces.go b/cmd/account/workspaces/workspaces.go index 1ec6230b..82d3d7db 100755 --- a/cmd/account/workspaces/workspaces.go +++ b/cmd/account/workspaces/workspaces.go @@ -133,9 +133,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -551,9 +557,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/api/api.go b/cmd/api/api.go index 03460f71..d33939a5 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -42,9 +42,9 @@ func makeCommand(method string) *cobra.Command { var path = args[0] var request any - err := payload.Unmarshal(&request) - if err != nil { - return err + diags := payload.Unmarshal(&request) + if diags.HasError() { + return diags.Error() } cfg := &config.Config{} diff --git a/cmd/workspace/alerts-legacy/alerts-legacy.go b/cmd/workspace/alerts-legacy/alerts-legacy.go index 1046b112..4f7d5f96 100755 --- a/cmd/workspace/alerts-legacy/alerts-legacy.go +++ b/cmd/workspace/alerts-legacy/alerts-legacy.go @@ -93,9 +93,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -357,9 +363,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go index cfaa3f55..fcf18652 100755 --- a/cmd/workspace/alerts/alerts.go +++ b/cmd/workspace/alerts/alerts.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -354,9 +360,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index 780f5594..4cee2f82 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -113,9 +113,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -267,9 +273,15 @@ func newDeploy() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deployJson.Unmarshal(&deployReq) - if err != nil { - return err + diags := deployJson.Unmarshal(&deployReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } deployReq.AppName = args[0] @@ -702,9 +714,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.AppName = args[0] @@ -936,9 +954,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -1005,9 +1029,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.AppName = args[0] diff --git a/cmd/workspace/artifact-allowlists/artifact-allowlists.go b/cmd/workspace/artifact-allowlists/artifact-allowlists.go index fc25e3cb..e6683369 100755 --- a/cmd/workspace/artifact-allowlists/artifact-allowlists.go +++ b/cmd/workspace/artifact-allowlists/artifact-allowlists.go @@ -145,9 +145,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go index 2385195b..dca88f3d 100755 --- a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go +++ b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go @@ -127,9 +127,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/catalogs/catalogs.go b/cmd/workspace/catalogs/catalogs.go index a17bb007..9294c192 100755 --- a/cmd/workspace/catalogs/catalogs.go +++ b/cmd/workspace/catalogs/catalogs.go @@ -105,9 +105,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -363,9 +369,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/clean-rooms/clean-rooms.go b/cmd/workspace/clean-rooms/clean-rooms.go index 9466c4b9..72560b84 100755 --- a/cmd/workspace/clean-rooms/clean-rooms.go +++ b/cmd/workspace/clean-rooms/clean-rooms.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -344,9 +350,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/cluster-policies/cluster-policies.go b/cmd/workspace/cluster-policies/cluster-policies.go index 830d44ca..b34dd53d 100755 --- a/cmd/workspace/cluster-policies/cluster-policies.go +++ b/cmd/workspace/cluster-policies/cluster-policies.go @@ -113,9 +113,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -185,9 +191,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -284,9 +296,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -630,9 +648,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -711,9 +735,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index b36102d9..0ed454de 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -134,9 +134,15 @@ func newChangeOwner() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = changeOwnerJson.Unmarshal(&changeOwnerReq) - if err != nil { - return err + diags := changeOwnerJson.Unmarshal(&changeOwnerReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -268,9 +274,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -362,9 +374,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -519,9 +537,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -617,9 +641,15 @@ func newEvents() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = eventsJson.Unmarshal(&eventsReq) - if err != nil { - return err + diags := eventsJson.Unmarshal(&eventsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1069,9 +1099,15 @@ func newPermanentDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = permanentDeleteJson.Unmarshal(&permanentDeleteReq) - if err != nil { - return err + diags := permanentDeleteJson.Unmarshal(&permanentDeleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1161,9 +1197,15 @@ func newPin() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = pinJson.Unmarshal(&pinReq) - if err != nil { - return err + diags := pinJson.Unmarshal(&pinReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1260,9 +1302,15 @@ func newResize() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = resizeJson.Unmarshal(&resizeReq) - if err != nil { - return err + diags := resizeJson.Unmarshal(&resizeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1370,9 +1418,15 @@ func newRestart() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restartJson.Unmarshal(&restartReq) - if err != nil { - return err + diags := restartJson.Unmarshal(&restartReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1464,9 +1518,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1608,9 +1668,15 @@ func newStart() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startJson.Unmarshal(&startReq) - if err != nil { - return err + diags := startJson.Unmarshal(&startReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1712,9 +1778,15 @@ func newUnpin() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = unpinJson.Unmarshal(&unpinReq) - if err != nil { - return err + diags := unpinJson.Unmarshal(&unpinReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1824,9 +1896,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1905,9 +1983,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/compliance-security-profile/compliance-security-profile.go b/cmd/workspace/compliance-security-profile/compliance-security-profile.go index a7b45901..58f3edda 100755 --- a/cmd/workspace/compliance-security-profile/compliance-security-profile.go +++ b/cmd/workspace/compliance-security-profile/compliance-security-profile.go @@ -130,9 +130,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go index f76420fb..161f5ab4 100755 --- a/cmd/workspace/connections/connections.go +++ b/cmd/workspace/connections/connections.go @@ -92,9 +92,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -355,9 +361,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go index 92f61789..1848cb8f 100755 --- a/cmd/workspace/consumer-installations/consumer-installations.go +++ b/cmd/workspace/consumer-installations/consumer-installations.go @@ -86,9 +86,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.ListingId = args[0] @@ -319,9 +325,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go index 8b0af3cc..6d751c63 100755 --- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go +++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/credentials-manager/credentials-manager.go b/cmd/workspace/credentials-manager/credentials-manager.go index 5a40232b..e29bc0bd 100755 --- a/cmd/workspace/credentials-manager/credentials-manager.go +++ b/cmd/workspace/credentials-manager/credentials-manager.go @@ -75,9 +75,15 @@ func newExchangeToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = exchangeTokenJson.Unmarshal(&exchangeTokenReq) - if err != nil { - return err + diags := exchangeTokenJson.Unmarshal(&exchangeTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/dashboard-widgets/dashboard-widgets.go b/cmd/workspace/dashboard-widgets/dashboard-widgets.go index 02b13739..e4281826 100755 --- a/cmd/workspace/dashboard-widgets/dashboard-widgets.go +++ b/cmd/workspace/dashboard-widgets/dashboard-widgets.go @@ -75,9 +75,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -196,9 +202,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/dashboards/dashboards.go b/cmd/workspace/dashboards/dashboards.go index fcab0aa2..5bdd7b13 100755 --- a/cmd/workspace/dashboards/dashboards.go +++ b/cmd/workspace/dashboards/dashboards.go @@ -78,9 +78,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -405,9 +411,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/default-namespace/default-namespace.go b/cmd/workspace/default-namespace/default-namespace.go index b15907be..e5039cba 100755 --- a/cmd/workspace/default-namespace/default-namespace.go +++ b/cmd/workspace/default-namespace/default-namespace.go @@ -199,9 +199,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/disable-legacy-access/disable-legacy-access.go b/cmd/workspace/disable-legacy-access/disable-legacy-access.go index fea2b3c4..c50de446 100755 --- a/cmd/workspace/disable-legacy-access/disable-legacy-access.go +++ b/cmd/workspace/disable-legacy-access/disable-legacy-access.go @@ -187,9 +187,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go index a8acc5cd..3d99ecef 100755 --- a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go +++ b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go @@ -132,9 +132,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index b1af2f86..4c6b57d1 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -130,9 +130,15 @@ func newCreateExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createExperimentJson.Unmarshal(&createExperimentReq) - if err != nil { - return err + diags := createExperimentJson.Unmarshal(&createExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -203,9 +209,15 @@ func newCreateRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createRunJson.Unmarshal(&createRunReq) - if err != nil { - return err + diags := createRunJson.Unmarshal(&createRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -277,9 +289,15 @@ func newDeleteExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteExperimentJson.Unmarshal(&deleteExperimentReq) - if err != nil { - return err + diags := deleteExperimentJson.Unmarshal(&deleteExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -352,9 +370,15 @@ func newDeleteRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunJson.Unmarshal(&deleteRunReq) - if err != nil { - return err + diags := deleteRunJson.Unmarshal(&deleteRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -435,9 +459,15 @@ func newDeleteRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunsJson.Unmarshal(&deleteRunsReq) - if err != nil { - return err + diags := deleteRunsJson.Unmarshal(&deleteRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -518,9 +548,15 @@ func newDeleteTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteTagJson.Unmarshal(&deleteTagReq) - if err != nil { - return err + diags := deleteTagJson.Unmarshal(&deleteTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1108,9 +1144,15 @@ func newLogBatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logBatchJson.Unmarshal(&logBatchReq) - if err != nil { - return err + diags := logBatchJson.Unmarshal(&logBatchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1174,9 +1216,15 @@ func newLogInputs() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logInputsJson.Unmarshal(&logInputsReq) - if err != nil { - return err + diags := logInputsJson.Unmarshal(&logInputsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1254,9 +1302,15 @@ func newLogMetric() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logMetricJson.Unmarshal(&logMetricReq) - if err != nil { - return err + diags := logMetricJson.Unmarshal(&logMetricReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1335,9 +1389,15 @@ func newLogModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logModelJson.Unmarshal(&logModelReq) - if err != nil { - return err + diags := logModelJson.Unmarshal(&logModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1414,9 +1474,15 @@ func newLogParam() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logParamJson.Unmarshal(&logParamReq) - if err != nil { - return err + diags := logParamJson.Unmarshal(&logParamReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1497,9 +1563,15 @@ func newRestoreExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreExperimentJson.Unmarshal(&restoreExperimentReq) - if err != nil { - return err + diags := restoreExperimentJson.Unmarshal(&restoreExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1572,9 +1644,15 @@ func newRestoreRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreRunJson.Unmarshal(&restoreRunReq) - if err != nil { - return err + diags := restoreRunJson.Unmarshal(&restoreRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1655,9 +1733,15 @@ func newRestoreRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreRunsJson.Unmarshal(&restoreRunsReq) - if err != nil { - return err + diags := restoreRunsJson.Unmarshal(&restoreRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1732,9 +1816,15 @@ func newSearchExperiments() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = searchExperimentsJson.Unmarshal(&searchExperimentsReq) - if err != nil { - return err + diags := searchExperimentsJson.Unmarshal(&searchExperimentsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1800,9 +1890,15 @@ func newSearchRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = searchRunsJson.Unmarshal(&searchRunsReq) - if err != nil { - return err + diags := searchRunsJson.Unmarshal(&searchRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1874,9 +1970,15 @@ func newSetExperimentTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setExperimentTagJson.Unmarshal(&setExperimentTagReq) - if err != nil { - return err + diags := setExperimentTagJson.Unmarshal(&setExperimentTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1951,9 +2053,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.ExperimentId = args[0] @@ -2032,9 +2140,15 @@ func newSetTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setTagJson.Unmarshal(&setTagReq) - if err != nil { - return err + diags := setTagJson.Unmarshal(&setTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2112,9 +2226,15 @@ func newUpdateExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateExperimentJson.Unmarshal(&updateExperimentReq) - if err != nil { - return err + diags := updateExperimentJson.Unmarshal(&updateExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2183,9 +2303,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.ExperimentId = args[0] @@ -2251,9 +2377,15 @@ func newUpdateRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateRunJson.Unmarshal(&updateRunReq) - if err != nil { - return err + diags := updateRunJson.Unmarshal(&updateRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/external-locations/external-locations.go b/cmd/workspace/external-locations/external-locations.go index 42493fc4..97d34df0 100755 --- a/cmd/workspace/external-locations/external-locations.go +++ b/cmd/workspace/external-locations/external-locations.go @@ -112,9 +112,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -381,9 +387,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/functions/functions.go b/cmd/workspace/functions/functions.go index c8de4879..86b29267 100755 --- a/cmd/workspace/functions/functions.go +++ b/cmd/workspace/functions/functions.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -381,9 +387,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index e4a05909..287bcde6 100755 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -105,9 +105,15 @@ func newCreateMessage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createMessageJson.Unmarshal(&createMessageReq) - if err != nil { - return err + diags := createMessageJson.Unmarshal(&createMessageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createMessageReq.SpaceId = args[0] @@ -392,9 +398,15 @@ func newStartConversation() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startConversationJson.Unmarshal(&startConversationReq) - if err != nil { - return err + diags := startConversationJson.Unmarshal(&startConversationReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } startConversationReq.SpaceId = args[0] diff --git a/cmd/workspace/git-credentials/git-credentials.go b/cmd/workspace/git-credentials/git-credentials.go index b5082d31..978ca6ba 100755 --- a/cmd/workspace/git-credentials/git-credentials.go +++ b/cmd/workspace/git-credentials/git-credentials.go @@ -103,9 +103,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -371,9 +377,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.CredentialId) diff --git a/cmd/workspace/global-init-scripts/global-init-scripts.go b/cmd/workspace/global-init-scripts/global-init-scripts.go index 92dcb259..52adde3f 100755 --- a/cmd/workspace/global-init-scripts/global-init-scripts.go +++ b/cmd/workspace/global-init-scripts/global-init-scripts.go @@ -101,9 +101,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -367,9 +373,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.ScriptId = args[0] diff --git a/cmd/workspace/grants/grants.go b/cmd/workspace/grants/grants.go index 876f0343..b1dd4e8c 100755 --- a/cmd/workspace/grants/grants.go +++ b/cmd/workspace/grants/grants.go @@ -223,9 +223,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.SecurableType) diff --git a/cmd/workspace/groups/groups.go b/cmd/workspace/groups/groups.go index 14650d98..a2d32310 100755 --- a/cmd/workspace/groups/groups.go +++ b/cmd/workspace/groups/groups.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -446,9 +458,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/instance-pools/instance-pools.go b/cmd/workspace/instance-pools/instance-pools.go index db96f146..8a84df94 100755 --- a/cmd/workspace/instance-pools/instance-pools.go +++ b/cmd/workspace/instance-pools/instance-pools.go @@ -128,9 +128,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -206,9 +212,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -309,9 +321,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -631,9 +649,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -712,9 +736,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/instance-profiles/instance-profiles.go b/cmd/workspace/instance-profiles/instance-profiles.go index 7134c16c..5c4bc8d9 100755 --- a/cmd/workspace/instance-profiles/instance-profiles.go +++ b/cmd/workspace/instance-profiles/instance-profiles.go @@ -99,9 +99,15 @@ func newAdd() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = addJson.Unmarshal(&addReq) - if err != nil { - return err + diags := addJson.Unmarshal(&addReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -192,9 +198,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -311,9 +323,15 @@ func newRemove() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = removeJson.Unmarshal(&removeReq) - if err != nil { - return err + diags := removeJson.Unmarshal(&removeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/ip-access-lists/ip-access-lists.go b/cmd/workspace/ip-access-lists/ip-access-lists.go index ec5958b5..070f279e 100755 --- a/cmd/workspace/ip-access-lists/ip-access-lists.go +++ b/cmd/workspace/ip-access-lists/ip-access-lists.go @@ -133,9 +133,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -414,9 +420,15 @@ func newReplace() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.IpAccessListId = args[0] @@ -510,9 +522,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 2d422fa8..d4ceb0c2 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -116,9 +116,15 @@ func newCancelAllRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = cancelAllRunsJson.Unmarshal(&cancelAllRunsReq) - if err != nil { - return err + diags := cancelAllRunsJson.Unmarshal(&cancelAllRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -193,9 +199,15 @@ func newCancelRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = cancelRunJson.Unmarshal(&cancelRunReq) - if err != nil { - return err + diags := cancelRunJson.Unmarshal(&cancelRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -291,9 +303,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -364,9 +382,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -457,9 +481,15 @@ func newDeleteRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunJson.Unmarshal(&deleteRunReq) - if err != nil { - return err + diags := deleteRunJson.Unmarshal(&deleteRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1143,9 +1173,15 @@ func newRepairRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = repairRunJson.Unmarshal(&repairRunReq) - if err != nil { - return err + diags := repairRunJson.Unmarshal(&repairRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1242,9 +1278,15 @@ func newReset() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = resetJson.Unmarshal(&resetReq) - if err != nil { - return err + diags := resetJson.Unmarshal(&resetReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -1332,9 +1374,15 @@ func newRunNow() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = runNowJson.Unmarshal(&runNowReq) - if err != nil { - return err + diags := runNowJson.Unmarshal(&runNowReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1436,9 +1484,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1538,9 +1592,15 @@ func newSubmit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = submitJson.Unmarshal(&submitReq) - if err != nil { - return err + diags := submitJson.Unmarshal(&submitReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1632,9 +1692,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1717,9 +1783,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/lakeview/lakeview.go b/cmd/workspace/lakeview/lakeview.go index ef2d6845..33a45c65 100755 --- a/cmd/workspace/lakeview/lakeview.go +++ b/cmd/workspace/lakeview/lakeview.go @@ -108,9 +108,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -180,9 +186,15 @@ func newCreateSchedule() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createScheduleJson.Unmarshal(&createScheduleReq) - if err != nil { - return err + diags := createScheduleJson.Unmarshal(&createScheduleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -250,9 +262,15 @@ func newCreateSubscription() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createSubscriptionJson.Unmarshal(&createSubscriptionReq) - if err != nil { - return err + diags := createSubscriptionJson.Unmarshal(&createSubscriptionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -870,9 +888,15 @@ func newMigrate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = migrateJson.Unmarshal(&migrateReq) - if err != nil { - return err + diags := migrateJson.Unmarshal(&migrateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -941,9 +965,15 @@ func newPublish() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = publishJson.Unmarshal(&publishReq) - if err != nil { - return err + diags := publishJson.Unmarshal(&publishReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } publishReq.DashboardId = args[0] @@ -1128,9 +1158,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.DashboardId = args[0] @@ -1200,9 +1236,15 @@ func newUpdateSchedule() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateScheduleJson.Unmarshal(&updateScheduleReq) - if err != nil { - return err + diags := updateScheduleJson.Unmarshal(&updateScheduleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/libraries/libraries.go b/cmd/workspace/libraries/libraries.go index 2c10d816..e6b332ae 100755 --- a/cmd/workspace/libraries/libraries.go +++ b/cmd/workspace/libraries/libraries.go @@ -190,9 +190,15 @@ func newInstall() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = installJson.Unmarshal(&installReq) - if err != nil { - return err + diags := installJson.Unmarshal(&installReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -251,9 +257,15 @@ func newUninstall() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = uninstallJson.Unmarshal(&uninstallReq) - if err != nil { - return err + diags := uninstallJson.Unmarshal(&uninstallReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/metastores/metastores.go b/cmd/workspace/metastores/metastores.go index 22bcd3dc..563beb2f 100755 --- a/cmd/workspace/metastores/metastores.go +++ b/cmd/workspace/metastores/metastores.go @@ -112,9 +112,15 @@ func newAssign() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = assignJson.Unmarshal(&assignReq) - if err != nil { - return err + diags := assignJson.Unmarshal(&assignReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &assignReq.WorkspaceId) @@ -201,9 +207,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -606,9 +618,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -690,9 +708,15 @@ func newUpdateAssignment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateAssignmentJson.Unmarshal(&updateAssignmentReq) - if err != nil { - return err + diags := updateAssignmentJson.Unmarshal(&updateAssignmentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/model-registry/model-registry.go b/cmd/workspace/model-registry/model-registry.go index 41f06ac4..b45d83e3 100755 --- a/cmd/workspace/model-registry/model-registry.go +++ b/cmd/workspace/model-registry/model-registry.go @@ -141,9 +141,15 @@ func newApproveTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = approveTransitionRequestJson.Unmarshal(&approveTransitionRequestReq) - if err != nil { - return err + diags := approveTransitionRequestJson.Unmarshal(&approveTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -235,9 +241,15 @@ func newCreateComment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createCommentJson.Unmarshal(&createCommentReq) - if err != nil { - return err + diags := createCommentJson.Unmarshal(&createCommentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -322,9 +334,15 @@ func newCreateModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createModelJson.Unmarshal(&createModelReq) - if err != nil { - return err + diags := createModelJson.Unmarshal(&createModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -403,9 +421,15 @@ func newCreateModelVersion() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createModelVersionJson.Unmarshal(&createModelVersionReq) - if err != nil { - return err + diags := createModelVersionJson.Unmarshal(&createModelVersionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -493,9 +517,15 @@ func newCreateTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createTransitionRequestJson.Unmarshal(&createTransitionRequestReq) - if err != nil { - return err + diags := createTransitionRequestJson.Unmarshal(&createTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -570,9 +600,15 @@ func newCreateWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createWebhookJson.Unmarshal(&createWebhookReq) - if err != nil { - return err + diags := createWebhookJson.Unmarshal(&createWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -1079,9 +1115,15 @@ func newGetLatestVersions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = getLatestVersionsJson.Unmarshal(&getLatestVersionsReq) - if err != nil { - return err + diags := getLatestVersionsJson.Unmarshal(&getLatestVersionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1629,9 +1671,15 @@ func newRejectTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = rejectTransitionRequestJson.Unmarshal(&rejectTransitionRequestReq) - if err != nil { - return err + diags := rejectTransitionRequestJson.Unmarshal(&rejectTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1715,9 +1763,15 @@ func newRenameModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = renameModelJson.Unmarshal(&renameModelReq) - if err != nil { - return err + diags := renameModelJson.Unmarshal(&renameModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1907,9 +1961,15 @@ func newSetModelTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setModelTagJson.Unmarshal(&setModelTagReq) - if err != nil { - return err + diags := setModelTagJson.Unmarshal(&setModelTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1996,9 +2056,15 @@ func newSetModelVersionTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setModelVersionTagJson.Unmarshal(&setModelVersionTagReq) - if err != nil { - return err + diags := setModelVersionTagJson.Unmarshal(&setModelVersionTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2076,9 +2142,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.RegisteredModelId = args[0] @@ -2166,9 +2238,15 @@ func newTestRegistryWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = testRegistryWebhookJson.Unmarshal(&testRegistryWebhookReq) - if err != nil { - return err + diags := testRegistryWebhookJson.Unmarshal(&testRegistryWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2259,9 +2337,15 @@ func newTransitionStage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = transitionStageJson.Unmarshal(&transitionStageReq) - if err != nil { - return err + diags := transitionStageJson.Unmarshal(&transitionStageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2350,9 +2434,15 @@ func newUpdateComment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateCommentJson.Unmarshal(&updateCommentReq) - if err != nil { - return err + diags := updateCommentJson.Unmarshal(&updateCommentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2430,9 +2520,15 @@ func newUpdateModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateModelJson.Unmarshal(&updateModelReq) - if err != nil { - return err + diags := updateModelJson.Unmarshal(&updateModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2508,9 +2604,15 @@ func newUpdateModelVersion() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateModelVersionJson.Unmarshal(&updateModelVersionReq) - if err != nil { - return err + diags := updateModelVersionJson.Unmarshal(&updateModelVersionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2582,9 +2684,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.RegisteredModelId = args[0] @@ -2663,9 +2771,15 @@ func newUpdateWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateWebhookJson.Unmarshal(&updateWebhookReq) - if err != nil { - return err + diags := updateWebhookJson.Unmarshal(&updateWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go index d2f05404..439e5f65 100755 --- a/cmd/workspace/model-versions/model-versions.go +++ b/cmd/workspace/model-versions/model-versions.go @@ -377,9 +377,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.FullName = args[0] diff --git a/cmd/workspace/notification-destinations/notification-destinations.go b/cmd/workspace/notification-destinations/notification-destinations.go index 5ad47cc9..47076587 100755 --- a/cmd/workspace/notification-destinations/notification-destinations.go +++ b/cmd/workspace/notification-destinations/notification-destinations.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -313,9 +319,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/online-tables/online-tables.go b/cmd/workspace/online-tables/online-tables.go index da2f8c04..1c25d1e2 100755 --- a/cmd/workspace/online-tables/online-tables.go +++ b/cmd/workspace/online-tables/online-tables.go @@ -79,9 +79,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/permission-migration/permission-migration.go b/cmd/workspace/permission-migration/permission-migration.go index 2e50b123..15ff1b75 100755 --- a/cmd/workspace/permission-migration/permission-migration.go +++ b/cmd/workspace/permission-migration/permission-migration.go @@ -92,9 +92,15 @@ func newMigratePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = migratePermissionsJson.Unmarshal(&migratePermissionsReq) - if err != nil { - return err + diags := migratePermissionsJson.Unmarshal(&migratePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/permissions/permissions.go b/cmd/workspace/permissions/permissions.go index c6033e4a..d007a425 100755 --- a/cmd/workspace/permissions/permissions.go +++ b/cmd/workspace/permissions/permissions.go @@ -265,9 +265,15 @@ func newSet() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setJson.Unmarshal(&setReq) - if err != nil { - return err + diags := setJson.Unmarshal(&setReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setReq.RequestObjectType = args[0] @@ -340,9 +346,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.RequestObjectType = args[0] diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index ac361e31..5bd94e0b 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -98,9 +98,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -699,9 +705,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -788,9 +800,15 @@ func newStartUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startUpdateJson.Unmarshal(&startUpdateReq) - if err != nil { - return err + diags := startUpdateJson.Unmarshal(&startUpdateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -977,9 +995,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1058,9 +1082,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go index 1274c879..d128d80b 100755 --- a/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go +++ b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go @@ -110,9 +110,15 @@ func newEnforceCompliance() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) - if err != nil { - return err + diags := enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go index d74caa57..384dab2c 100755 --- a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go +++ b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go @@ -104,9 +104,15 @@ func newEnforceCompliance() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) - if err != nil { - return err + diags := enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go index a3f74621..fea836d2 100755 --- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go +++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go @@ -73,9 +73,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -259,9 +265,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go index b9240375..a96f0673 100755 --- a/cmd/workspace/provider-exchanges/provider-exchanges.go +++ b/cmd/workspace/provider-exchanges/provider-exchanges.go @@ -91,9 +91,15 @@ func newAddListingToExchange() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = addListingToExchangeJson.Unmarshal(&addListingToExchangeReq) - if err != nil { - return err + diags := addListingToExchangeJson.Unmarshal(&addListingToExchangeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -154,9 +160,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -546,9 +558,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go index 62dcb6de..392ed289 100755 --- a/cmd/workspace/provider-files/provider-files.go +++ b/cmd/workspace/provider-files/provider-files.go @@ -77,9 +77,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -273,9 +279,15 @@ func newList() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = listJson.Unmarshal(&listReq) - if err != nil { - return err + diags := listJson.Unmarshal(&listReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go index 18c99c53..4c7c6c56 100755 --- a/cmd/workspace/provider-listings/provider-listings.go +++ b/cmd/workspace/provider-listings/provider-listings.go @@ -75,9 +75,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -326,9 +332,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go index d18e2e57..48c444f1 100755 --- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go +++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go @@ -142,9 +142,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.ListingId = args[0] diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go index bb3ca966..a8d151a2 100755 --- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go +++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go @@ -208,9 +208,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go index 94d12d6f..3c9c024e 100755 --- a/cmd/workspace/provider-providers/provider-providers.go +++ b/cmd/workspace/provider-providers/provider-providers.go @@ -74,9 +74,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -325,9 +331,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index af2737a0..504beac5 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -446,9 +452,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/quality-monitors/quality-monitors.go b/cmd/workspace/quality-monitors/quality-monitors.go index 1ff9b017..58075aa5 100755 --- a/cmd/workspace/quality-monitors/quality-monitors.go +++ b/cmd/workspace/quality-monitors/quality-monitors.go @@ -196,9 +196,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.TableName = args[0] @@ -559,9 +565,15 @@ func newRegenerateDashboard() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = regenerateDashboardJson.Unmarshal(®enerateDashboardReq) - if err != nil { - return err + diags := regenerateDashboardJson.Unmarshal(®enerateDashboardReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } regenerateDashboardReq.TableName = args[0] @@ -724,9 +736,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.TableName = args[0] diff --git a/cmd/workspace/queries-legacy/queries-legacy.go b/cmd/workspace/queries-legacy/queries-legacy.go index fa78bb2b..e35e1828 100755 --- a/cmd/workspace/queries-legacy/queries-legacy.go +++ b/cmd/workspace/queries-legacy/queries-legacy.go @@ -96,9 +96,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -454,9 +460,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index fea01451..208f887d 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -425,9 +431,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go index 4f45ab23..f48acff1 100755 --- a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go +++ b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go @@ -87,9 +87,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -222,9 +228,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go index 04259452..62166195 100755 --- a/cmd/workspace/query-visualizations/query-visualizations.go +++ b/cmd/workspace/query-visualizations/query-visualizations.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -217,9 +223,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index f4472cf3..56abd201 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -118,9 +118,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -404,9 +410,15 @@ func newRotateToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = rotateTokenJson.Unmarshal(&rotateTokenReq) - if err != nil { - return err + diags := rotateTokenJson.Unmarshal(&rotateTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } rotateTokenReq.Name = args[0] @@ -554,9 +566,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/registered-models/registered-models.go b/cmd/workspace/registered-models/registered-models.go index 5aa6cdf1..63f307a3 100755 --- a/cmd/workspace/registered-models/registered-models.go +++ b/cmd/workspace/registered-models/registered-models.go @@ -135,9 +135,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -509,9 +515,15 @@ func newSetAlias() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setAliasJson.Unmarshal(&setAliasReq) - if err != nil { - return err + diags := setAliasJson.Unmarshal(&setAliasReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setAliasReq.FullName = args[0] @@ -589,9 +601,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/repos/overrides.go b/cmd/workspace/repos/overrides.go index 9546d1c1..aad38ecc 100644 --- a/cmd/workspace/repos/overrides.go +++ b/cmd/workspace/repos/overrides.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -35,9 +36,15 @@ func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepoReq ctx := cmd.Context() w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(createReq) - if err != nil { - return err + diags := createJson.Unmarshal(createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { createReq.Url = args[0] @@ -101,11 +108,18 @@ func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepoReq updateJson := updateCmd.Flag("json").Value.(*flags.JsonFlag) updateCmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() + var diags diag.Diagnostics w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags = updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { updateReq.RepoId, err = repoArgumentToRepoID(ctx, w, args) diff --git a/cmd/workspace/repos/repos.go b/cmd/workspace/repos/repos.go index f11dd3ac..b77347b0 100755 --- a/cmd/workspace/repos/repos.go +++ b/cmd/workspace/repos/repos.go @@ -111,9 +111,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -521,9 +527,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -604,9 +616,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -688,9 +706,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go b/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go index 5e9f59d2..5d0fba92 100755 --- a/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go +++ b/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go @@ -197,9 +197,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/schemas/schemas.go b/cmd/workspace/schemas/schemas.go index 3a398251..3ce573ba 100755 --- a/cmd/workspace/schemas/schemas.go +++ b/cmd/workspace/schemas/schemas.go @@ -100,9 +100,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -386,9 +392,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/secrets/put_secret.go b/cmd/workspace/secrets/put_secret.go index e323c7a1..f24814f0 100644 --- a/cmd/workspace/secrets/put_secret.go +++ b/cmd/workspace/secrets/put_secret.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" @@ -60,6 +61,7 @@ func newPutSecret() *cobra.Command { cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() + var diags diag.Diagnostics w := root.WorkspaceClient(ctx) bytesValueChanged := cmd.Flags().Changed("bytes-value") @@ -69,9 +71,15 @@ func newPutSecret() *cobra.Command { } if cmd.Flags().Changed("json") { - err = putSecretJson.Unmarshal(&putSecretReq) - if err != nil { - return err + diags = putSecretJson.Unmarshal(&putSecretReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { putSecretReq.Scope = args[0] diff --git a/cmd/workspace/secrets/secrets.go b/cmd/workspace/secrets/secrets.go index f836a267..e9547b62 100755 --- a/cmd/workspace/secrets/secrets.go +++ b/cmd/workspace/secrets/secrets.go @@ -110,9 +110,15 @@ func newCreateScope() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createScopeJson.Unmarshal(&createScopeReq) - if err != nil { - return err + diags := createScopeJson.Unmarshal(&createScopeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -191,9 +197,15 @@ func newDeleteAcl() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteAclJson.Unmarshal(&deleteAclReq) - if err != nil { - return err + diags := deleteAclJson.Unmarshal(&deleteAclReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -273,9 +285,15 @@ func newDeleteScope() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteScopeJson.Unmarshal(&deleteScopeReq) - if err != nil { - return err + diags := deleteScopeJson.Unmarshal(&deleteScopeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -354,9 +372,15 @@ func newDeleteSecret() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteSecretJson.Unmarshal(&deleteSecretReq) - if err != nil { - return err + diags := deleteSecretJson.Unmarshal(&deleteSecretReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -759,9 +783,15 @@ func newPutAcl() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putAclJson.Unmarshal(&putAclReq) - if err != nil { - return err + diags := putAclJson.Unmarshal(&putAclReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/service-principals/service-principals.go b/cmd/workspace/service-principals/service-principals.go index 957cb126..317779f3 100755 --- a/cmd/workspace/service-principals/service-principals.go +++ b/cmd/workspace/service-principals/service-principals.go @@ -95,9 +95,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -448,9 +460,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index 0837652d..363e9ea1 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -169,9 +169,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -706,9 +712,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } patchReq.Name = args[0] @@ -777,9 +789,15 @@ func newPut() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putJson.Unmarshal(&putReq) - if err != nil { - return err + diags := putJson.Unmarshal(&putReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } putReq.Name = args[0] @@ -850,9 +868,15 @@ func newPutAiGateway() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putAiGatewayJson.Unmarshal(&putAiGatewayReq) - if err != nil { - return err + diags := putAiGatewayJson.Unmarshal(&putAiGatewayReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } putAiGatewayReq.Name = args[0] @@ -928,9 +952,15 @@ func newQuery() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryJson.Unmarshal(&queryReq) - if err != nil { - return err + diags := queryJson.Unmarshal(&queryReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } queryReq.Name = args[0] @@ -997,9 +1027,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.ServingEndpointId = args[0] @@ -1076,9 +1112,15 @@ func newUpdateConfig() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateConfigJson.Unmarshal(&updateConfigReq) - if err != nil { - return err + diags := updateConfigJson.Unmarshal(&updateConfigReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateConfigReq.Name = args[0] @@ -1158,9 +1200,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.ServingEndpointId = args[0] diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index 67f87017..62c3407f 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -100,9 +100,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -427,9 +433,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -501,9 +513,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.Name = args[0] diff --git a/cmd/workspace/storage-credentials/storage-credentials.go b/cmd/workspace/storage-credentials/storage-credentials.go index f4ec5eb4..2caf0904 100755 --- a/cmd/workspace/storage-credentials/storage-credentials.go +++ b/cmd/workspace/storage-credentials/storage-credentials.go @@ -111,9 +111,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -377,9 +383,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -478,9 +490,15 @@ func newValidate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = validateJson.Unmarshal(&validateReq) - if err != nil { - return err + diags := validateJson.Unmarshal(&validateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/table-constraints/table-constraints.go b/cmd/workspace/table-constraints/table-constraints.go index 166da146..4ac7cb9f 100755 --- a/cmd/workspace/table-constraints/table-constraints.go +++ b/cmd/workspace/table-constraints/table-constraints.go @@ -92,9 +92,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/tables/tables.go b/cmd/workspace/tables/tables.go index ec297f29..35775f17 100755 --- a/cmd/workspace/tables/tables.go +++ b/cmd/workspace/tables/tables.go @@ -479,9 +479,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go index 8718f7ba..210a59f8 100755 --- a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go +++ b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go @@ -94,9 +94,15 @@ func newGenerateTemporaryTableCredentials() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) - if err != nil { - return err + diags := generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/token-management/token-management.go b/cmd/workspace/token-management/token-management.go index dea94edb..6deb8d12 100755 --- a/cmd/workspace/token-management/token-management.go +++ b/cmd/workspace/token-management/token-management.go @@ -96,9 +96,15 @@ func newCreateOboToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createOboTokenJson.Unmarshal(&createOboTokenReq) - if err != nil { - return err + diags := createOboTokenJson.Unmarshal(&createOboTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -458,9 +464,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -523,9 +535,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/tokens/tokens.go b/cmd/workspace/tokens/tokens.go index afe4b9a0..5c9b4994 100755 --- a/cmd/workspace/tokens/tokens.go +++ b/cmd/workspace/tokens/tokens.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -158,9 +164,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { diff --git a/cmd/workspace/users/users.go b/cmd/workspace/users/users.go index 53ba2e85..b085ab41 100755 --- a/cmd/workspace/users/users.go +++ b/cmd/workspace/users/users.go @@ -107,9 +107,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -463,9 +469,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -546,9 +558,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -619,9 +637,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -702,9 +726,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go index dd9d5783..0cfb7617 100755 --- a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go +++ b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go @@ -95,9 +95,15 @@ func newCreateEndpoint() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createEndpointJson.Unmarshal(&createEndpointReq) - if err != nil { - return err + diags := createEndpointJson.Unmarshal(&createEndpointReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go index 15847477..832f4a6d 100755 --- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go +++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go @@ -114,9 +114,15 @@ func newCreateIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createIndexJson.Unmarshal(&createIndexReq) - if err != nil { - return err + diags := createIndexJson.Unmarshal(&createIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -195,9 +201,15 @@ func newDeleteDataVectorIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteDataVectorIndexJson.Unmarshal(&deleteDataVectorIndexReq) - if err != nil { - return err + diags := deleteDataVectorIndexJson.Unmarshal(&deleteDataVectorIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -443,9 +455,15 @@ func newQueryIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryIndexJson.Unmarshal(&queryIndexReq) - if err != nil { - return err + diags := queryIndexJson.Unmarshal(&queryIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -515,9 +533,15 @@ func newQueryNextPage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryNextPageJson.Unmarshal(&queryNextPageReq) - if err != nil { - return err + diags := queryNextPageJson.Unmarshal(&queryNextPageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } queryNextPageReq.IndexName = args[0] @@ -585,9 +609,15 @@ func newScanIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = scanIndexJson.Unmarshal(&scanIndexReq) - if err != nil { - return err + diags := scanIndexJson.Unmarshal(&scanIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } scanIndexReq.IndexName = args[0] @@ -718,9 +748,15 @@ func newUpsertDataVectorIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = upsertDataVectorIndexJson.Unmarshal(&upsertDataVectorIndexReq) - if err != nil { - return err + diags := upsertDataVectorIndexJson.Unmarshal(&upsertDataVectorIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } upsertDataVectorIndexReq.IndexName = args[0] diff --git a/cmd/workspace/volumes/volumes.go b/cmd/workspace/volumes/volumes.go index 3fc1f447..2f455573 100755 --- a/cmd/workspace/volumes/volumes.go +++ b/cmd/workspace/volumes/volumes.go @@ -119,9 +119,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -427,9 +433,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/warehouses/warehouses.go b/cmd/workspace/warehouses/warehouses.go index cdf10636..43d6c8ab 100755 --- a/cmd/workspace/warehouses/warehouses.go +++ b/cmd/workspace/warehouses/warehouses.go @@ -109,9 +109,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -277,9 +283,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -688,9 +700,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -779,9 +797,15 @@ func newSetWorkspaceWarehouseConfig() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setWorkspaceWarehouseConfigJson.Unmarshal(&setWorkspaceWarehouseConfigReq) - if err != nil { - return err + diags := setWorkspaceWarehouseConfigJson.Unmarshal(&setWorkspaceWarehouseConfigReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1030,9 +1054,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/workspace-bindings/workspace-bindings.go b/cmd/workspace/workspace-bindings/workspace-bindings.go index 4993f1af..20f54e1d 100755 --- a/cmd/workspace/workspace-bindings/workspace-bindings.go +++ b/cmd/workspace/workspace-bindings/workspace-bindings.go @@ -226,9 +226,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -297,9 +303,15 @@ func newUpdateBindings() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateBindingsJson.Unmarshal(&updateBindingsReq) - if err != nil { - return err + diags := updateBindingsJson.Unmarshal(&updateBindingsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateBindingsReq.SecurableType) diff --git a/cmd/workspace/workspace-conf/workspace-conf.go b/cmd/workspace/workspace-conf/workspace-conf.go index 92b2f0f3..a17bc163 100755 --- a/cmd/workspace/workspace-conf/workspace-conf.go +++ b/cmd/workspace/workspace-conf/workspace-conf.go @@ -127,9 +127,15 @@ func newSetStatus() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setStatusJson.Unmarshal(&setStatusReq) - if err != nil { - return err + diags := setStatusJson.Unmarshal(&setStatusReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/workspace/workspace.go b/cmd/workspace/workspace/workspace.go index 183cac89..21da478c 100755 --- a/cmd/workspace/workspace/workspace.go +++ b/cmd/workspace/workspace/workspace.go @@ -106,9 +106,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -482,9 +488,15 @@ func newImport() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = importJson.Unmarshal(&importReq) - if err != nil { - return err + diags := importJson.Unmarshal(&importReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -622,9 +634,15 @@ func newMkdirs() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = mkdirsJson.Unmarshal(&mkdirsReq) - if err != nil { - return err + diags := mkdirsJson.Unmarshal(&mkdirsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -710,9 +728,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.WorkspaceObjectType = args[0] @@ -781,9 +805,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.WorkspaceObjectType = args[0] diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index 4114db5c..72d95978 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -13,6 +13,7 @@ import ( "text/template" "time" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/listing" "github.com/fatih/color" @@ -285,70 +286,72 @@ func RenderIteratorJson[T any](ctx context.Context, i listing.Iterator[T]) error return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, c.headerTemplate, c.template) } +var renderFuncMap = template.FuncMap{ + // we render colored output if stdout is TTY, otherwise we render text. + // in the future we'll check if we can explicitly check for stderr being + // a TTY + "header": color.BlueString, + "red": color.RedString, + "green": color.GreenString, + "blue": color.BlueString, + "yellow": color.YellowString, + "magenta": color.MagentaString, + "cyan": color.CyanString, + "bold": func(format string, a ...interface{}) string { + return color.New(color.Bold).Sprintf(format, a...) + }, + "italic": func(format string, a ...interface{}) string { + return color.New(color.Italic).Sprintf(format, a...) + }, + "replace": strings.ReplaceAll, + "join": strings.Join, + "bool": func(v bool) string { + if v { + return color.GreenString("YES") + } + return color.RedString("NO") + }, + "pretty_json": func(in string) (string, error) { + var tmp any + err := json.Unmarshal([]byte(in), &tmp) + if err != nil { + return "", err + } + b, err := fancyJSON(tmp) + if err != nil { + return "", err + } + return string(b), nil + }, + "pretty_date": func(t time.Time) string { + return t.Format("2006-01-02T15:04:05Z") + }, + "b64_encode": func(in string) (string, error) { + var out bytes.Buffer + enc := base64.NewEncoder(base64.StdEncoding, &out) + _, err := enc.Write([]byte(in)) + if err != nil { + return "", err + } + err = enc.Close() + if err != nil { + return "", err + } + return out.String(), nil + }, + "b64_decode": func(in string) (string, error) { + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(in)) + out, err := io.ReadAll(dec) + if err != nil { + return "", err + } + return string(out), nil + }, +} + func renderUsingTemplate(ctx context.Context, r templateRenderer, w io.Writer, headerTmpl, tmpl string) error { tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - base := template.New("command").Funcs(template.FuncMap{ - // we render colored output if stdout is TTY, otherwise we render text. - // in the future we'll check if we can explicitly check for stderr being - // a TTY - "header": color.BlueString, - "red": color.RedString, - "green": color.GreenString, - "blue": color.BlueString, - "yellow": color.YellowString, - "magenta": color.MagentaString, - "cyan": color.CyanString, - "bold": func(format string, a ...interface{}) string { - return color.New(color.Bold).Sprintf(format, a...) - }, - "italic": func(format string, a ...interface{}) string { - return color.New(color.Italic).Sprintf(format, a...) - }, - "replace": strings.ReplaceAll, - "join": strings.Join, - "bool": func(v bool) string { - if v { - return color.GreenString("YES") - } - return color.RedString("NO") - }, - "pretty_json": func(in string) (string, error) { - var tmp any - err := json.Unmarshal([]byte(in), &tmp) - if err != nil { - return "", err - } - b, err := fancyJSON(tmp) - if err != nil { - return "", err - } - return string(b), nil - }, - "pretty_date": func(t time.Time) string { - return t.Format("2006-01-02T15:04:05Z") - }, - "b64_encode": func(in string) (string, error) { - var out bytes.Buffer - enc := base64.NewEncoder(base64.StdEncoding, &out) - _, err := enc.Write([]byte(in)) - if err != nil { - return "", err - } - err = enc.Close() - if err != nil { - return "", err - } - return out.String(), nil - }, - "b64_decode": func(in string) (string, error) { - dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(in)) - out, err := io.ReadAll(dec) - if err != nil { - return "", err - } - return string(out), nil - }, - }) + base := template.New("command").Funcs(renderFuncMap) if headerTmpl != "" { headerT, err := base.Parse(headerTmpl) if err != nil { @@ -388,3 +391,77 @@ func fancyJSON(v any) ([]byte, error) { return jsoncolor.MarshalIndentWithFormatter(v, "", " ", f) } + +const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +func RenderDiagnosticsToErrorOut(ctx context.Context, diags diag.Diagnostics) error { + c := fromContext(ctx) + return RenderDiagnostics(c.err, diags) +} + +func RenderDiagnostics(out io.Writer, diags diag.Diagnostics) error { + errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) + warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) + recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate)) + + // Print errors and warnings. + for _, d := range diags { + var t *template.Template + switch d.Severity { + case diag.Error: + t = errorT + case diag.Warning: + t = warningT + case diag.Recommendation: + t = recommendationT + } + + // Render the diagnostic with the appropriate template. + err := t.Execute(out, d) + if err != nil { + return fmt.Errorf("failed to render template: %w", err) + } + } + + return nil +} diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 839d0111..e2aa5d26 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -221,10 +221,10 @@ func toTypedBool(dst reflect.Value, src dyn.Value) error { case dyn.KindString: // See https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/decode.go#L684-L693. switch src.MustString() { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", "true": dst.SetBool(true) return nil - case "n", "N", "no", "No", "NO", "off", "Off", "OFF": + case "n", "N", "no", "No", "NO", "off", "Off", "OFF", "false": dst.SetBool(false) return nil } @@ -246,6 +246,19 @@ func toTypedInt(dst reflect.Value, src dyn.Value) error { case dyn.KindInt: dst.SetInt(src.MustInt()) return nil + case dyn.KindFloat: + v := src.MustFloat() + if v == float64(int64(v)) { + // If the destination is smaller than int64, but the value to set is bigger + // then destination overflows and is set to -1 + dst.SetInt(int64(src.MustFloat())) + return nil + } + + return TypeError{ + value: src, + msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), + } case dyn.KindString: if i64, err := strconv.ParseInt(src.MustString(), 10, 64); err == nil { dst.SetInt(i64) diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go index 37d85539..78221c29 100644 --- a/libs/dyn/convert/to_typed_test.go +++ b/libs/dyn/convert/to_typed_test.go @@ -341,14 +341,14 @@ func TestToTypedBoolFromString(t *testing.T) { var out bool // True-ish - for _, v := range []string{"y", "yes", "on"} { + for _, v := range []string{"y", "yes", "on", "true"} { err := ToTyped(&out, dyn.V(v)) require.NoError(t, err) assert.Equal(t, true, out) } // False-ish - for _, v := range []string{"n", "no", "off"} { + for _, v := range []string{"n", "no", "off", "false"} { err := ToTyped(&out, dyn.V(v)) require.NoError(t, err) assert.Equal(t, false, out) @@ -428,6 +428,19 @@ func TestToTypedIntFromStringVariableReference(t *testing.T) { assert.Equal(t, int(0), out) } +func TestToTypedIntFromFloat(t *testing.T) { + var out int + err := ToTyped(&out, dyn.V(1.0)) + require.NoError(t, err) + assert.Equal(t, int(1), out) +} + +func TestToTypedIntFromFloatError(t *testing.T) { + var out int + err := ToTyped(&out, dyn.V(1.2)) + require.ErrorContains(t, err, "expected an int, found a float") +} + func TestToTypedFloat32(t *testing.T) { var out float32 err := ToTyped(&out, dyn.V(float32(1.0))) diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go new file mode 100644 index 00000000..cbf53926 --- /dev/null +++ b/libs/dyn/jsonloader/json.go @@ -0,0 +1,107 @@ +package jsonloader + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/databricks/cli/libs/dyn" +) + +func LoadJSON(data []byte, source string) (dyn.Value, error) { + offset := BuildLineOffsets(data) + offset.SetSource(source) + + reader := bytes.NewReader(data) + decoder := json.NewDecoder(reader) + + // Start decoding from the top-level value + value, err := decodeValue(decoder, &offset) + if err != nil { + if err == io.EOF { + err = fmt.Errorf("unexpected end of JSON input") + } + return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %v", value.Location(), err) + } + return value, nil +} + +func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { + // Read the next JSON token + token, err := decoder.Token() + if err != nil { + return dyn.InvalidValue, err + } + + // Get the current byte offset and the location. + // We will later use this location to store the location of the value in the file + // For objects and arrays, we will store the location of the opening '{' or '[' + // For primitive types, we will store the location of the value itself (end of the value) + // We can't reliably calculate the beginning of the value for primitive types because + // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. + offset := decoder.InputOffset() + location := o.GetPosition(offset) + + switch tok := token.(type) { + case json.Delim: + if tok == '{' { + location = o.GetPosition(offset - 1) + // Decode JSON object + obj := dyn.NewMapping() + for decoder.More() { + // Decode the key + keyToken, err := decoder.Token() + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + key, ok := keyToken.(string) + if !ok { + return invalidValueWithLocation(decoder, o), fmt.Errorf("expected string for object key") + } + + // Get the offset of the key by subtracting the length of the key and the '"' character + keyOffset := decoder.InputOffset() - int64(len(key)+1) + keyVal := dyn.NewValue(key, []dyn.Location{o.GetPosition(keyOffset)}) + + // Decode the value recursively + val, err := decodeValue(decoder, o) + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + + obj.Set(keyVal, val) + } + // Consume the closing '}' + if _, err := decoder.Token(); err != nil { + return invalidValueWithLocation(decoder, o), err + } + return dyn.NewValue(obj, []dyn.Location{location}), nil + } else if tok == '[' { + location = o.GetPosition(offset - 1) + // Decode JSON array + var arr []dyn.Value + for decoder.More() { + val, err := decodeValue(decoder, o) + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + arr = append(arr, val) + } + // Consume the closing ']' + if _, err := decoder.Token(); err != nil { + return invalidValueWithLocation(decoder, o), err + } + return dyn.NewValue(arr, []dyn.Location{location}), nil + } + default: + return dyn.NewValue(tok, []dyn.Location{location}), nil + } + + return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) +} + +func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { + location := o.GetPosition(decoder.InputOffset()) + return dyn.InvalidValue.WithLocations([]dyn.Location{location}) +} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go new file mode 100644 index 00000000..7fdc4d7f --- /dev/null +++ b/libs/dyn/jsonloader/json_test.go @@ -0,0 +1,93 @@ +package jsonloader + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +const jsonData = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "webhook_notifications": { + "on_start": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": 0, + "max_concurrent_runs": 1, + "tasks": [ + { + "task_key": "xxx", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonLoader(t *testing.T) { + v, err := LoadJSON([]byte(jsonData), "(inline)") + dynassert.NoError(t, err) + + var r jobs.ResetJob + err = convert.ToTyped(&r, v) + dynassert.NoError(t, err) +} + +const malformedMap = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "wrong", + } +} +` + +func TestJsonLoaderMalformedMap(t *testing.T) { + _, err := LoadJSON([]byte(malformedMap), "(inline)") + dynassert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") +} + +const malformedArray = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "tasks": [1, "asd",] + } +}` + +func TestJsonLoaderMalformedArray(t *testing.T) { + _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") + dynassert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:28: invalid character ']' looking for beginning of value") +} + +const eofData = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx",` + +func TestJsonLoaderEOF(t *testing.T) { + _, err := LoadJSON([]byte(eofData), "path/to/file.json") + dynassert.ErrorContains(t, err, "unexpected end of JSON input") +} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go new file mode 100644 index 00000000..a692c7d0 --- /dev/null +++ b/libs/dyn/jsonloader/locations.go @@ -0,0 +1,53 @@ +package jsonloader + +import ( + "sort" + + "github.com/databricks/cli/libs/dyn" +) + +type LineOffset struct { + Line int + Start int64 +} + +type Offset struct { + offsets []LineOffset + source string +} + +// buildLineOffsets scans the input data and records the starting byte offset of each line. +func BuildLineOffsets(data []byte) Offset { + offsets := []LineOffset{{Line: 1, Start: 0}} + line := 1 + for i, b := range data { + if b == '\n' { + line++ + offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) + } + } + return Offset{offsets: offsets} +} + +// GetPosition maps a byte offset to its corresponding line and column numbers. +func (o Offset) GetPosition(offset int64) dyn.Location { + // Binary search to find the line + idx := sort.Search(len(o.offsets), func(i int) bool { + return o.offsets[i].Start > offset + }) - 1 + + if idx < 0 { + idx = 0 + } + + lineOffset := o.offsets[idx] + return dyn.Location{ + File: o.source, + Line: lineOffset.Line, + Column: int(offset-lineOffset.Start) + 1, + } +} + +func (o *Offset) SetSource(source string) { + o.source = source +} diff --git a/libs/flags/json_flag.go b/libs/flags/json_flag.go index 8dbc3b2d..0ed8be74 100644 --- a/libs/flags/json_flag.go +++ b/libs/flags/json_flag.go @@ -4,10 +4,16 @@ import ( "encoding/json" "fmt" "os" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/databricks-sdk-go/marshal" ) type JsonFlag struct { - raw []byte + raw []byte + source string } func (j *JsonFlag) String() string { @@ -19,21 +25,52 @@ func (j *JsonFlag) Set(v string) error { // Load request from file if it starts with '@' (like curl). if v[0] != '@' { j.raw = []byte(v) + j.source = "(inline)" return nil } - buf, err := os.ReadFile(v[1:]) + filePath := v[1:] + buf, err := os.ReadFile(filePath) + j.source = filePath if err != nil { - return fmt.Errorf("read %s: %w", v, err) + return fmt.Errorf("read %s: %w", filePath, err) } j.raw = buf return nil } -func (j *JsonFlag) Unmarshal(v any) error { +func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { if j.raw == nil { return nil } - return json.Unmarshal(j.raw, v) + + dv, err := jsonloader.LoadJSON(j.raw, j.source) + if err != nil { + return diag.FromErr(err) + } + + // First normalize the input data. + // It will convert all the values to the correct types. + // For example string literals for booleans and integers will be converted to the correct types. + nv, diags := convert.Normalize(v, dv) + if diags.HasError() { + return diags + } + + // Then marshal the normalized data to the output. + // It will serialize all set data with the correct types. + data, err := json.Marshal(nv.AsAny()) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + + // Finally unmarshal the normalized data to the output. + // It will fill in the ForceSendFields field if the struct contains it. + err = marshal.Unmarshal(data, v) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + + return diags } func (j *JsonFlag) Type() string { diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 2a8170fe..16769f4e 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -6,18 +6,26 @@ import ( "path" "testing" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type requestType struct { + Foo string `json:"foo"` +} + func TestJsonFlagEmpty(t *testing.T) { var body JsonFlag var request any - err := body.Unmarshal(&request) + diags := body.Unmarshal(&request) assert.Equal(t, "JSON (0 bytes)", body.String()) - assert.NoError(t, err) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) assert.Nil(t, request) } @@ -27,12 +35,13 @@ func TestJsonFlagInline(t *testing.T) { err := body.Set(`{"foo": "bar"}`) assert.NoError(t, err) - var request any - err = body.Unmarshal(&request) - assert.NoError(t, err) + var request requestType + diags := body.Unmarshal(&request) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) assert.Equal(t, "JSON (14 bytes)", body.String()) - assert.Equal(t, map[string]any{"foo": "bar"}, request) + assert.Equal(t, requestType{"bar"}, request) } func TestJsonFlagError(t *testing.T) { @@ -41,18 +50,18 @@ func TestJsonFlagError(t *testing.T) { err := body.Set(`{"foo":`) assert.NoError(t, err) - var request any - err = body.Unmarshal(&request) - assert.EqualError(t, err, "unexpected end of JSON input") + var request requestType + diags := body.Unmarshal(&request) + assert.EqualError(t, diags.Error(), "error decoding JSON at (inline):1:8: unexpected end of JSON input") assert.Equal(t, "JSON (7 bytes)", body.String()) } func TestJsonFlagFile(t *testing.T) { var body JsonFlag - var request any + var request requestType var fpath string - var payload = []byte(`"hello world"`) + var payload = []byte(`{"foo": "bar"}`) { f, err := os.Create(path.Join(t.TempDir(), "file")) @@ -65,8 +74,217 @@ func TestJsonFlagFile(t *testing.T) { err := body.Set(fmt.Sprintf("@%s", fpath)) require.NoError(t, err) - err = body.Unmarshal(&request) + diags := body.Unmarshal(&request) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, requestType{"bar"}, request) +} + +const jsonData = ` +{ + "job_id": 123, + "new_settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_canceled_runs": false + }, + "timeout_seconds": 0, + "max_concurrent_runs": 1, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalForRequest(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(jsonData) require.NoError(t, err) - assert.Equal(t, "hello world", request) + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, int64(123), r.JobId) + assert.Equal(t, "new job", r.NewSettings.Name) + assert.Equal(t, 0, r.NewSettings.TimeoutSeconds) + assert.Equal(t, 1, r.NewSettings.MaxConcurrentRuns) + assert.Equal(t, 1, len(r.NewSettings.Tasks)) + assert.Equal(t, "new task", r.NewSettings.Tasks[0].TaskKey) + assert.Equal(t, 0, r.NewSettings.Tasks[0].TimeoutSeconds) + assert.Equal(t, 0, r.NewSettings.Tasks[0].MaxRetries) + assert.Equal(t, 0, r.NewSettings.Tasks[0].MinRetryIntervalMillis) + assert.Equal(t, true, r.NewSettings.Tasks[0].RetryOnTimeout) +} + +const incorrectJsonData = `{ + "job_id": 123, + "settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": {}, + "max_concurrent_runs": {}, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalRequestMismatch(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(incorrectJsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "unknown field: settings", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 3, + Column: 6, + }, + }, + Paths: []dyn.Path{{}}, + }) +} + +const wrontTypeJsonData = `{ + "job_id": 123, + "new_settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": "wrong_type", + "max_concurrent_runs": {}, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(`{ + "job_id": [1, 2, 3] +} +`) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "expected int, found sequence", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 2, + Column: 15, + }, + }, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("job_id"))}, + }) +} + +func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(wrontTypeJsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "cannot parse \"wrong_type\" as an integer", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 14, + Column: 40, + }, + }, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))}, + }) +} + +func TestJsonUnmarshalForRequestWithForceSendFields(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(jsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.NotContains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForSkippedRuns") + assert.Contains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForCanceledRuns") }