From d30c4c730d1e0b674a14471a401e9b0d07d64dd7 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Mon, 8 Jul 2024 15:32:56 +0200 Subject: [PATCH 01/88] Add new template (#1578) ## Changes Add a new hidden experimental template ## Tests Tested manually --- cmd/bundle/init.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index c8c59c149..c25391577 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -49,6 +49,12 @@ var nativeTemplates = []nativeTemplate{ description: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)", aliases: []string{"mlops-stack"}, }, + { + name: "default-pydabs", + gitUrl: "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git", + hidden: true, + description: "The default PyDABs template", + }, { name: customTemplate, description: "Bring your own template", From 056e2af743dda2f50e4d4b45764f020ef2c26e6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:01:11 +0200 Subject: [PATCH 02/88] Bump golang.org/x/mod from 0.18.0 to 0.19.0 (#1576) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.18.0 to 0.19.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.18.0&new-version=0.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 385a93b09..175591d23 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.18.0 + golang.org/x/mod v0.19.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 golang.org/x/term v0.21.0 diff --git a/go.sum b/go.sum index 864b7919b..2121224bf 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 4d13c7fbe3525dabc9706c54dbe661511b4c5441 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:01:30 +0200 Subject: [PATCH 03/88] Bump golang.org/x/term from 0.21.0 to 0.22.0 (#1577) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.21.0 to 0.22.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 175591d23..ce7ad0c1e 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( golang.org/x/mod v0.19.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 + golang.org/x/term v0.22.0 golang.org/x/text v0.16.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -61,7 +61,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 2121224bf..eb7a87a89 100644 --- a/go.sum +++ b/go.sum @@ -208,10 +208,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= From 8b468b423ff1166f104211548f08a5ab941732e0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 9 Jul 2024 13:12:42 +0200 Subject: [PATCH 04/88] Change SetVariables mutator to mutate dynamic configuration instead (#1573) ## Changes Previously `SetVariables` mutator mutated typed configuration by using `v.Set` for variables. This lead to variables `value` field not having location information. By using dynamic configuration mutation, we keep the same functionality but also preserve location information for value when it's set from default. Fixes #1568 #1538 ## Tests Added unit tests --- bundle/config/mutator/set_variables.go | 59 ++++++++++------- bundle/config/mutator/set_variables_test.go | 55 ++++++++++++---- bundle/config/mutator/translate_paths_test.go | 64 +++++++++++++++++++ 3 files changed, 143 insertions(+), 35 deletions(-) diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index b3a9cf400..47ce2ad03 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -2,10 +2,12 @@ package mutator import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" ) @@ -21,52 +23,63 @@ func (m *setVariables) Name() string { return "SetVariables" } -func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Diagnostics { +func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) { // case: variable already has value initialized, so skip - if v.HasValue() { - return nil + if variable.HasValue() { + return v, nil } // case: read and set variable value from process environment envVarName := bundleVarPrefix + name if val, ok := env.Lookup(ctx, envVarName); ok { - if v.IsComplex() { - return diag.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) + if variable.IsComplex() { + return dyn.InvalidValue, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) } - err := v.Set(val) + v, err := dyn.Set(v, "value", dyn.V(val)) if err != nil { - return diag.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) + return dyn.InvalidValue, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) } - return nil + return v, nil } // case: Defined a variable for named lookup for a resource // It will be resolved later in ResolveResourceReferences mutator - if v.Lookup != nil { - return nil + if variable.Lookup != nil { + return v, nil } // case: Set the variable to its default value - if v.HasDefault() { - err := v.Set(v.Default) + if variable.HasDefault() { + vDefault, err := dyn.Get(v, "default") if err != nil { - return diag.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, v.Default, name, err) + return dyn.InvalidValue, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err) } - return nil + + v, err := dyn.Set(v, "value", vDefault) + if err != nil { + return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) + } + return v, nil } // We should have had a value to set for the variable at this point. - return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - for name, variable := range b.Config.Variables { - diags = diags.Extend(setVariable(ctx, variable, name)) - if diags.HasError() { - return diags - } - } - return diags + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { + name := p[1].Key() + v, ok := b.Config.Variables[name] + if !ok { + return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) + } + + return setVariable(ctx, variable, v, name) + })) + }) + + return diag.FromErr(err) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index 65dedee97..d9719793f 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -7,6 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,9 +22,14 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "process-env") } @@ -33,8 +40,14 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { Default: defaultVal, } - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "default") } @@ -49,8 +62,14 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the default value - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "assigned-value") } @@ -68,8 +87,14 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the value from environment - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "assigned-value") } @@ -79,8 +104,11 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { } // fails because we could not resolve a value for the variable - diags := setVariable(context.Background(), &variable, "foo") - assert.ErrorContains(t, diags.Error(), "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + _, err = setVariable(context.Background(), v, &variable, "foo") + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") } func TestSetVariablesMutator(t *testing.T) { @@ -126,6 +154,9 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - diags := setVariable(context.Background(), &variable, "foo") - assert.ErrorContains(t, diags.Error(), "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + _, err = setVariable(context.Background(), v, &variable, "foo") + assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8476ee38a..780a540df 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -11,7 +11,10 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -708,3 +711,64 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) } + +func TestTranslatePathWithComplexVariables(t *testing.T) { + dir := t.TempDir() + b := &bundle.Bundle{ + RootPath: dir, + BundleRoot: vfs.MustNew(dir), + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "cluster_libraries": { + Type: variable.VariableTypeComplex, + Default: [](map[string]string){ + { + "whl": "./local/whl.whl", + }, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "test", + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml")) + bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + + ctx := context.Background() + // Assign the variables to the dynamic configuration. + diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + p := dyn.MustPathFromString("resources.jobs.job.tasks[0]") + return dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}")) + }) + return diag.FromErr(err) + }) + require.NoError(t, diags.Error()) + + diags = bundle.Apply(ctx, b, + bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferences("variables"), + mutator.TranslatePaths(), + )) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.Join("variables", "local", "whl.whl"), + b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl, + ) +} From 5bc5c3c26acf671026217b71bd52afb72cf3a472 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:38:38 +0530 Subject: [PATCH 05/88] Return early in bundle destroy if no deployment exists (#1581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This PR: 1. Moves the if mutator to the bundle package, to live with all-time greats such as `bundle.Seq` and `bundle.Defer`. Also adds unit tests. 2. `bundle destroy` now returns early if `root_path` does not exist. We do this by leveraging a `bundle.If` condition. ## Tests Unit tests and manually. Here's an example of what it'll look like once the bundle is destroyed. ``` ➜ bundle-playground git:(master) ✗ cli bundle destroy No active deployment found to destroy! ``` I would have added some e2e coverage for this as well, but the `cobraTestRunner.Run()` method does not seem to return stdout/stderr logs correctly. We can probably punt looking into it. --- bundle/config/mutator/if.go | 36 ------------------------- bundle/if.go | 40 ++++++++++++++++++++++++++++ bundle/if_test.go | 53 +++++++++++++++++++++++++++++++++++++ bundle/phases/destroy.go | 29 ++++++++++++++++++-- bundle/python/transform.go | 8 +++--- 5 files changed, 125 insertions(+), 41 deletions(-) delete mode 100644 bundle/config/mutator/if.go create mode 100644 bundle/if.go create mode 100644 bundle/if_test.go diff --git a/bundle/config/mutator/if.go b/bundle/config/mutator/if.go deleted file mode 100644 index 1b7856b3c..000000000 --- a/bundle/config/mutator/if.go +++ /dev/null @@ -1,36 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" -) - -type ifMutator struct { - condition func(*bundle.Bundle) bool - onTrueMutator bundle.Mutator - onFalseMutator bundle.Mutator -} - -func If( - condition func(*bundle.Bundle) bool, - onTrueMutator bundle.Mutator, - onFalseMutator bundle.Mutator, -) bundle.Mutator { - return &ifMutator{ - condition, onTrueMutator, onFalseMutator, - } -} - -func (m *ifMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.condition(b) { - return bundle.Apply(ctx, b, m.onTrueMutator) - } else { - return bundle.Apply(ctx, b, m.onFalseMutator) - } -} - -func (m *ifMutator) Name() string { - return "If" -} diff --git a/bundle/if.go b/bundle/if.go new file mode 100644 index 000000000..bad1d72d2 --- /dev/null +++ b/bundle/if.go @@ -0,0 +1,40 @@ +package bundle + +import ( + "context" + + "github.com/databricks/cli/libs/diag" +) + +type ifMutator struct { + condition func(context.Context, *Bundle) (bool, error) + onTrueMutator Mutator + onFalseMutator Mutator +} + +func If( + condition func(context.Context, *Bundle) (bool, error), + onTrueMutator Mutator, + onFalseMutator Mutator, +) Mutator { + return &ifMutator{ + condition, onTrueMutator, onFalseMutator, + } +} + +func (m *ifMutator) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { + v, err := m.condition(ctx, b) + if err != nil { + return diag.FromErr(err) + } + + if v { + return Apply(ctx, b, m.onTrueMutator) + } else { + return Apply(ctx, b, m.onFalseMutator) + } +} + +func (m *ifMutator) Name() string { + return "If" +} diff --git a/bundle/if_test.go b/bundle/if_test.go new file mode 100644 index 000000000..b3fc0b9d9 --- /dev/null +++ b/bundle/if_test.go @@ -0,0 +1,53 @@ +package bundle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIfMutatorTrue(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return true, nil + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.NoError(t, diags.Error()) + + assert.Equal(t, 1, m1.applyCalled) + assert.Equal(t, 0, m2.applyCalled) +} + +func TestIfMutatorFalse(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return false, nil + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.NoError(t, diags.Error()) + + assert.Equal(t, 0, m1.applyCalled) + assert.Equal(t, 1, m2.applyCalled) +} + +func TestIfMutatorError(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return true, assert.AnError + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.Error(t, diags.Error()) + + assert.Equal(t, 0, m1.applyCalled) + assert.Equal(t, 0, m2.applyCalled) +} diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index f974a0565..f1beace84 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -1,15 +1,33 @@ package phases import ( + "context" + "errors" + "net/http" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" ) +func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { + w := b.WorkspaceClient() + _, err := w.Workspace.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + + var aerr *apierr.APIError + if errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound { + log.Infof(ctx, "Root path does not exist: %s", b.Config.Workspace.RootPath) + return false, nil + } + + return true, err +} + // The destroy phase deletes artifacts and resources. func Destroy() bundle.Mutator { - destroyMutator := bundle.Seq( lock.Acquire(), bundle.Defer( @@ -29,6 +47,13 @@ func Destroy() bundle.Mutator { return newPhase( "destroy", - []bundle.Mutator{destroyMutator}, + []bundle.Mutator{ + // Only run deploy mutator if root path exists. + bundle.If( + assertRootPathExists, + destroyMutator, + bundle.LogString("No active deployment found to destroy!"), + ), + }, ) } diff --git a/bundle/python/transform.go b/bundle/python/transform.go index 457b45f78..9d3b1ab6a 100644 --- a/bundle/python/transform.go +++ b/bundle/python/transform.go @@ -1,6 +1,7 @@ package python import ( + "context" "fmt" "strconv" "strings" @@ -63,9 +64,10 @@ dbutils.notebook.exit(s) // which installs uploaded wheels using %pip and then calling corresponding // entry point. func TransformWheelTask() bundle.Mutator { - return mutator.If( - func(b *bundle.Bundle) bool { - return b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper + return bundle.If( + func(_ context.Context, b *bundle.Bundle) (bool, error) { + res := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper + return res, nil }, mutator.NewTrampoline( "python_wheel", From 8f56ca39a26296edabe141a9d974b971ee728849 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 10 Jul 2024 08:37:47 +0200 Subject: [PATCH 06/88] Let notebook detection code use underlying metadata if available (#1574) ## Changes If we're using a `vfs.Path` backed by a workspace filesystem filer, we have access to the `workspace.ObjectInfo` value for every file. By providing access to this value we can use it directly and avoid reading the first line of the underlying file. A follow-up change will implement the interface defined in this change for the workspace filesystem filer. ## Tests Unit tests. --- libs/notebook/detect.go | 76 ++++++++++++++++++++++++++++++----- libs/notebook/detect_test.go | 18 +++++++++ libs/notebook/fakefs_test.go | 77 ++++++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 libs/notebook/fakefs_test.go diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index 0b7c04d6d..582a88479 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -12,27 +12,69 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +// FileInfoWithWorkspaceObjectInfo is an interface implemented by [fs.FileInfo] values that +// contain a file's underlying [workspace.ObjectInfo]. +// +// This may be the case when working with a [filer.Filer] backed by the workspace API. +// For these files we do not need to read a file's header to know if it is a notebook; +// we can use the [workspace.ObjectInfo] value directly. +type FileInfoWithWorkspaceObjectInfo interface { + WorkspaceObjectInfo() workspace.ObjectInfo +} + // Maximum length in bytes of the notebook header. const headerLength = 32 -// readHeader reads the first N bytes from a file. -func readHeader(fsys fs.FS, name string) ([]byte, error) { +// file wraps an fs.File and implements a few helper methods such that +// they don't need to be inlined in the [DetectWithFS] function below. +type file struct { + f fs.File +} + +func openFile(fsys fs.FS, name string) (*file, error) { f, err := fsys.Open(name) if err != nil { return nil, err } - defer f.Close() + return &file{f: f}, nil +} +func (f file) close() error { + return f.f.Close() +} + +func (f file) readHeader() (string, error) { // Scan header line with some padding. var buf = make([]byte, headerLength) - n, err := f.Read([]byte(buf)) + n, err := f.f.Read([]byte(buf)) if err != nil && err != io.EOF { - return nil, err + return "", err } // Trim buffer to actual read bytes. - return buf[:n], nil + buf = buf[:n] + + // Read the first line from the buffer. + scanner := bufio.NewScanner(bytes.NewReader(buf)) + scanner.Scan() + return scanner.Text(), nil +} + +// getObjectInfo returns the [workspace.ObjectInfo] for the file if it is +// part of the [fs.FileInfo] value returned by the [fs.Stat] call. +func (f file) getObjectInfo() (oi workspace.ObjectInfo, ok bool, err error) { + stat, err := f.f.Stat() + if err != nil { + return workspace.ObjectInfo{}, false, err + } + + // Use object info if available. + if i, ok := stat.(FileInfoWithWorkspaceObjectInfo); ok { + return i.WorkspaceObjectInfo(), true, nil + } + + return workspace.ObjectInfo{}, false, nil } // Detect returns whether the file at path is a Databricks notebook. @@ -40,13 +82,27 @@ func readHeader(fsys fs.FS, name string) ([]byte, error) { func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { header := "" - buf, err := readHeader(fsys, name) + f, err := openFile(fsys, name) + if err != nil { + return false, "", err + } + + defer f.close() + + // Use object info if available. + oi, ok, err := f.getObjectInfo() + if err != nil { + return false, "", err + } + if ok { + return oi.ObjectType == workspace.ObjectTypeNotebook, oi.Language, nil + } + + // Read the first line of the file. + fileHeader, err := f.readHeader() if err != nil { return false, "", err } - scanner := bufio.NewScanner(bytes.NewReader(buf)) - scanner.Scan() - fileHeader := scanner.Text() // Determine which header to expect based on filename extension. ext := strings.ToLower(filepath.Ext(name)) diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index fd3337579..ad89d6dd5 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -99,3 +99,21 @@ func TestDetectFileWithLongHeader(t *testing.T) { require.NoError(t, err) assert.False(t, nb) } + +func TestDetectWithObjectInfo(t *testing.T) { + fakeFS := &fakeFS{ + fakeFile{ + fakeFileInfo{ + workspace.ObjectInfo{ + ObjectType: workspace.ObjectTypeNotebook, + Language: workspace.LanguagePython, + }, + }, + }, + } + + nb, lang, err := DetectWithFS(fakeFS, "doesntmatter") + require.NoError(t, err) + assert.True(t, nb) + assert.Equal(t, workspace.LanguagePython, lang) +} diff --git a/libs/notebook/fakefs_test.go b/libs/notebook/fakefs_test.go new file mode 100644 index 000000000..4ac135dd4 --- /dev/null +++ b/libs/notebook/fakefs_test.go @@ -0,0 +1,77 @@ +package notebook + +import ( + "fmt" + "io/fs" + "time" + + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type fakeFS struct { + fakeFile +} + +type fakeFile struct { + fakeFileInfo +} + +func (f fakeFile) Close() error { + return nil +} + +func (f fakeFile) Read(p []byte) (n int, err error) { + return 0, fmt.Errorf("not implemented") +} + +func (f fakeFile) Stat() (fs.FileInfo, error) { + return f.fakeFileInfo, nil +} + +type fakeFileInfo struct { + oi workspace.ObjectInfo +} + +func (f fakeFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo { + return f.oi +} + +func (f fakeFileInfo) Name() string { + return "" +} + +func (f fakeFileInfo) Size() int64 { + return 0 +} + +func (f fakeFileInfo) Mode() fs.FileMode { + return 0 +} + +func (f fakeFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (f fakeFileInfo) IsDir() bool { + return false +} + +func (f fakeFileInfo) Sys() any { + return nil +} + +func (f fakeFS) Open(name string) (fs.File, error) { + return f.fakeFile, nil +} + +func (f fakeFS) Stat(name string) (fs.FileInfo, error) { + panic("not implemented") +} + +func (f fakeFS) ReadDir(name string) ([]fs.DirEntry, error) { + panic("not implemented") +} + +func (f fakeFS) ReadFile(name string) ([]byte, error) { + panic("not implemented") +} From 25737bbb5d7baf44d6bfe06cea32df8593a67c51 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 10 Jul 2024 08:38:06 +0200 Subject: [PATCH 07/88] Add regression tests for CLI error output (#1566) ## Changes Add regression tests for https://github.com/databricks/cli/issues/1563 We test 2 code paths: - if there is an error, we can print to stderr - if there is a valid output, we can print to stdout We should also consider adding black-box tests that will run the CLI binary as a black box and inspect its output to stderr/stdout. ## Tests Unit tests --- cmd/root/root.go | 7 ++----- internal/helpers.go | 34 ++++++++++++++++++++++---------- internal/unknown_command_test.go | 15 ++++++++++++++ main.go | 7 ++++++- 4 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 internal/unknown_command_test.go diff --git a/cmd/root/root.go b/cmd/root/root.go index 61baa4da0..eda873d12 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -92,9 +92,8 @@ func flagErrorFunc(c *cobra.Command, err error) error { // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute(cmd *cobra.Command) { +func Execute(ctx context.Context, cmd *cobra.Command) error { // TODO: deferred panic recovery - ctx := context.Background() // Run the command cmd, err := cmd.ExecuteContextC(ctx) @@ -118,7 +117,5 @@ func Execute(cmd *cobra.Command) { } } - if err != nil { - os.Exit(1) - } + return err } diff --git a/internal/helpers.go b/internal/helpers.go index 3923e7e1e..67a258ba4 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -19,6 +19,9 @@ import ( "testing" "time" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/cmd" _ "github.com/databricks/cli/cmd/version" "github.com/databricks/cli/libs/cmdio" @@ -105,7 +108,12 @@ func (t *cobraTestRunner) registerFlagCleanup(c *cobra.Command) { // Find target command that will be run. Example: if the command run is `databricks fs cp`, // target command corresponds to `cp` targetCmd, _, err := c.Find(t.args) - require.NoError(t, err) + if err != nil && strings.HasPrefix(err.Error(), "unknown command") { + // even if command is unknown, we can proceed + require.NotNil(t, targetCmd) + } else { + require.NoError(t, err) + } // Force initialization of default flags. // These are initialized by cobra at execution time and would otherwise @@ -169,22 +177,28 @@ func (t *cobraTestRunner) RunBackground() { var stdoutW, stderrW io.WriteCloser stdoutR, stdoutW = io.Pipe() stderrR, stderrW = io.Pipe() - root := cmd.New(t.ctx) - root.SetOut(stdoutW) - root.SetErr(stderrW) - root.SetArgs(t.args) + ctx := cmdio.NewContext(t.ctx, &cmdio.Logger{ + Mode: flags.ModeAppend, + Reader: bufio.Reader{}, + Writer: stderrW, + }) + + cli := cmd.New(ctx) + cli.SetOut(stdoutW) + cli.SetErr(stderrW) + cli.SetArgs(t.args) if t.stdinW != nil { - root.SetIn(t.stdinR) + cli.SetIn(t.stdinR) } // Register cleanup function to restore flags to their original values // once test has been executed. This is needed because flag values reside // in a global singleton data-structure, and thus subsequent tests might // otherwise interfere with each other - t.registerFlagCleanup(root) + t.registerFlagCleanup(cli) errch := make(chan error) - ctx, cancel := context.WithCancel(t.ctx) + ctx, cancel := context.WithCancel(ctx) // Tee stdout/stderr to buffers. stdoutR = io.TeeReader(stdoutR, &t.stdout) @@ -197,7 +211,7 @@ func (t *cobraTestRunner) RunBackground() { // Run command in background. go func() { - cmd, err := root.ExecuteContextC(ctx) + err := root.Execute(ctx, cli) if err != nil { t.Logf("Error running command: %s", err) } @@ -230,7 +244,7 @@ func (t *cobraTestRunner) RunBackground() { // These commands are globals so we have to clean up to the best of our ability after each run. // See https://github.com/spf13/cobra/blob/a6f198b635c4b18fff81930c40d464904e55b161/command.go#L1062-L1066 //lint:ignore SA1012 cobra sets the context and doesn't clear it - cmd.SetContext(nil) + cli.SetContext(nil) // Make caller aware of error. errch <- err diff --git a/internal/unknown_command_test.go b/internal/unknown_command_test.go new file mode 100644 index 000000000..62b84027f --- /dev/null +++ b/internal/unknown_command_test.go @@ -0,0 +1,15 @@ +package internal + +import ( + "testing" + + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestUnknownCommand(t *testing.T) { + stdout, stderr, err := RequireErrorRun(t, "unknown-command") + + assert.Error(t, err, "unknown command", `unknown command "unknown-command" for "databricks"`) + assert.Equal(t, "", stdout.String()) + assert.Contains(t, stderr.String(), "unknown command") +} diff --git a/main.go b/main.go index 8c8516d9d..c568e6adb 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,16 @@ package main import ( "context" + "os" "github.com/databricks/cli/cmd" "github.com/databricks/cli/cmd/root" ) func main() { - root.Execute(cmd.New(context.Background())) + ctx := context.Background() + err := root.Execute(ctx, cmd.New(ctx)) + if err != nil { + os.Exit(1) + } } From 1da04a43182c038ff9f4c1da87e5dd2a27025e38 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:27:27 +0530 Subject: [PATCH 08/88] Remove schema override for variable default value (#1536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This PR: 1. Removes the custom added in https://github.com/databricks/cli/pull/1396/files for `variables.*.default`. It's no longer needed because with complex variables (https://github.com/databricks/cli/pull/1467) `default` has a type of any. 2. Retains, and extends the override on `targets.*.variables.*`. Target override values can now be complex objects, not just primitive values. ## Tests Manually Before: Only primitive types were allowed. Screenshot 2024-06-27 at 3 58 34 PM After: An empty JSON schema is generated. All YAML values are acceptable. Screenshot 2024-06-27 at 3 57 15 PM --- cmd/bundle/schema.go | 54 +++----------------------------------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index b0d6b3dd5..813aebbae 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -11,54 +11,6 @@ import ( "github.com/spf13/cobra" ) -func overrideVariables(s *jsonschema.Schema) error { - // Override schema for default values to allow for multiple primitive types. - // These are normalized to strings when converted to the typed representation. - err := s.SetByPath("variables.*.default", jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: jsonschema.StringType, - }, - { - Type: jsonschema.BooleanType, - }, - { - Type: jsonschema.NumberType, - }, - { - Type: jsonschema.IntegerType, - }, - }, - }) - if err != nil { - return err - } - - // Override schema for variables in targets to allow just specifying the value - // along side overriding the variable definition if needed. - ns, err := s.GetByPath("variables.*") - if err != nil { - return err - } - return s.SetByPath("targets.*.variables.*", jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: jsonschema.StringType, - }, - { - Type: jsonschema.BooleanType, - }, - { - Type: jsonschema.NumberType, - }, - { - Type: jsonschema.IntegerType, - }, - &ns, - }, - }) -} - func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schema", @@ -79,9 +31,9 @@ func newSchemaCommand() *cobra.Command { return err } - // Override schema for variables to take into account normalization of default - // variable values and variable overrides in a target. - err = overrideVariables(schema) + // Target variable value overrides can be primitives, maps or sequences. + // Set an empty schema for them. + err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) if err != nil { return err } From af975ca64ba16cbad58a657d39bc6063b4d71311 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 10 Jul 2024 13:14:57 +0200 Subject: [PATCH 09/88] Print diagnostics in 'bundle deploy' (#1579) ## Changes Print diagnostics in 'bundle deploy' similar to 'bundle validate'. This way if a bundle has any errors or warnings, they are going to be easy to notice. NB: due to how we render errors, there is one extra trailing new line in output, preserved in examples below ## Example: No errors or warnings ``` % databricks bundle deploy Building default... Deploying resources... Updating deployment state... Deployment complete! ``` ## Example: Error on load ``` % databricks bundle deploy Error: Databricks CLI version constraint not satisfied. Required: >= 1337.0.0, current: 0.0.0-dev ``` ## Example: Warning on load ``` % databricks bundle deploy Building default... Deploying resources... Updating deployment state... Deployment complete! Warning: unknown field: foo in databricks.yml:6:1 ``` ## Example: Error + warning on load ``` % databricks bundle deploy Warning: unknown field: foo in databricks.yml:6:1 Error: something went wrong ``` ## Example: Warning on load + error in init ``` % databricks bundle deploy Warning: unknown field: foo in databricks.yml:6:1 Error: Failed to xxx in yyy.yml Detailed explanation in multiple lines ``` ## Tests Tested manually --- bundle/render/render_text_output.go | 19 +++++--- bundle/render/render_text_output_test.go | 46 ++++++++++++++++++- bundle/scripts/scripts.go | 9 +++- cmd/bundle/deploy.go | 57 ++++++++++++++---------- cmd/bundle/validate.go | 3 +- 5 files changed, 102 insertions(+), 32 deletions(-) diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 37ea188f7..439ae6132 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -142,7 +142,7 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) } // Make file relative to bundle root - if d.Location.File != "" { + if d.Location.File != "" && b != nil { out, err := filepath.Rel(b.RootPath, d.Location.File) // if we can't relativize the path, just use path as-is if err == nil { @@ -160,16 +160,25 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) return nil } +// RenderOptions contains options for rendering diagnostics. +type RenderOptions struct { + // variable to include leading new line + + RenderSummaryTable bool +} + // RenderTextOutput renders the diagnostics in a human-readable format. -func RenderTextOutput(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { +func RenderTextOutput(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics, opts RenderOptions) error { err := renderDiagnostics(out, b, diags) if err != nil { return fmt.Errorf("failed to render diagnostics: %w", err) } - err = renderSummaryTemplate(out, b, diags) - if err != nil { - return fmt.Errorf("failed to render summary: %w", err) + if opts.RenderSummaryTable { + err = renderSummaryTemplate(out, b, diags) + if err != nil { + return fmt.Errorf("failed to render summary: %w", err) + } } return nil diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 4ae86ded7..b7aec8864 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -17,6 +17,7 @@ type renderTestOutputTestCase struct { name string bundle *bundle.Bundle diags diag.Diagnostics + opts RenderOptions expected string } @@ -39,6 +40,7 @@ func TestRenderTextOutput(t *testing.T) { Summary: "failed to load xxx", }, }, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: failed to load xxx\n" + "\n" + "Found 1 error\n", @@ -47,6 +49,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 1 error", bundle: loadingBundle, diags: diag.Errorf("failed to load bundle"), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: failed to load bundle\n" + "\n" + "Name: test-bundle\n" + @@ -58,6 +61,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 1 warning", bundle: loadingBundle, diags: diag.Warningf("failed to load bundle"), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Warning: failed to load bundle\n" + "\n" + "Name: test-bundle\n" + @@ -69,6 +73,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 2 warnings", bundle: loadingBundle, diags: diag.Warningf("warning (1)").Extend(diag.Warningf("warning (2)")), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Warning: warning (1)\n" + "\n" + "Warning: warning (2)\n" + @@ -113,6 +118,7 @@ func TestRenderTextOutput(t *testing.T) { }, }, }, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: error (1)\n" + " in foo.py:1:1\n" + "\n" + @@ -153,6 +159,7 @@ func TestRenderTextOutput(t *testing.T) { }, }, diags: nil, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Name: test-bundle\n" + "Target: test-target\n" + "Workspace:\n" + @@ -162,13 +169,50 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "Validation OK!\n", }, + { + name: "nil bundle without summary with 1 error and 1 warning", + bundle: nil, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Location: dyn.Location{ + File: "foo.py", + Line: 1, + Column: 1, + }, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Location: dyn.Location{ + File: "foo.py", + Line: 3, + Column: 1, + }, + }, + }, + opts: RenderOptions{RenderSummaryTable: false}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:3:1\n" + + "\n" + + "detail (2)\n" + + "\n", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { writer := &bytes.Buffer{} - err := RenderTextOutput(writer, tc.bundle, tc.diags) + err := RenderTextOutput(writer, tc.bundle, tc.diags, tc.opts) require.NoError(t, err) assert.Equal(t, tc.expected, writer.String()) diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 38d204f99..629b3a8ab 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -37,7 +37,7 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmd, out, err := executeHook(ctx, executor, b, m.scriptHook) if err != nil { - return diag.FromErr(err) + return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) } if cmd == nil { log.Debugf(ctx, "No script defined for %s, skipping", m.scriptHook) @@ -53,7 +53,12 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { line, err = reader.ReadString('\n') } - return diag.FromErr(cmd.Wait()) + err = cmd.Wait() + if err != nil { + return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) + } + + return nil } func executeHook(ctx context.Context, executor *exec.Executor, b *bundle.Bundle, hook config.ScriptHook) (exec.Command, io.Reader, error) { diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 919b15a72..1232c8de5 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -2,9 +2,11 @@ package bundle import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" @@ -30,32 +32,41 @@ func newDeployCommand() *cobra.Command { cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() b, diags := utils.ConfigureBundleWithVariables(cmd) - if err := diags.Error(); err != nil { - return diags.Error() + + if !diags.HasError() { + bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { + b.Config.Bundle.Force = force + b.Config.Bundle.Deployment.Lock.Force = forceLock + if cmd.Flag("compute-id").Changed { + b.Config.Bundle.ComputeID = computeID + } + + if cmd.Flag("fail-on-active-runs").Changed { + b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns + } + + return nil + }) + + diags = diags.Extend( + bundle.Apply(ctx, b, bundle.Seq( + phases.Initialize(), + phases.Build(), + phases.Deploy(), + )), + ) } - bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { - b.Config.Bundle.Force = force - b.Config.Bundle.Deployment.Lock.Force = forceLock - if cmd.Flag("compute-id").Changed { - b.Config.Bundle.ComputeID = computeID - } - - if cmd.Flag("fail-on-active-runs").Changed { - b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns - } - - return nil - }) - - diags = bundle.Apply(ctx, b, bundle.Seq( - phases.Initialize(), - phases.Build(), - phases.Deploy(), - )) - if err := diags.Error(); err != nil { - return err + renderOpts := render.RenderOptions{RenderSummaryTable: false} + err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) + if err != nil { + return fmt.Errorf("failed to render output: %w", err) } + + if diags.HasError() { + return root.ErrAlreadyPrinted + } + return nil } diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 59a977047..496d5d2b5 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -53,7 +53,8 @@ func newValidateCommand() *cobra.Command { switch root.OutputType(cmd) { case flags.OutputText: - err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags) + renderOpts := render.RenderOptions{RenderSummaryTable: true} + err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) if err != nil { return fmt.Errorf("failed to render output: %w", err) } From 61cb0f269526ad0bc18e43f6cf17acea52cd4093 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 10 Jul 2024 14:04:59 +0200 Subject: [PATCH 10/88] Release v0.223.2 (#1587) Bundles: * Override complex variables with target overrides instead of merging ([#1567](https://github.com/databricks/cli/pull/1567)). * Rewrite local path for libraries in foreach tasks ([#1569](https://github.com/databricks/cli/pull/1569)). * Change SetVariables mutator to mutate dynamic configuration instead ([#1573](https://github.com/databricks/cli/pull/1573)). * Return early in bundle destroy if no deployment exists ([#1581](https://github.com/databricks/cli/pull/1581)). * Let notebook detection code use underlying metadata if available ([#1574](https://github.com/databricks/cli/pull/1574)). * Remove schema override for variable default value ([#1536](https://github.com/databricks/cli/pull/1536)). * Print diagnostics in 'bundle deploy' ([#1579](https://github.com/databricks/cli/pull/1579)). Internal: * Update actions/upload-artifact to v4 ([#1559](https://github.com/databricks/cli/pull/1559)). * Use Go 1.22 to build and test ([#1562](https://github.com/databricks/cli/pull/1562)). * Move bespoke status call to main workspace files filer ([#1570](https://github.com/databricks/cli/pull/1570)). * Add new template ([#1578](https://github.com/databricks/cli/pull/1578)). * Add regression tests for CLI error output ([#1566](https://github.com/databricks/cli/pull/1566)). Dependency updates: * Bump golang.org/x/mod from 0.18.0 to 0.19.0 ([#1576](https://github.com/databricks/cli/pull/1576)). * Bump golang.org/x/term from 0.21.0 to 0.22.0 ([#1577](https://github.com/databricks/cli/pull/1577)). --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d81f822..eb902e0b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Version changelog +## 0.223.2 + +Bundles: + * Override complex variables with target overrides instead of merging ([#1567](https://github.com/databricks/cli/pull/1567)). + * Rewrite local path for libraries in foreach tasks ([#1569](https://github.com/databricks/cli/pull/1569)). + * Change SetVariables mutator to mutate dynamic configuration instead ([#1573](https://github.com/databricks/cli/pull/1573)). + * Return early in bundle destroy if no deployment exists ([#1581](https://github.com/databricks/cli/pull/1581)). + * Let notebook detection code use underlying metadata if available ([#1574](https://github.com/databricks/cli/pull/1574)). + * Remove schema override for variable default value ([#1536](https://github.com/databricks/cli/pull/1536)). + * Print diagnostics in 'bundle deploy' ([#1579](https://github.com/databricks/cli/pull/1579)). + +Internal: + * Update actions/upload-artifact to v4 ([#1559](https://github.com/databricks/cli/pull/1559)). + * Use Go 1.22 to build and test ([#1562](https://github.com/databricks/cli/pull/1562)). + * Move bespoke status call to main workspace files filer ([#1570](https://github.com/databricks/cli/pull/1570)). + * Add new template ([#1578](https://github.com/databricks/cli/pull/1578)). + * Add regression tests for CLI error output ([#1566](https://github.com/databricks/cli/pull/1566)). + +Dependency updates: + * Bump golang.org/x/mod from 0.18.0 to 0.19.0 ([#1576](https://github.com/databricks/cli/pull/1576)). + * Bump golang.org/x/term from 0.21.0 to 0.22.0 ([#1577](https://github.com/databricks/cli/pull/1577)). + ## 0.223.1 This bugfix release fixes missing error messages in v0.223.0. From 434bcbb01858b57b6d1d4b8f1f2a3201dc9bd82b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 16 Jul 2024 10:57:04 +0200 Subject: [PATCH 11/88] Allow artifacts (JARs, wheels) to be uploaded to UC Volumes (#1591) ## Changes This change allows to specify UC volumes path as an artifact paths so all artifacts (JARs, wheels) are uploaded to UC Volumes. Example configuration is here: ``` bundle: name: jar-bundle workspace: host: https://foo.com artifact_path: /Volumes/main/default/foobar artifacts: my_java_code: path: ./sample-java build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class" files: - source: ./sample-java/PrintArgs.jar resources: jobs: jar_job: name: "Test Spark Jar Job" tasks: - task_key: TestSparkJarTask new_cluster: num_workers: 1 spark_version: "14.3.x-scala2.12" node_type_id: "i3.xlarge" spark_jar_task: main_class_name: PrintArgs libraries: - jar: ./sample-java/PrintArgs.jar ``` ## Tests Manually + added E2E test for Java jobs E2E test is temporarily skipped until auth related issues for UC for tests are resolved --- bundle/artifacts/artifacts.go | 24 ++++- bundle/artifacts/artifacts_test.go | 91 ++++++++++++++++++- bundle/artifacts/build.go | 64 ++++++++----- bundle/artifacts/upload.go | 19 ++-- internal/bundle/artifacts_test.go | 69 ++++++++++++++ .../databricks_template_schema.json | 29 ++++++ .../template/databricks.yml.tmpl | 28 ++++++ .../{{.project_name}}/META-INF/MANIFEST.MF | 1 + .../template/{{.project_name}}/PrintArgs.java | 8 ++ internal/bundle/helpers.go | 6 +- internal/bundle/spark_jar_test.go | 52 +++++++++++ internal/helpers.go | 4 +- 12 files changed, 357 insertions(+), 38 deletions(-) create mode 100644 internal/bundle/bundles/spark_jar_task/databricks_template_schema.json create mode 100644 internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF create mode 100644 internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java create mode 100644 internal/bundle/spark_jar_test.go diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index a5f41ae4b..15565cd60 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts/whl" @@ -17,6 +18,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" ) type mutatorFactory = func(name string) bundle.Mutator @@ -103,7 +105,7 @@ func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return diag.FromErr(err) } - client, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath) + client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) if err != nil { return diag.FromErr(err) } @@ -116,6 +118,17 @@ func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return nil } +func getFilerForArtifacts(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { + if isVolumesPath(uploadPath) { + return filer.NewFilesClient(w, uploadPath) + } + return filer.NewWorkspaceFilesClient(w, uploadPath) +} + +func isVolumesPath(path string) bool { + return strings.HasPrefix(path, "/Volumes/") +} + func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, uploadPath string, client filer.Filer) error { for i := range a.Files { f := &a.Files[i] @@ -130,14 +143,15 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u log.Infof(ctx, "Upload succeeded") f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source)) + remotePath := f.RemotePath - // TODO: confirm if we still need to update the remote path to start with /Workspace - wsfsBase := "/Workspace" - remotePath := path.Join(wsfsBase, f.RemotePath) + if !strings.HasPrefix(f.RemotePath, "/Workspace/") && !strings.HasPrefix(f.RemotePath, "/Volumes/") { + wsfsBase := "/Workspace" + remotePath = path.Join(wsfsBase, f.RemotePath) + } for _, job := range b.Config.Resources.Jobs { rewriteArtifactPath(b, f, job, remotePath) - } } diff --git a/bundle/artifacts/artifacts_test.go b/bundle/artifacts/artifacts_test.go index 53c2798ed..6d85f3af9 100644 --- a/bundle/artifacts/artifacts_test.go +++ b/bundle/artifacts/artifacts_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestArtifactUpload(t *testing.T) { +func TestArtifactUploadForWorkspace(t *testing.T) { tmpDir := t.TempDir() whlFolder := filepath.Join(tmpDir, "whl") testutil.Touch(t, whlFolder, "source.whl") @@ -105,3 +105,92 @@ func TestArtifactUpload(t *testing.T) { require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) } + +func TestArtifactUploadForVolumes(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + artifact := b.Config.Artifacts["whl"] + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*bytes.Reader"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + err := uploadArtifact(context.Background(), b, artifact, "/Volumes/foo/bar/artifacts", mockFiler) + require.NoError(t, err) + + // Test that libraries path is updated + require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 722891ada..c8c3bf67c 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -44,27 +44,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } } - // Expand any glob reference in files source path - files := make([]config.ArtifactFile, 0, len(artifact.Files)) - for _, f := range artifact.Files { - matches, err := filepath.Glob(f.Source) - if err != nil { - return diag.Errorf("unable to find files for %s: %v", f.Source, err) - } - - if len(matches) == 0 { - return diag.Errorf("no files found for %s", f.Source) - } - - for _, match := range matches { - files = append(files, config.ArtifactFile{ - Source: match, - }) - } - } - - artifact.Files = files - // Skip building if build command is not specified or infered if artifact.BuildCommand == "" { // If no build command was specified or infered and there is no @@ -72,7 +51,11 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if len(artifact.Files) == 0 { return diag.Errorf("misconfigured artifact: please specify 'build' or 'files' property") } - return nil + + // We can skip calling build mutator if there is no build command + // But we still need to expand glob references in files source path. + diags := expandGlobReference(artifact) + return diags } // If artifact path is not provided, use bundle root dir @@ -85,5 +68,40 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { artifact.Path = filepath.Join(dirPath, artifact.Path) } - return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) + diags := bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) + if diags.HasError() { + return diags + } + + // We need to expand glob reference after build mutator is applied because + // if we do it before, any files that are generated by build command will + // not be included into artifact.Files and thus will not be uploaded. + d := expandGlobReference(artifact) + return diags.Extend(d) +} + +func expandGlobReference(artifact *config.Artifact) diag.Diagnostics { + var diags diag.Diagnostics + + // Expand any glob reference in files source path + files := make([]config.ArtifactFile, 0, len(artifact.Files)) + for _, f := range artifact.Files { + matches, err := filepath.Glob(f.Source) + if err != nil { + return diags.Extend(diag.Errorf("unable to find files for %s: %v", f.Source, err)) + } + + if len(matches) == 0 { + return diags.Extend(diag.Errorf("no files found for %s", f.Source)) + } + + for _, match := range matches { + files = append(files, config.ArtifactFile{ + Source: match, + }) + } + } + + artifact.Files = files + return diags } diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 5c12c9444..3af50021e 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -6,7 +6,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" ) func UploadAll() bundle.Mutator { @@ -57,12 +58,18 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diag.FromErr(err) } - b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{ - Path: uploadPath, - Recursive: true, - }) + client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) + if err != nil { + return diag.FromErr(err) + } - err = b.WorkspaceClient().Workspace.MkdirsByPath(ctx, uploadPath) + // We intentionally ignore the error because it is not critical to the deployment + err = client.Delete(ctx, ".", filer.DeleteRecursively) + if err != nil { + log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err) + } + + err = client.Mkdir(ctx, ".") if err != nil { return diag.Errorf("unable to create directory for %s: %v", uploadPath, err) } diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 222b23047..46c236a4e 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -153,3 +153,72 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) b.Config.Resources.Jobs["test"].JobSettings.Environments[0].Spec.Dependencies[0], ) } + +func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") + } + + volumePath := internal.TemporaryUcVolume(t, w) + + dir := t.TempDir() + whlPath := filepath.Join(dir, "dist", "test.whl") + touchEmptyFile(t, whlPath) + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + }, + Workspace: config.Workspace{ + ArtifactPath: volumePath, + }, + Artifacts: config.Artifacts{ + "test": &config.Artifact{ + Type: "whl", + Files: []config.ArtifactFile{ + { + Source: whlPath, + }, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "dist/test.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + require.NoError(t, diags.Error()) + + // The remote path attribute on the artifact file should have been set. + require.Regexp(t, + regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + b.Config.Artifacts["test"].Files[0].RemotePath, + ) + + // The task library path should have been updated to the remote path. + require.Regexp(t, + regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, + ) +} diff --git a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json new file mode 100644 index 000000000..078dff976 --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json @@ -0,0 +1,29 @@ +{ + "properties": { + "project_name": { + "type": "string", + "default": "my_java_project", + "description": "Unique name for this project" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + }, + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "root": { + "type": "string", + "description": "Path to the root of the template" + }, + "artifact_path": { + "type": "string", + "description": "Path to the remote base path for artifacts" + } + } +} diff --git a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl new file mode 100644 index 000000000..24a6d7d8a --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl @@ -0,0 +1,28 @@ +bundle: + name: spark-jar-task + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + artifact_path: {{.artifact_path}} + +artifacts: + my_java_code: + path: ./{{.project_name}} + build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class" + files: + - source: ./{{.project_name}}/PrintArgs.jar + +resources: + jobs: + jar_job: + name: "[${bundle.target}] Test Spark Jar Job {{.unique_id}}" + tasks: + - task_key: TestSparkJarTask + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + spark_jar_task: + main_class_name: PrintArgs + libraries: + - jar: ./{{.project_name}}/PrintArgs.jar diff --git a/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF new file mode 100644 index 000000000..40b023dbd --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Main-Class: PrintArgs \ No newline at end of file diff --git a/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java new file mode 100644 index 000000000..b7430f25f --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java @@ -0,0 +1,8 @@ +import java.util.Arrays; + +public class PrintArgs { + public static void main(String[] args) { + System.out.println("Hello from Jar!"); + System.out.println(Arrays.toString(args)); + } +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index a17964b16..c33c15331 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -21,9 +21,13 @@ import ( const defaultSparkVersion = "13.3.x-snapshot-scala2.12" func initTestTemplate(t *testing.T, ctx context.Context, templateName string, config map[string]any) (string, error) { + bundleRoot := t.TempDir() + return initTestTemplateWithBundleRoot(t, ctx, templateName, config, bundleRoot) +} + +func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateName string, config map[string]any, bundleRoot string) (string, error) { templateRoot := filepath.Join("bundles", templateName) - bundleRoot := t.TempDir() configFilePath, err := writeConfigFile(t, config) if err != nil { return "", err diff --git a/internal/bundle/spark_jar_test.go b/internal/bundle/spark_jar_test.go new file mode 100644 index 000000000..c981e7750 --- /dev/null +++ b/internal/bundle/spark_jar_test.go @@ -0,0 +1,52 @@ +package bundle + +import ( + "os" + "testing" + + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func runSparkJarTest(t *testing.T, sparkVersion string) { + t.Skip("Temporarily skipping the test until auth / permission issues for UC volumes are resolved.") + + env := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + t.Log(env) + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") + } + + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + volumePath := internal.TemporaryUcVolume(t, w) + + nodeTypeId := internal.GetNodeTypeId(env) + tmpDir := t.TempDir() + bundleRoot, err := initTestTemplateWithBundleRoot(t, ctx, "spark_jar_task", map[string]any{ + "node_type_id": nodeTypeId, + "unique_id": uuid.New().String(), + "spark_version": sparkVersion, + "root": tmpDir, + "artifact_path": volumePath, + }, tmpDir) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + out, err := runResource(t, ctx, bundleRoot, "jar_job") + require.NoError(t, err) + require.Contains(t, out, "Hello from Jar!") +} + +func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { + runSparkJarTest(t, "14.3.x-scala2.12") +} diff --git a/internal/helpers.go b/internal/helpers.go index 67a258ba4..972a2322b 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -472,7 +472,7 @@ func TemporaryDbfsDir(t *testing.T, w *databricks.WorkspaceClient) string { } // Create a new UC volume in a catalog called "main" in the workspace. -func temporaryUcVolume(t *testing.T, w *databricks.WorkspaceClient) string { +func TemporaryUcVolume(t *testing.T, w *databricks.WorkspaceClient) string { ctx := context.Background() // Create a schema @@ -607,7 +607,7 @@ func setupUcVolumesFiler(t *testing.T) (filer.Filer, string) { w, err := databricks.NewWorkspaceClient() require.NoError(t, err) - tmpDir := temporaryUcVolume(t, w) + tmpDir := TemporaryUcVolume(t, w) f, err := filer.NewFilesClient(w, tmpDir) require.NoError(t, err) From 39c2633773bd6a90cfe85216db34e2e0adbeca6b Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:31:58 +0530 Subject: [PATCH 12/88] Add UUID to uniquely identify a deployment state (#1595) ## Changes We need a mechanism to invalidate the locally cached deployment state if a user uses the same working directory to deploy to multiple distinct deployments (separate targets, root_paths or even hosts). This PR just adds the UUID to the deployment state in preparation for invalidating this cache. The actual invalidation will follow up at a later date (tracked in internal backlog). ## Tests Unit test. Manually checked the deployment state is actually being written. --- bundle/deploy/state.go | 4 ++++ bundle/deploy/state_update.go | 6 ++++++ bundle/deploy/state_update_test.go | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/bundle/deploy/state.go b/bundle/deploy/state.go index 97048811b..4f2bc4ee4 100644 --- a/bundle/deploy/state.go +++ b/bundle/deploy/state.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/vfs" + "github.com/google/uuid" ) const DeploymentStateFileName = "deployment.json" @@ -46,6 +47,9 @@ type DeploymentState struct { // Files is a list of files which has been deployed as part of this deployment. Files Filelist `json:"files"` + + // UUID uniquely identifying the deployment. + ID uuid.UUID `json:"id"` } // We use this entry type as a proxy to fs.DirEntry. diff --git a/bundle/deploy/state_update.go b/bundle/deploy/state_update.go index bfdb308c4..9ab1bacf1 100644 --- a/bundle/deploy/state_update.go +++ b/bundle/deploy/state_update.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" + "github.com/google/uuid" ) type stateUpdate struct { @@ -46,6 +47,11 @@ func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost } state.Files = fl + // Generate a UUID for the deployment, if one does not already exist + if state.ID == uuid.Nil { + state.ID = uuid.New() + } + statePath, err := getPathToStateFile(ctx, b) if err != nil { return diag.FromErr(err) diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index ed72439d2..2982546d5 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -88,6 +89,9 @@ func TestStateUpdate(t *testing.T) { }, }) require.Equal(t, build.GetInfo().Version, state.CliVersion) + + // Valid non-empty UUID is generated. + require.NotEqual(t, uuid.Nil, state.ID) } func TestStateUpdateWithExistingState(t *testing.T) { @@ -109,6 +113,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { LocalPath: "bar/t1.py", }, }, + ID: uuid.MustParse("123e4567-e89b-12d3-a456-426614174000"), } data, err := json.Marshal(state) @@ -135,4 +140,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { }, }) require.Equal(t, build.GetInfo().Version, state.CliVersion) + + // Existing UUID is not overwritten. + require.Equal(t, uuid.MustParse("123e4567-e89b-12d3-a456-426614174000"), state.ID) } From 8ed996448206e5870a1d026331a88fd6392c3ede Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:57:27 +0530 Subject: [PATCH 13/88] Track multiple locations associated with a `dyn.Value` (#1510) ## Changes This PR changes the location metadata associated with a `dyn.Value` to a slice of locations. This will allow us to keep track of location metadata across merges and overrides. The convention is to treat the first location in the slice as the primary location. Also, the semantics are the same as before if there's only one location associated with a value, that is: 1. For complex values (maps, sequences) the location of the v1 is primary in Merge(v1, v2) 2. For primitive values the location of v2 is primary in Merge(v1, v2) ## Tests Modifying existing merge unit tests. Other existing unit tests and integration tests pass. --------- Co-authored-by: Pieter Noordhuis --- bundle/config/generate/job.go | 2 +- .../mutator/expand_pipeline_glob_paths.go | 4 +- .../mutator/python/python_mutator_test.go | 20 +- bundle/config/mutator/rewrite_sync_paths.go | 2 +- bundle/config/mutator/translate_paths.go | 2 +- bundle/config/root.go | 10 +- bundle/internal/bundletest/location.go | 4 +- libs/dyn/convert/from_typed.go | 4 +- libs/dyn/convert/from_typed_test.go | 60 ++--- libs/dyn/convert/normalize.go | 16 +- libs/dyn/convert/normalize_test.go | 54 ++-- libs/dyn/dynvar/resolve.go | 4 +- libs/dyn/merge/elements_by_key.go | 2 +- libs/dyn/merge/merge.go | 35 ++- libs/dyn/merge/merge_test.go | 138 +++++++++-- libs/dyn/merge/override.go | 4 +- libs/dyn/merge/override_test.go | 233 +++++++++--------- libs/dyn/pattern.go | 4 +- libs/dyn/value.go | 48 +++- libs/dyn/value_test.go | 11 +- libs/dyn/value_underlying_test.go | 4 +- libs/dyn/visit_map.go | 4 +- libs/dyn/yamlloader/loader.go | 22 +- libs/dyn/yamlsaver/saver_test.go | 65 ++--- libs/dyn/yamlsaver/utils.go | 2 +- libs/dyn/yamlsaver/utils_test.go | 31 ++- 26 files changed, 472 insertions(+), 313 deletions(-) diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 3ab5e0122..28bc86412 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -22,7 +22,7 @@ func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { tasks = append(tasks, v) } // We're using location lines to define the order of keys in exported YAML. - value["tasks"] = dyn.NewValue(tasks, dyn.Location{Line: jobOrder.Get("tasks")}) + value["tasks"] = dyn.NewValue(tasks, []dyn.Location{{Line: jobOrder.Get("tasks")}}) } return yamlsaver.ConvertToMapValue(job.Settings, jobOrder, []string{"format", "new_cluster", "existing_cluster_id"}, value) diff --git a/bundle/config/mutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/expand_pipeline_glob_paths.go index 268d8fa48..5703332fa 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths.go @@ -59,7 +59,7 @@ func (m *expandPipelineGlobPaths) expandLibrary(v dyn.Value) ([]dyn.Value, error if err != nil { return nil, err } - nv, err := dyn.SetByPath(v, p, dyn.NewValue(m, pv.Location())) + nv, err := dyn.SetByPath(v, p, dyn.NewValue(m, pv.Locations())) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (m *expandPipelineGlobPaths) expandSequence(p dyn.Path, v dyn.Value) (dyn.V vs = append(vs, v...) } - return dyn.NewValue(vs, v.Location()), nil + return dyn.NewValue(vs, v.Locations()), nil } func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 9a0ed8c3a..588589831 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -305,8 +305,8 @@ type createOverrideVisitorTestCase struct { } func TestCreateOverrideVisitor(t *testing.T) { - left := dyn.NewValue(42, dyn.Location{}) - right := dyn.NewValue(1337, dyn.Location{}) + left := dyn.V(42) + right := dyn.V(1337) testCases := []createOverrideVisitorTestCase{ { @@ -470,21 +470,21 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { // this is not happening, but adding for completeness name: "undo delete of empty variables", path: dyn.MustPathFromString("variables"), - left: dyn.NewValue([]dyn.Value{}, location), + left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "undo delete of empty job clusters", path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{}, location), + left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "allow delete of non-empty job clusters", path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", location)}, location), + left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", []dyn.Location{location})}, []dyn.Location{location}), expectedErr: nil, // deletions aren't allowed in 'load' phase phases: []phase{PythonMutatorPhaseInit}, @@ -492,17 +492,15 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { { name: "undo delete of empty tags", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{}, location), + left: dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "allow delete of non-empty tags", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue( - map[string]dyn.Value{"dev": dyn.NewValue("true", location)}, - location, - ), + left: dyn.NewValue(map[string]dyn.Value{"dev": dyn.NewValue("true", []dyn.Location{location})}, []dyn.Location{location}), + expectedErr: nil, // deletions aren't allowed in 'load' phase phases: []phase{PythonMutatorPhaseInit}, @@ -510,7 +508,7 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { { name: "undo delete of nil", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NilValue.WithLocation(location), + left: dyn.NilValue.WithLocations([]dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 85db79797..cfdc55f36 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -38,7 +38,7 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { return dyn.InvalidValue, err } - return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Location()), nil + return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Locations()), nil } } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a01d3d6a7..28f7d3d30 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -182,7 +182,7 @@ func (t *translateContext) rewriteValue(p dyn.Path, v dyn.Value, fn rewriteFunc, return dyn.InvalidValue, err } - return dyn.NewValue(out, v.Location()), nil + return dyn.NewValue(out, v.Locations()), nil } func (t *translateContext) rewriteRelativeTo(p dyn.Path, v dyn.Value, fn rewriteFunc, dir, fallback string) (dyn.Value, error) { diff --git a/bundle/config/root.go b/bundle/config/root.go index 2bbb78696..594a9105f 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -378,7 +378,7 @@ func (r *Root) MergeTargetOverrides(name string) error { // Below, we're setting fields on the bundle key, so make sure it exists. if root.Get("bundle").Kind() == dyn.KindInvalid { - root, err = dyn.Set(root, "bundle", dyn.NewValue(map[string]dyn.Value{}, dyn.Location{})) + root, err = dyn.Set(root, "bundle", dyn.V(map[string]dyn.Value{})) if err != nil { return err } @@ -404,7 +404,7 @@ func (r *Root) MergeTargetOverrides(name string) error { if v := target.Get("git"); v.Kind() != dyn.KindInvalid { ref, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git"))) if err != nil { - ref = dyn.NewValue(map[string]dyn.Value{}, dyn.Location{}) + ref = dyn.V(map[string]dyn.Value{}) } // Merge the override into the reference. @@ -415,7 +415,7 @@ func (r *Root) MergeTargetOverrides(name string) error { // If the branch was overridden, we need to clear the inferred flag. if branch := v.Get("branch"); branch.Kind() != dyn.KindInvalid { - out, err = dyn.SetByPath(out, dyn.NewPath(dyn.Key("inferred")), dyn.NewValue(false, dyn.Location{})) + out, err = dyn.SetByPath(out, dyn.NewPath(dyn.Key("inferred")), dyn.V(false)) if err != nil { return err } @@ -456,7 +456,7 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { // configuration will convert this to a string if necessary. return dyn.NewValue(map[string]dyn.Value{ "default": variable, - }, variable.Location()), nil + }, variable.Locations()), nil case dyn.KindMap, dyn.KindSequence: // Check if the original definition of variable has a type field. @@ -469,7 +469,7 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { return dyn.NewValue(map[string]dyn.Value{ "type": typeV, "default": variable, - }, variable.Location()), nil + }, variable.Locations()), nil } return variable, nil diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index 1fd6f968c..ebec43d30 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -14,9 +14,9 @@ func SetLocation(b *bundle.Bundle, prefix string, filePath string) { return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { // If the path has the given prefix, set the location. if p.HasPrefix(start) { - return v.WithLocation(dyn.Location{ + return v.WithLocations([]dyn.Location{{ File: filePath, - }), nil + }}), nil } // The path is not nested under the given prefix. diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index e8d321f66..cd92ad0eb 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -42,7 +42,7 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, // Dereference pointer if necessary for srcv.Kind() == reflect.Pointer { if srcv.IsNil() { - return dyn.NilValue.WithLocation(ref.Location()), nil + return dyn.NilValue.WithLocations(ref.Locations()), nil } srcv = srcv.Elem() @@ -83,7 +83,7 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, if err != nil { return dyn.InvalidValue, err } - return v.WithLocation(ref.Location()), err + return v.WithLocations(ref.Locations()), err } func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go index 9141a6948..0cddff3be 100644 --- a/libs/dyn/convert/from_typed_test.go +++ b/libs/dyn/convert/from_typed_test.go @@ -115,16 +115,16 @@ func TestFromTypedStructSetFieldsRetainLocation(t *testing.T) { } ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", dyn.Location{File: "foo"}), - "bar": dyn.NewValue("baz", dyn.Location{File: "bar"}), + "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), + "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their location. - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", dyn.Location{File: "bar"}), nv.Get("bar")) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) + assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) } func TestFromTypedStringMapWithZeroValue(t *testing.T) { @@ -359,16 +359,16 @@ func TestFromTypedMapNonEmptyRetainLocation(t *testing.T) { } ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", dyn.Location{File: "foo"}), - "bar": dyn.NewValue("baz", dyn.Location{File: "bar"}), + "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), + "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", dyn.Location{File: "bar"}), nv.Get("bar")) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) + assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) } func TestFromTypedMapFieldWithZeroValue(t *testing.T) { @@ -432,16 +432,16 @@ func TestFromTypedSliceNonEmptyRetainLocation(t *testing.T) { } ref := dyn.V([]dyn.Value{ - dyn.NewValue("foo", dyn.Location{File: "foo"}), - dyn.NewValue("bar", dyn.Location{File: "bar"}), + dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), + dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("foo", dyn.Location{File: "foo"}), nv.Index(0)) - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "bar"}), nv.Index(1)) + assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv.Index(0)) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), nv.Index(1)) } func TestFromTypedStringEmpty(t *testing.T) { @@ -477,19 +477,19 @@ func TestFromTypedStringNonEmptyOverwrite(t *testing.T) { } func TestFromTypedStringRetainsLocations(t *testing.T) { - var ref = dyn.NewValue("foo", dyn.Location{File: "foo"}) + var ref = dyn.NewValue("foo", []dyn.Location{{File: "foo"}}) // case: value has not been changed var src string = "foo" nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue("foo", dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = "bar" nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedStringTypeError(t *testing.T) { @@ -532,19 +532,19 @@ func TestFromTypedBoolNonEmptyOverwrite(t *testing.T) { } func TestFromTypedBoolRetainsLocations(t *testing.T) { - var ref = dyn.NewValue(true, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(true, []dyn.Location{{File: "foo"}}) // case: value has not been changed var src bool = true nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(true, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(true, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = false nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(false, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedBoolVariableReference(t *testing.T) { @@ -595,19 +595,19 @@ func TestFromTypedIntNonEmptyOverwrite(t *testing.T) { } func TestFromTypedIntRetainsLocations(t *testing.T) { - var ref = dyn.NewValue(1234, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(1234, []dyn.Location{{File: "foo"}}) // case: value has not been changed var src int = 1234 nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1234, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1234, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = 1235 nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(int64(1235), dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(int64(1235), []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedIntVariableReference(t *testing.T) { @@ -659,19 +659,19 @@ func TestFromTypedFloatNonEmptyOverwrite(t *testing.T) { func TestFromTypedFloatRetainsLocations(t *testing.T) { var src float64 - var ref = dyn.NewValue(1.23, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}) // case: value has not been changed src = 1.23 nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.23, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = 1.24 nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.24, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1.24, []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedFloatVariableReference(t *testing.T) { @@ -740,27 +740,27 @@ func TestFromTypedNilPointerRetainsLocations(t *testing.T) { } var src *Tmp - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } func TestFromTypedNilMapRetainsLocation(t *testing.T) { var src map[string]string - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } func TestFromTypedNilSliceRetainsLocation(t *testing.T) { var src []string - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index ad82e20ef..246c97eaf 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -120,7 +120,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen // Return the normalized value if missing fields are not included. if !n.includeMissingFields { - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } // Populate missing fields with their zero values. @@ -165,7 +165,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen } } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -203,7 +203,7 @@ func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []r out.Set(pk, nv) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -238,7 +238,7 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [ out = append(out, v) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -273,7 +273,7 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindString, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -306,7 +306,7 @@ func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dy return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -349,7 +349,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindInt, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -392,7 +392,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindFloat, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeInterface(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 299ffcabd..452ed4eb1 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -229,7 +229,7 @@ func TestNormalizeStructVariableReference(t *testing.T) { } var typ Tmp - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -241,7 +241,7 @@ func TestNormalizeStructRandomStringError(t *testing.T) { } var typ Tmp - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -258,7 +258,7 @@ func TestNormalizeStructIntError(t *testing.T) { } var typ Tmp - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -360,7 +360,7 @@ func TestNormalizeMapNestedError(t *testing.T) { func TestNormalizeMapVariableReference(t *testing.T) { var typ map[string]string - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -368,7 +368,7 @@ func TestNormalizeMapVariableReference(t *testing.T) { func TestNormalizeMapRandomStringError(t *testing.T) { var typ map[string]string - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -381,7 +381,7 @@ func TestNormalizeMapRandomStringError(t *testing.T) { func TestNormalizeMapIntError(t *testing.T) { var typ map[string]string - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -482,7 +482,7 @@ func TestNormalizeSliceNestedError(t *testing.T) { func TestNormalizeSliceVariableReference(t *testing.T) { var typ []string - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -490,7 +490,7 @@ func TestNormalizeSliceVariableReference(t *testing.T) { func TestNormalizeSliceRandomStringError(t *testing.T) { var typ []string - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -503,7 +503,7 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { func TestNormalizeSliceIntError(t *testing.T) { var typ []string - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -524,7 +524,7 @@ func TestNormalizeString(t *testing.T) { func TestNormalizeStringNil(t *testing.T) { var typ string - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -537,26 +537,26 @@ func TestNormalizeStringNil(t *testing.T) { func TestNormalizeStringFromBool(t *testing.T) { var typ string - vin := dyn.NewValue(true, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(true, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("true", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("true", vin.Locations()), vout) } func TestNormalizeStringFromInt(t *testing.T) { var typ string - vin := dyn.NewValue(123, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(123, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("123", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("123", vin.Locations()), vout) } func TestNormalizeStringFromFloat(t *testing.T) { var typ string - vin := dyn.NewValue(1.20, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1.20, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("1.2", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) } func TestNormalizeStringError(t *testing.T) { @@ -582,7 +582,7 @@ func TestNormalizeBool(t *testing.T) { func TestNormalizeBoolNil(t *testing.T) { var typ bool - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -658,7 +658,7 @@ func TestNormalizeInt(t *testing.T) { func TestNormalizeIntNil(t *testing.T) { var typ int - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -742,7 +742,7 @@ func TestNormalizeFloat(t *testing.T) { func TestNormalizeFloatNil(t *testing.T) { var typ float64 - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ @@ -842,26 +842,26 @@ func TestNormalizeAnchors(t *testing.T) { func TestNormalizeBoolToAny(t *testing.T) { var typ any - vin := dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } func TestNormalizeIntToAny(t *testing.T) { var typ any - vin := dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } func TestNormalizeSliceToAny(t *testing.T) { var typ any - v1 := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) - v2 := dyn.NewValue(2, dyn.Location{File: "file", Line: 1, Column: 1}) - vin := dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1}) + v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + v2 := dyn.NewValue(2, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vin := dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index d2494bc21..111da25c8 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -155,7 +155,7 @@ func (r *resolver) resolveRef(ref ref, seen []string) (dyn.Value, error) { // of where it is used. This also means that relative path resolution is done // relative to where a variable is used, not where it is defined. // - return dyn.NewValue(resolved[0].Value(), ref.value.Location()), nil + return dyn.NewValue(resolved[0].Value(), ref.value.Locations()), nil } // Not pure; perform string interpolation. @@ -178,7 +178,7 @@ func (r *resolver) resolveRef(ref ref, seen []string) (dyn.Value, error) { ref.str = strings.Replace(ref.str, ref.matches[j][0], s, 1) } - return dyn.NewValue(ref.str, ref.value.Location()), nil + return dyn.NewValue(ref.str, ref.value.Locations()), nil } func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go index da20ee849..e6e640d14 100644 --- a/libs/dyn/merge/elements_by_key.go +++ b/libs/dyn/merge/elements_by_key.go @@ -52,7 +52,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { out = append(out, nv) } - return dyn.NewValue(out, v.Location()), nil + return dyn.NewValue(out, v.Locations()), nil } // ElementsByKey returns a [dyn.MapFunc] that operates on a sequence diff --git a/libs/dyn/merge/merge.go b/libs/dyn/merge/merge.go index ffe000da3..29decd779 100644 --- a/libs/dyn/merge/merge.go +++ b/libs/dyn/merge/merge.go @@ -12,6 +12,26 @@ import ( // * Merging x with nil or nil with x always yields x. // * Merging maps a and b means entries from map b take precedence. // * Merging sequences a and b means concatenating them. +// +// Merging retains and accumulates the locations metadata associated with the values. +// This allows users of the module to track the provenance of values across merging of +// configuration trees, which is useful for reporting errors and warnings. +// +// Semantics for location metadata in the merged value are similar to the semantics +// for the values themselves: +// +// - When merging x with nil or nil with x, the location of x is retained. +// +// - When merging maps or sequences, the combined value retains the location of a and +// accumulates the location of b. The individual elements of the map or sequence retain +// their original locations, i.e., whether they were originally defined in a or b. +// +// The rationale for retaining location of a is that we would like to return +// the first location a bit of configuration showed up when reporting errors and warnings. +// +// - Merging primitive values means using the incoming value `b`. The location of the +// incoming value is retained and the location of the existing value `a` is accumulated. +// This is because the incoming value overwrites the existing value. func Merge(a, b dyn.Value) (dyn.Value, error) { return merge(a, b) } @@ -22,12 +42,12 @@ func merge(a, b dyn.Value) (dyn.Value, error) { // If a is nil, return b. if ak == dyn.KindNil { - return b, nil + return b.AppendLocationsFromValue(a), nil } // If b is nil, return a. if bk == dyn.KindNil { - return a, nil + return a.AppendLocationsFromValue(b), nil } // Call the appropriate merge function based on the kind of a and b. @@ -75,8 +95,8 @@ func mergeMap(a, b dyn.Value) (dyn.Value, error) { } } - // Preserve the location of the first value. - return dyn.NewValue(out, a.Location()), nil + // Preserve the location of the first value. Accumulate the locations of the second value. + return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil } func mergeSequence(a, b dyn.Value) (dyn.Value, error) { @@ -88,11 +108,10 @@ func mergeSequence(a, b dyn.Value) (dyn.Value, error) { copy(out[:], as) copy(out[len(as):], bs) - // Preserve the location of the first value. - return dyn.NewValue(out, a.Location()), nil + // Preserve the location of the first value. Accumulate the locations of the second value. + return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil } - func mergePrimitive(a, b dyn.Value) (dyn.Value, error) { // Merging primitive values means using the incoming value. - return b, nil + return b.AppendLocationsFromValue(a), nil } diff --git a/libs/dyn/merge/merge_test.go b/libs/dyn/merge/merge_test.go index 3706dbd77..4a4bf9e6c 100644 --- a/libs/dyn/merge/merge_test.go +++ b/libs/dyn/merge/merge_test.go @@ -8,15 +8,17 @@ import ( ) func TestMergeMaps(t *testing.T) { - v1 := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + v1 := dyn.NewValue(map[string]dyn.Value{ + "foo": dyn.NewValue("bar", []dyn.Location{l1}), + "bar": dyn.NewValue("baz", []dyn.Location{l1}), + }, []dyn.Location{l1}) - v2 := dyn.V(map[string]dyn.Value{ - "bar": dyn.V("qux"), - "qux": dyn.V("foo"), - }) + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + v2 := dyn.NewValue(map[string]dyn.Value{ + "bar": dyn.NewValue("qux", []dyn.Location{l2}), + "qux": dyn.NewValue("foo", []dyn.Location{l2}), + }, []dyn.Location{l2}) // Merge v2 into v1. { @@ -27,6 +29,23 @@ func TestMergeMaps(t *testing.T) { "bar": "qux", "qux": "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) + assert.Equal(t, []dyn.Location{l2, l1}, out.Get("bar").Locations()) + assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) + assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) + + // Location of the merged value should be the location of v1. + assert.Equal(t, l1, out.Location()) + + // Value of bar is "qux" which comes from v2. This .Location() should + // return the location of v2. + assert.Equal(t, l2, out.Get("bar").Location()) + + // Original locations of keys that were not overwritten should be preserved. + assert.Equal(t, l1, out.Get("foo").Location()) + assert.Equal(t, l2, out.Get("qux").Location()) } // Merge v1 into v2. @@ -38,30 +57,64 @@ func TestMergeMaps(t *testing.T) { "bar": "baz", "qux": "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) + assert.Equal(t, []dyn.Location{l1, l2}, out.Get("bar").Locations()) + assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) + assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) + + // Location of the merged value should be the location of v2. + assert.Equal(t, l2, out.Location()) + + // Value of bar is "baz" which comes from v1. This .Location() should + // return the location of v1. + assert.Equal(t, l1, out.Get("bar").Location()) + + // Original locations of keys that were not overwritten should be preserved. + assert.Equal(t, l1, out.Get("foo").Location()) + assert.Equal(t, l2, out.Get("qux").Location()) } + } func TestMergeMapsNil(t *testing.T) { - v := dyn.V(map[string]dyn.Value{ + l := dyn.Location{File: "file", Line: 1, Column: 2} + v := dyn.NewValue(map[string]dyn.Value{ "foo": dyn.V("bar"), - }) + }, []dyn.Location{l}) + + nilL := dyn.Location{File: "file", Line: 3, Column: 4} + nilV := dyn.NewValue(nil, []dyn.Location{nilL}) // Merge nil into v. { - out, err := Merge(v, dyn.NilValue) + out, err := Merge(v, nilV) assert.NoError(t, err) assert.Equal(t, map[string]any{ "foo": "bar", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) + + // Location of the non-nil value should be returned by .Location(). + assert.Equal(t, l, out.Location()) } // Merge v into nil. { - out, err := Merge(dyn.NilValue, v) + out, err := Merge(nilV, v) assert.NoError(t, err) assert.Equal(t, map[string]any{ "foo": "bar", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) + + // Location of the non-nil value should be returned by .Location(). + assert.Equal(t, l, out.Location()) } } @@ -81,15 +134,18 @@ func TestMergeMapsError(t *testing.T) { } func TestMergeSequences(t *testing.T) { - v1 := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + v1 := dyn.NewValue([]dyn.Value{ + dyn.NewValue("bar", []dyn.Location{l1}), + dyn.NewValue("baz", []dyn.Location{l1}), + }, []dyn.Location{l1}) - v2 := dyn.V([]dyn.Value{ - dyn.V("qux"), - dyn.V("foo"), - }) + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + l3 := dyn.Location{File: "file3", Line: 5, Column: 6} + v2 := dyn.NewValue([]dyn.Value{ + dyn.NewValue("qux", []dyn.Location{l2}), + dyn.NewValue("foo", []dyn.Location{l3}), + }, []dyn.Location{l2, l3}) // Merge v2 into v1. { @@ -101,6 +157,18 @@ func TestMergeSequences(t *testing.T) { "qux", "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2, l3}, out.Locations()) + + // Location of the merged value should be the location of v1. + assert.Equal(t, l1, out.Location()) + + // Location of the individual values should be preserved. + assert.Equal(t, l1, out.Index(0).Location()) // "bar" + assert.Equal(t, l1, out.Index(1).Location()) // "baz" + assert.Equal(t, l2, out.Index(2).Location()) // "qux" + assert.Equal(t, l3, out.Index(3).Location()) // "foo" } // Merge v1 into v2. @@ -113,6 +181,18 @@ func TestMergeSequences(t *testing.T) { "bar", "baz", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l3, l1}, out.Locations()) + + // Location of the merged value should be the location of v2. + assert.Equal(t, l2, out.Location()) + + // Location of the individual values should be preserved. + assert.Equal(t, l2, out.Index(0).Location()) // "qux" + assert.Equal(t, l3, out.Index(1).Location()) // "foo" + assert.Equal(t, l1, out.Index(2).Location()) // "bar" + assert.Equal(t, l1, out.Index(3).Location()) // "baz" } } @@ -156,14 +236,22 @@ func TestMergeSequencesError(t *testing.T) { } func TestMergePrimitives(t *testing.T) { - v1 := dyn.V("bar") - v2 := dyn.V("baz") + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + v1 := dyn.NewValue("bar", []dyn.Location{l1}) + v2 := dyn.NewValue("baz", []dyn.Location{l2}) // Merge v2 into v1. { out, err := Merge(v1, v2) assert.NoError(t, err) assert.Equal(t, "baz", out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) + + // Location of the merged value should be the location of v2, the second value. + assert.Equal(t, l2, out.Location()) } // Merge v1 into v2. @@ -171,6 +259,12 @@ func TestMergePrimitives(t *testing.T) { out, err := Merge(v2, v1) assert.NoError(t, err) assert.Equal(t, "bar", out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) + + // Location of the merged value should be the location of v1, the second value. + assert.Equal(t, l1, out.Location()) } } diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go index 823fb1933..7a8667cd6 100644 --- a/libs/dyn/merge/override.go +++ b/libs/dyn/merge/override.go @@ -51,7 +51,7 @@ func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor Overri return dyn.InvalidValue, err } - return dyn.NewValue(merged, left.Location()), nil + return dyn.NewValue(merged, left.Locations()), nil case dyn.KindSequence: // some sequences are keyed, and we can detect which elements are added/removed/updated, @@ -62,7 +62,7 @@ func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor Overri return dyn.InvalidValue, err } - return dyn.NewValue(merged, left.Location()), nil + return dyn.NewValue(merged, left.Locations()), nil case dyn.KindString: if left.MustString() == right.MustString() { diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go index d9ca97486..9d41a526e 100644 --- a/libs/dyn/merge/override_test.go +++ b/libs/dyn/merge/override_test.go @@ -27,79 +27,79 @@ func TestOverride_Primitive(t *testing.T) { { name: "string (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue("b", rightLocation), - expected: dyn.NewValue("b", rightLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue("b", []dyn.Location{rightLocation}), + expected: dyn.NewValue("b", []dyn.Location{rightLocation}), }, { name: "string (not updated)", state: visitorState{}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue("a", rightLocation), - expected: dyn.NewValue("a", leftLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue("a", []dyn.Location{rightLocation}), + expected: dyn.NewValue("a", []dyn.Location{leftLocation}), }, { name: "bool (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, leftLocation), - right: dyn.NewValue(false, rightLocation), - expected: dyn.NewValue(false, rightLocation), + left: dyn.NewValue(true, []dyn.Location{leftLocation}), + right: dyn.NewValue(false, []dyn.Location{rightLocation}), + expected: dyn.NewValue(false, []dyn.Location{rightLocation}), }, { name: "bool (not updated)", state: visitorState{}, - left: dyn.NewValue(true, leftLocation), - right: dyn.NewValue(true, rightLocation), - expected: dyn.NewValue(true, leftLocation), + left: dyn.NewValue(true, []dyn.Location{leftLocation}), + right: dyn.NewValue(true, []dyn.Location{rightLocation}), + expected: dyn.NewValue(true, []dyn.Location{leftLocation}), }, { name: "int (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, leftLocation), - right: dyn.NewValue(2, rightLocation), - expected: dyn.NewValue(2, rightLocation), + left: dyn.NewValue(1, []dyn.Location{leftLocation}), + right: dyn.NewValue(2, []dyn.Location{rightLocation}), + expected: dyn.NewValue(2, []dyn.Location{rightLocation}), }, { name: "int (not updated)", state: visitorState{}, - left: dyn.NewValue(int32(1), leftLocation), - right: dyn.NewValue(int64(1), rightLocation), - expected: dyn.NewValue(int32(1), leftLocation), + left: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), + right: dyn.NewValue(int64(1), []dyn.Location{rightLocation}), + expected: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), }, { name: "float (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, leftLocation), - right: dyn.NewValue(2.0, rightLocation), - expected: dyn.NewValue(2.0, rightLocation), + left: dyn.NewValue(1.0, []dyn.Location{leftLocation}), + right: dyn.NewValue(2.0, []dyn.Location{rightLocation}), + expected: dyn.NewValue(2.0, []dyn.Location{rightLocation}), }, { name: "float (not updated)", state: visitorState{}, - left: dyn.NewValue(float32(1.0), leftLocation), - right: dyn.NewValue(float64(1.0), rightLocation), - expected: dyn.NewValue(float32(1.0), leftLocation), + left: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), + right: dyn.NewValue(float64(1.0), []dyn.Location{rightLocation}), + expected: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), }, { name: "time (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(time.UnixMilli(10000), leftLocation), - right: dyn.NewValue(time.UnixMilli(10001), rightLocation), - expected: dyn.NewValue(time.UnixMilli(10001), rightLocation), + left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), + right: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), + expected: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), }, { name: "time (not updated)", state: visitorState{}, - left: dyn.NewValue(time.UnixMilli(10000), leftLocation), - right: dyn.NewValue(time.UnixMilli(10000), rightLocation), - expected: dyn.NewValue(time.UnixMilli(10000), leftLocation), + left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), + right: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{rightLocation}), + expected: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), }, { name: "different types (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, { name: "map - remove 'a', update 'b'", @@ -109,23 +109,22 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, leftLocation), - "b": dyn.NewValue(10, leftLocation), + "a": dyn.NewValue(42, []dyn.Location{leftLocation}), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, - ), + []dyn.Location{leftLocation}), + right: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(20, rightLocation), + "b": dyn.NewValue(20, []dyn.Location{rightLocation}), }, - rightLocation, - ), + []dyn.Location{rightLocation}), + expected: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(20, rightLocation), + "b": dyn.NewValue(20, []dyn.Location{rightLocation}), }, - leftLocation, - ), + []dyn.Location{leftLocation}), }, { name: "map - add 'a'", @@ -134,24 +133,26 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, rightLocation), - "b": dyn.NewValue(10, rightLocation), + "a": dyn.NewValue(42, []dyn.Location{rightLocation}), + "b": dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, rightLocation), + "a": dyn.NewValue(42, []dyn.Location{rightLocation}), // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -161,23 +162,25 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, leftLocation), - "b": dyn.NewValue(10, leftLocation), + "a": dyn.NewValue(42, []dyn.Location{leftLocation}), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(10, rightLocation), + "b": dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -189,36 +192,38 @@ func TestOverride_Primitive(t *testing.T) { map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, rightLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), }, - rightLocation, + []dyn.Location{rightLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -228,35 +233,35 @@ func TestOverride_Primitive(t *testing.T) { map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -264,23 +269,23 @@ func TestOverride_Primitive(t *testing.T) { state: visitorState{added: []string{"root[1]"}}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), - dyn.NewValue(10, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), + dyn.NewValue(10, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), - dyn.NewValue(10, rightLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), + dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -288,67 +293,67 @@ func TestOverride_Primitive(t *testing.T) { state: visitorState{removed: []string{"root[1]"}}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), - dyn.NewValue(10, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), + dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - // location hasn't changed because value hasn't changed - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + // location hasn't changed because value hasn't changed }, { name: "sequence (not updated)", state: visitorState{}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { name: "nil (not updated)", state: visitorState{}, - left: dyn.NilValue.WithLocation(leftLocation), - right: dyn.NilValue.WithLocation(rightLocation), - expected: dyn.NilValue.WithLocation(leftLocation), + left: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), + right: dyn.NilValue.WithLocations([]dyn.Location{rightLocation}), + expected: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), }, { name: "nil (updated)", state: visitorState{updated: []string{"root"}}, left: dyn.NilValue, - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, { name: "change kind (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, leftLocation), - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + left: dyn.NewValue(42.0, []dyn.Location{leftLocation}), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, } @@ -375,7 +380,7 @@ func TestOverride_Primitive(t *testing.T) { }) t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.NewValue("return value", dyn.Location{}) + expected := dyn.V("return value") s, visitor := createVisitor(visitorOpts{returnValue: &expected}) out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) @@ -427,17 +432,17 @@ func TestOverride_PreserveMappingKeys(t *testing.T) { rightValueLocation := dyn.Location{File: "right.yml", Line: 3, Column: 1} left := dyn.NewMapping() - left.Set(dyn.NewValue("a", leftKeyLocation), dyn.NewValue(42, leftValueLocation)) + left.Set(dyn.NewValue("a", []dyn.Location{leftKeyLocation}), dyn.NewValue(42, []dyn.Location{leftValueLocation})) right := dyn.NewMapping() - right.Set(dyn.NewValue("a", rightKeyLocation), dyn.NewValue(7, rightValueLocation)) + right.Set(dyn.NewValue("a", []dyn.Location{rightKeyLocation}), dyn.NewValue(7, []dyn.Location{rightValueLocation})) state, visitor := createVisitor(visitorOpts{}) out, err := override( dyn.EmptyPath, - dyn.NewValue(left, leftLocation), - dyn.NewValue(right, rightLocation), + dyn.NewValue(left, []dyn.Location{leftLocation}), + dyn.NewValue(right, []dyn.Location{rightLocation}), visitor, ) diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go index a265dad08..aecdc3ca6 100644 --- a/libs/dyn/pattern.go +++ b/libs/dyn/pattern.go @@ -72,7 +72,7 @@ func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitO m.Set(pk, nv) } - return NewValue(m, v.Location()), nil + return NewValue(m, v.Locations()), nil } type anyIndexComponent struct{} @@ -103,5 +103,5 @@ func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visi s[i] = nv } - return NewValue(s, v.Location()), nil + return NewValue(s, v.Locations()), nil } diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 3d62ea1f5..2aed2f6cd 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -2,13 +2,18 @@ package dyn import ( "fmt" + "slices" ) type Value struct { v any k Kind - l Location + + // List of locations this value is defined at. The first location in the slice + // is the location returned by the `.Location()` method and is typically used + // for reporting errors and warnings associated with the value. + l []Location // Whether or not this value is an anchor. // If this node doesn't map to a type, we don't need to warn about it. @@ -27,11 +32,11 @@ var NilValue = Value{ // V constructs a new Value with the given value. func V(v any) Value { - return NewValue(v, Location{}) + return NewValue(v, []Location{}) } // NewValue constructs a new Value with the given value and location. -func NewValue(v any, loc Location) Value { +func NewValue(v any, loc []Location) Value { switch vin := v.(type) { case map[string]Value: v = newMappingFromGoMap(vin) @@ -40,16 +45,30 @@ func NewValue(v any, loc Location) Value { return Value{ v: v, k: kindOf(v), - l: loc, + + // create a copy of the locations, so that mutations to the original slice + // don't affect new value. + l: slices.Clone(loc), } } -// WithLocation returns a new Value with its location set to the given value. -func (v Value) WithLocation(loc Location) Value { +// WithLocations returns a new Value with its location set to the given value. +func (v Value) WithLocations(loc []Location) Value { return Value{ v: v.v, k: v.k, - l: loc, + + // create a copy of the locations, so that mutations to the original slice + // don't affect new value. + l: slices.Clone(loc), + } +} + +func (v Value) AppendLocationsFromValue(w Value) Value { + return Value{ + v: v.v, + k: v.k, + l: append(v.l, w.l...), } } @@ -61,10 +80,18 @@ func (v Value) Value() any { return v.v } -func (v Value) Location() Location { +func (v Value) Locations() []Location { return v.l } +func (v Value) Location() Location { + if len(v.l) == 0 { + return Location{} + } + + return v.l[0] +} + func (v Value) IsValid() bool { return v.k != KindInvalid } @@ -153,7 +180,10 @@ func (v Value) IsAnchor() bool { // We need a custom implementation because maps and slices // cannot be compared with the regular == operator. func (v Value) eq(w Value) bool { - if v.k != w.k || v.l != w.l { + if v.k != w.k { + return false + } + if !slices.Equal(v.l, w.l) { return false } diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go index bbdc2c96b..6a0a27b8d 100644 --- a/libs/dyn/value_test.go +++ b/libs/dyn/value_test.go @@ -25,16 +25,19 @@ func TestValueAsMap(t *testing.T) { _, ok := zeroValue.AsMap() assert.False(t, ok) - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) _, ok = intValue.AsMap() assert.False(t, ok) var mapValue = dyn.NewValue( map[string]dyn.Value{ - "key": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key": dyn.NewValue( + "value", + []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) + m, ok := mapValue.AsMap() assert.True(t, ok) assert.Equal(t, 1, m.Len()) @@ -43,6 +46,6 @@ func TestValueAsMap(t *testing.T) { func TestValueIsValid(t *testing.T) { var zeroValue dyn.Value assert.False(t, zeroValue.IsValid()) - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) assert.True(t, intValue.IsValid()) } diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go index 83cffb772..e35cde582 100644 --- a/libs/dyn/value_underlying_test.go +++ b/libs/dyn/value_underlying_test.go @@ -11,7 +11,7 @@ import ( func TestValueUnderlyingMap(t *testing.T) { v := dyn.V( map[string]dyn.Value{ - "key": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, ) @@ -33,7 +33,7 @@ func TestValueUnderlyingMap(t *testing.T) { func TestValueUnderlyingSequence(t *testing.T) { v := dyn.V( []dyn.Value{ - dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, ) diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go index 56a9cf9f3..cd2cd4831 100644 --- a/libs/dyn/visit_map.go +++ b/libs/dyn/visit_map.go @@ -27,7 +27,7 @@ func Foreach(fn MapFunc) MapFunc { } m.Set(pk, nv) } - return NewValue(m, v.Location()), nil + return NewValue(m, v.Locations()), nil case KindSequence: s := slices.Clone(v.MustSequence()) for i, value := range s { @@ -37,7 +37,7 @@ func Foreach(fn MapFunc) MapFunc { return InvalidValue, err } } - return NewValue(s, v.Location()), nil + return NewValue(s, v.Locations()), nil default: return InvalidValue, fmt.Errorf("expected a map or sequence, found %s", v.Kind()) } diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index e6a16f79e..fbb52b504 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -86,7 +86,7 @@ func (d *loader) loadSequence(node *yaml.Node, loc dyn.Location) (dyn.Value, err acc[i] = v } - return dyn.NewValue(acc, loc), nil + return dyn.NewValue(acc, []dyn.Location{loc}), nil } func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { @@ -130,7 +130,7 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro } if merge == nil { - return dyn.NewValue(acc, loc), nil + return dyn.NewValue(acc, []dyn.Location{loc}), nil } // Build location for the merge node. @@ -171,20 +171,20 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro out.Merge(m) } - return dyn.NewValue(out, loc), nil + return dyn.NewValue(out, []dyn.Location{loc}), nil } func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { st := node.ShortTag() switch st { case "!!str": - return dyn.NewValue(node.Value, loc), nil + return dyn.NewValue(node.Value, []dyn.Location{loc}), nil case "!!bool": switch strings.ToLower(node.Value) { case "true": - return dyn.NewValue(true, loc), nil + return dyn.NewValue(true, []dyn.Location{loc}), nil case "false": - return dyn.NewValue(false, loc), nil + return dyn.NewValue(false, []dyn.Location{loc}), nil default: return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) } @@ -195,17 +195,17 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error } // Use regular int type instead of int64 if possible. if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), loc), nil + return dyn.NewValue(int(i64), []dyn.Location{loc}), nil } - return dyn.NewValue(i64, loc), nil + return dyn.NewValue(i64, []dyn.Location{loc}), nil case "!!float": f64, err := strconv.ParseFloat(node.Value, 64) if err != nil { return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) } - return dyn.NewValue(f64, loc), nil + return dyn.NewValue(f64, []dyn.Location{loc}), nil case "!!null": - return dyn.NewValue(nil, loc), nil + return dyn.NewValue(nil, []dyn.Location{loc}), nil case "!!timestamp": // Try a couple of layouts for _, layout := range []string{ @@ -216,7 +216,7 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error } { t, terr := time.Parse(layout, node.Value) if terr == nil { - return dyn.NewValue(t, loc), nil + return dyn.NewValue(t, []dyn.Location{loc}), nil } } return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) diff --git a/libs/dyn/yamlsaver/saver_test.go b/libs/dyn/yamlsaver/saver_test.go index bdf1891cd..387090104 100644 --- a/libs/dyn/yamlsaver/saver_test.go +++ b/libs/dyn/yamlsaver/saver_test.go @@ -19,7 +19,7 @@ func TestMarshalNilValue(t *testing.T) { func TestMarshalIntValue(t *testing.T) { s := NewSaver() - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) v, err := s.toYamlNode(intValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) @@ -28,7 +28,7 @@ func TestMarshalIntValue(t *testing.T) { func TestMarshalFloatValue(t *testing.T) { s := NewSaver() - var floatValue = dyn.NewValue(1.0, dyn.Location{}) + var floatValue = dyn.V(1.0) v, err := s.toYamlNode(floatValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) @@ -37,7 +37,7 @@ func TestMarshalFloatValue(t *testing.T) { func TestMarshalBoolValue(t *testing.T) { s := NewSaver() - var boolValue = dyn.NewValue(true, dyn.Location{}) + var boolValue = dyn.V(true) v, err := s.toYamlNode(boolValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) @@ -46,7 +46,7 @@ func TestMarshalBoolValue(t *testing.T) { func TestMarshalTimeValue(t *testing.T) { s := NewSaver() - var timeValue = dyn.NewValue(time.Unix(0, 0), dyn.Location{}) + var timeValue = dyn.V(time.Unix(0, 0)) v, err := s.toYamlNode(timeValue) assert.NoError(t, err) assert.Equal(t, "1970-01-01 00:00:00 +0000 UTC", v.Value) @@ -57,10 +57,10 @@ func TestMarshalSequenceValue(t *testing.T) { s := NewSaver() var sequenceValue = dyn.NewValue( []dyn.Value{ - dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), - dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), + dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) v, err := s.toYamlNode(sequenceValue) assert.NoError(t, err) @@ -71,7 +71,7 @@ func TestMarshalSequenceValue(t *testing.T) { func TestMarshalStringValue(t *testing.T) { s := NewSaver() - var stringValue = dyn.NewValue("value", dyn.Location{}) + var stringValue = dyn.V("value") v, err := s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "value", v.Value) @@ -82,12 +82,13 @@ func TestMarshalMapValue(t *testing.T) { s := NewSaver() var mapValue = dyn.NewValue( map[string]dyn.Value{ - "key3": dyn.NewValue("value3", dyn.Location{File: "file", Line: 3, Column: 2}), - "key2": dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), - "key1": dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), + "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 3, Column: 2}}), + "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), + "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) + v, err := s.toYamlNode(mapValue) assert.NoError(t, err) assert.Equal(t, yaml.MappingNode, v.Kind) @@ -107,12 +108,12 @@ func TestMarshalNestedValues(t *testing.T) { map[string]dyn.Value{ "key1": dyn.NewValue( map[string]dyn.Value{ - "key2": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key2": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) v, err := s.toYamlNode(mapValue) assert.NoError(t, err) @@ -125,14 +126,14 @@ func TestMarshalNestedValues(t *testing.T) { func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { s := NewSaver() - var hexValue = dyn.NewValue(0x123, dyn.Location{}) + var hexValue = dyn.V(0x123) v, err := s.toYamlNode(hexValue) assert.NoError(t, err) assert.Equal(t, "291", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0x123", dyn.Location{}) + var stringValue = dyn.V("0x123") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0x123", v.Value) @@ -142,14 +143,14 @@ func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { func TestMarshalBinaryValueIsQuoted(t *testing.T) { s := NewSaver() - var binaryValue = dyn.NewValue(0b101, dyn.Location{}) + var binaryValue = dyn.V(0b101) v, err := s.toYamlNode(binaryValue) assert.NoError(t, err) assert.Equal(t, "5", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0b101", dyn.Location{}) + var stringValue = dyn.V("0b101") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0b101", v.Value) @@ -159,14 +160,14 @@ func TestMarshalBinaryValueIsQuoted(t *testing.T) { func TestMarshalOctalValueIsQuoted(t *testing.T) { s := NewSaver() - var octalValue = dyn.NewValue(0123, dyn.Location{}) + var octalValue = dyn.V(0123) v, err := s.toYamlNode(octalValue) assert.NoError(t, err) assert.Equal(t, "83", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0123", dyn.Location{}) + var stringValue = dyn.V("0123") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0123", v.Value) @@ -176,14 +177,14 @@ func TestMarshalOctalValueIsQuoted(t *testing.T) { func TestMarshalFloatValueIsQuoted(t *testing.T) { s := NewSaver() - var floatValue = dyn.NewValue(1.0, dyn.Location{}) + var floatValue = dyn.V(1.0) v, err := s.toYamlNode(floatValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("1.0", dyn.Location{}) + var stringValue = dyn.V("1.0") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "1.0", v.Value) @@ -193,14 +194,14 @@ func TestMarshalFloatValueIsQuoted(t *testing.T) { func TestMarshalBoolValueIsQuoted(t *testing.T) { s := NewSaver() - var boolValue = dyn.NewValue(true, dyn.Location{}) + var boolValue = dyn.V(true) v, err := s.toYamlNode(boolValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("true", dyn.Location{}) + var stringValue = dyn.V("true") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) @@ -215,18 +216,18 @@ func TestCustomStylingWithNestedMap(t *testing.T) { var styledMap = dyn.NewValue( map[string]dyn.Value{ - "key1": dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), - "key2": dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), + "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: -2, Column: 2}, + []dyn.Location{{File: "file", Line: -2, Column: 2}}, ) var unstyledMap = dyn.NewValue( map[string]dyn.Value{ - "key3": dyn.NewValue("value3", dyn.Location{File: "file", Line: 1, Column: 2}), - "key4": dyn.NewValue("value4", dyn.Location{File: "file", Line: 2, Column: 2}), + "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + "key4": dyn.NewValue("value4", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: -1, Column: 2}, + []dyn.Location{{File: "file", Line: -1, Column: 2}}, ) var val = dyn.NewValue( @@ -234,7 +235,7 @@ func TestCustomStylingWithNestedMap(t *testing.T) { "styled": styledMap, "unstyled": unstyledMap, }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) mv, err := s.toYamlNode(val) diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index fa5ab08fb..a162bf31f 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -44,7 +44,7 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin continue } - dst[k] = dyn.NewValue(v.Value(), dyn.Location{Line: order.Get(k)}) + dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) } return dyn.V(dst), nil diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 04b4c404f..1afab601a 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -33,16 +33,25 @@ func TestConvertToMapValueWithOrder(t *testing.T) { assert.NoError(t, err) assert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.NewValue([]dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, dyn.Location{Line: -3}), - "name": dyn.NewValue("test", dyn.Location{Line: -2}), - "map": dyn.NewValue(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, dyn.Location{Line: -1}), - "long_name_field": dyn.NewValue("long name goes here", dyn.Location{Line: 1}), + "list": dyn.NewValue( + []dyn.Value{ + dyn.V("a"), + dyn.V("b"), + dyn.V("c"), + }, + []dyn.Location{{Line: -3}}, + ), + "name": dyn.NewValue( + "test", + []dyn.Location{{Line: -2}}, + ), + "map": dyn.NewValue( + map[string]dyn.Value{ + "key1": dyn.V("value1"), + "key2": dyn.V("value2"), + }, + []dyn.Location{{Line: -1}}, + ), + "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{{Line: 1}}), }), result) } From 10fe02075fec0b2e18d2eacf7412816d6e81d6bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:40:12 +0200 Subject: [PATCH 14/88] Bump github.com/databricks/databricks-sdk-go from 0.43.0 to 0.43.2 (#1594) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.43.0 to 0.43.2.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.43.2

Release v0.43.2

Internal Changes

  • Enforce Tag on PRs (#969).
  • Generate SDK for apierr changes (#970).
  • Add Release tag and Workflow Fix (#972).

v0.43.1

0.43.1

Major Changes and Improvements:

  • Add a credentials provider for Github Azure OIDC (#965).
  • Add DataPlane API Support (#936).
  • Added more error messages for retriable errors (timeouts, etc.) (#963).

Internal Changes

  • Add ChangelogConfig to Generator struct (#967).
  • Improve Changelog by grouping changes (#962).
  • Parse API Error messages with int error codes (#960).
Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

0.43.2

Internal Changes

  • Enforce Tag on PRs (#969).
  • Generate SDK for apierr changes (#970).
  • Add Release tag and Workflow Fix (#972).

0.43.1

Major Changes and Improvements:

  • Add a credentials provider for Github Azure OIDC (#965).
  • Add DataPlane API Support (#936).
  • Added more error messages for retriable errors (timeouts, etc.) (#963).

Internal Changes

  • Add ChangelogConfig to Generator struct (#967).
  • Improve Changelog by grouping changes (#962).
  • Parse API Error messages with int error codes (#960).
Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.43.0&new-version=0.43.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ce7ad0c1e..5e29d295e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.43.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.43.2 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index eb7a87a89..8f774a47a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.43.0 h1:x4laolWhYlsQg2t8yWEGyRPZy4/Wv3pKnLEoJfVin7I= -github.com/databricks/databricks-sdk-go v0.43.0/go.mod h1:a9rr0FOHLL26kOjQjZZVFjIYmRABCbrAWVeundDEVG8= +github.com/databricks/databricks-sdk-go v0.43.2 h1:4B+sHAYO5kFqwZNQRmsF70eecqsFX6i/0KfXoDFQT/E= +github.com/databricks/databricks-sdk-go v0.43.2/go.mod h1:nlzeOEgJ1Tmb5HyknBJ3GEorCZKWqEBoHprvPmTSNq8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 235973e7b19db5b1418ce24f9a280c769b991dd7 Mon Sep 17 00:00:00 2001 From: Renaud Hartert Date: Wed, 17 Jul 2024 09:14:02 +0200 Subject: [PATCH 15/88] [Fix] Do not buffer files in memory when downloading (#1599) ## Changes This PR fixes a performance bug that led downloaded files (e.g. with `databricks fs cp dbfs:/Volumes/.../somefile .`) to be buffered in memory before being written. Results from profiling the download of a ~100MB file: Before: ``` Type: alloc_space Showing nodes accounting for 374.02MB, 98.50% of 379.74MB total ``` After: ``` Type: alloc_space Showing nodes accounting for 3748.67kB, 100% of 3748.67kB total ``` Note that this fix is temporary. A longer term solution should be to use the API provided by the Go SDK rather than making an HTTP request directly from the CLI. fix #1575 ## Tests Verified that the CLI properly download the file when doing the profiling. --- libs/filer/files_client.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 9fc68bd56..7ea1d0f03 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,7 +1,6 @@ package filer import ( - "bytes" "context" "errors" "fmt" @@ -179,12 +178,12 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, err } - var buf bytes.Buffer - err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &buf) + var reader io.ReadCloser + err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader) // Return early on success. if err == nil { - return io.NopCloser(&buf), nil + return reader, nil } // Special handling of this error only if it is an API error. From e1474a38f9473391ee7ff2c3d1b4a10ed99f21cf Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 17 Jul 2024 10:49:19 +0200 Subject: [PATCH 16/88] Upgrade TF provider to 1.48.3 (#1600) ## Changes This includes a fix for using periodic triggers. ## Tests Manually confirmed this works with https://github.com/databricks/bundle-examples/pull/32. --- bundle/internal/tf/codegen/schema/version.go | 2 +- bundle/internal/tf/schema/config.go | 62 ++++++++++--------- bundle/internal/tf/schema/data_source_job.go | 6 ++ .../tf/schema/resource_external_location.go | 1 + .../schema/resource_metastore_data_access.go | 1 + .../tf/schema/resource_storage_credential.go | 1 + bundle/internal/tf/schema/root.go | 2 +- 7 files changed, 43 insertions(+), 32 deletions(-) diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index a99f15a40..63a4b1b78 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.48.0" +const ProviderVersion = "1.48.3" diff --git a/bundle/internal/tf/schema/config.go b/bundle/internal/tf/schema/config.go index a2de987ec..e807cdc53 100644 --- a/bundle/internal/tf/schema/config.go +++ b/bundle/internal/tf/schema/config.go @@ -3,34 +3,36 @@ package schema type Config struct { - AccountId string `json:"account_id,omitempty"` - AuthType string `json:"auth_type,omitempty"` - AzureClientId string `json:"azure_client_id,omitempty"` - AzureClientSecret string `json:"azure_client_secret,omitempty"` - AzureEnvironment string `json:"azure_environment,omitempty"` - AzureLoginAppId string `json:"azure_login_app_id,omitempty"` - AzureTenantId string `json:"azure_tenant_id,omitempty"` - AzureUseMsi bool `json:"azure_use_msi,omitempty"` - AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` - ClientId string `json:"client_id,omitempty"` - ClientSecret string `json:"client_secret,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ConfigFile string `json:"config_file,omitempty"` - DatabricksCliPath string `json:"databricks_cli_path,omitempty"` - DebugHeaders bool `json:"debug_headers,omitempty"` - DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` - GoogleCredentials string `json:"google_credentials,omitempty"` - GoogleServiceAccount string `json:"google_service_account,omitempty"` - Host string `json:"host,omitempty"` - HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` - MetadataServiceUrl string `json:"metadata_service_url,omitempty"` - Password string `json:"password,omitempty"` - Profile string `json:"profile,omitempty"` - RateLimit int `json:"rate_limit,omitempty"` - RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` - ServerlessComputeId string `json:"serverless_compute_id,omitempty"` - SkipVerify bool `json:"skip_verify,omitempty"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + AccountId string `json:"account_id,omitempty"` + ActionsIdTokenRequestToken string `json:"actions_id_token_request_token,omitempty"` + ActionsIdTokenRequestUrl string `json:"actions_id_token_request_url,omitempty"` + AuthType string `json:"auth_type,omitempty"` + AzureClientId string `json:"azure_client_id,omitempty"` + AzureClientSecret string `json:"azure_client_secret,omitempty"` + AzureEnvironment string `json:"azure_environment,omitempty"` + AzureLoginAppId string `json:"azure_login_app_id,omitempty"` + AzureTenantId string `json:"azure_tenant_id,omitempty"` + AzureUseMsi bool `json:"azure_use_msi,omitempty"` + AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` + ClientId string `json:"client_id,omitempty"` + ClientSecret string `json:"client_secret,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ConfigFile string `json:"config_file,omitempty"` + DatabricksCliPath string `json:"databricks_cli_path,omitempty"` + DebugHeaders bool `json:"debug_headers,omitempty"` + DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` + GoogleCredentials string `json:"google_credentials,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + Host string `json:"host,omitempty"` + HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` + MetadataServiceUrl string `json:"metadata_service_url,omitempty"` + Password string `json:"password,omitempty"` + Profile string `json:"profile,omitempty"` + RateLimit int `json:"rate_limit,omitempty"` + RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` + ServerlessComputeId string `json:"serverless_compute_id,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + WarehouseId string `json:"warehouse_id,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go index 727848ced..91806d670 100644 --- a/bundle/internal/tf/schema/data_source_job.go +++ b/bundle/internal/tf/schema/data_source_job.go @@ -1224,6 +1224,11 @@ type DataSourceJobJobSettingsSettingsTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } +type DataSourceJobJobSettingsSettingsTriggerPeriodic struct { + Interval int `json:"interval"` + Unit string `json:"unit"` +} + type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1234,6 +1239,7 @@ type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { type DataSourceJobJobSettingsSettingsTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *DataSourceJobJobSettingsSettingsTriggerFileArrival `json:"file_arrival,omitempty"` + Periodic *DataSourceJobJobSettingsSettingsTriggerPeriodic `json:"periodic,omitempty"` TableUpdate *DataSourceJobJobSettingsSettingsTriggerTableUpdate `json:"table_update,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_external_location.go b/bundle/internal/tf/schema/resource_external_location.go index af64c677c..da28271bc 100644 --- a/bundle/internal/tf/schema/resource_external_location.go +++ b/bundle/internal/tf/schema/resource_external_location.go @@ -18,6 +18,7 @@ type ResourceExternalLocation struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_metastore_data_access.go b/bundle/internal/tf/schema/resource_metastore_data_access.go index 155730055..2e2ff4eb4 100644 --- a/bundle/internal/tf/schema/resource_metastore_data_access.go +++ b/bundle/internal/tf/schema/resource_metastore_data_access.go @@ -37,6 +37,7 @@ type ResourceMetastoreDataAccess struct { ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` IsDefault bool `json:"is_default,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_storage_credential.go b/bundle/internal/tf/schema/resource_storage_credential.go index b565a5c78..1c62cf8df 100644 --- a/bundle/internal/tf/schema/resource_storage_credential.go +++ b/bundle/internal/tf/schema/resource_storage_credential.go @@ -36,6 +36,7 @@ type ResourceStorageCredential struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 39db3ea2f..a79e998cf 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.48.0" +const ProviderVersion = "1.48.3" func NewRoot() *Root { return &Root{ From 6d710a411a4cef3575582b3240093f03a48e4410 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 17 Jul 2024 14:33:49 +0200 Subject: [PATCH 17/88] Fixed job name normalisation for bundle generate (#1601) ## Changes Fixes #1537 ## Tests Added unit test --- libs/auth/user_test.go | 4 ++-- libs/textutil/textutil.go | 10 +++++++++- libs/textutil/textutil_test.go | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/libs/auth/user_test.go b/libs/auth/user_test.go index eb579fc98..62b2d29ac 100644 --- a/libs/auth/user_test.go +++ b/libs/auth/user_test.go @@ -22,7 +22,7 @@ func TestGetShortUserName(t *testing.T) { }, { email: "test$.user@example.com", - expected: "test__user", + expected: "test_user", }, { email: `jöhn.dœ@domain.com`, // Using non-ASCII characters. @@ -38,7 +38,7 @@ func TestGetShortUserName(t *testing.T) { }, { email: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. - expected: "__quoted_", + expected: "quoted", }, { email: `name-o'mally@website.org`, // Single quote in the local-part. diff --git a/libs/textutil/textutil.go b/libs/textutil/textutil.go index a5d17d55f..ee9b0f0f1 100644 --- a/libs/textutil/textutil.go +++ b/libs/textutil/textutil.go @@ -1,6 +1,7 @@ package textutil import ( + "regexp" "strings" "unicode" ) @@ -9,7 +10,14 @@ import ( // including spaces and dots, which are not supported in e.g. experiment names or YAML keys. func NormalizeString(name string) string { name = strings.ToLower(name) - return strings.Map(replaceNonAlphanumeric, name) + s := strings.Map(replaceNonAlphanumeric, name) + + // replacing multiple underscores with a single one + re := regexp.MustCompile(`_+`) + s = re.ReplaceAllString(s, "_") + + // removing leading and trailing underscores + return strings.Trim(s, "_") } func replaceNonAlphanumeric(r rune) rune { diff --git a/libs/textutil/textutil_test.go b/libs/textutil/textutil_test.go index fb8bf0b60..f6834a1ef 100644 --- a/libs/textutil/textutil_test.go +++ b/libs/textutil/textutil_test.go @@ -46,6 +46,10 @@ func TestNormalizeString(t *testing.T) { { input: "TestTestTest", expected: "testtesttest", + }, + { + input: ".test//test..test", + expected: "test_test_test", }} for _, c := range cases { From c6c2692368572377ba61e79f63532a5c13b7eb66 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:08:09 +0530 Subject: [PATCH 18/88] Attribute Terraform API requests the CLI (#1598) ## Changes This PR adds cli to the user agent sent downstream to the databricks terraform provider when invoked via DABs. ## Tests Unit tests. Based on the comment here (https://github.com/databricks/cli/blob/10fe02075fec0b2e18d2eacf7412816d6e81d6bc/bundle/config/mutator/verify_cli_version_test.go#L113) we don't need to set the version to make the test assertion work correctly. This is likely because we use `go test` to run the tests while the CLI is compiled and the version is set via `goreleaser`. --- bundle/deploy/terraform/init.go | 7 +++++-- bundle/deploy/terraform/init_test.go | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index d480242ce..e7f720d08 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" @@ -219,8 +220,10 @@ func setProxyEnvVars(ctx context.Context, environ map[string]string, b *bundle.B } func setUserAgentExtraEnvVar(environ map[string]string, b *bundle.Bundle) error { - var products []string - + // Add "cli" to the user agent in set by the Databricks Terraform provider. + // This will allow us to attribute downstream requests made by the Databricks + // Terraform provider to the CLI. + products := []string{fmt.Sprintf("cli/%s", build.GetInfo().Version)} if experimental := b.Config.Experimental; experimental != nil { if experimental.PyDABs.Enabled { products = append(products, "databricks-pydabs/0.0.0") diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index aa9b2f77f..94e47dbc1 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -262,10 +262,9 @@ func TestSetUserAgentExtraEnvVar(t *testing.T) { env := make(map[string]string, 0) err := setUserAgentExtraEnvVar(env, b) - require.NoError(t, err) assert.Equal(t, map[string]string{ - "DATABRICKS_USER_AGENT_EXTRA": "databricks-pydabs/0.0.0", + "DATABRICKS_USER_AGENT_EXTRA": "cli/0.0.0-dev databricks-pydabs/0.0.0", }, env) } From af0114a5a69a258cdb0a309fc383a366d7038361 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 18 Jul 2024 11:45:10 +0200 Subject: [PATCH 19/88] Implement readahead cache for Workspace API calls (#1582) ## Changes The reason this readahead cache exists is that we frequently need to recursively find all files in the bundle root directory, especially for sync include and exclude processing. By caching the response for every file/directory and frontloading the latency cost of these calls, we significantly improve performance and eliminate redundant operations. ## Tests * [ ] Working on unit tests --- libs/filer/workspace_files_cache.go | 428 ++++++++++++++++++ libs/filer/workspace_files_client.go | 4 + .../workspace_files_extensions_client.go | 3 +- 3 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 libs/filer/workspace_files_cache.go diff --git a/libs/filer/workspace_files_cache.go b/libs/filer/workspace_files_cache.go new file mode 100644 index 000000000..5837ad27d --- /dev/null +++ b/libs/filer/workspace_files_cache.go @@ -0,0 +1,428 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "slices" + "sync" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// This readahead cache is designed to optimize file system operations by caching the results of +// directory reads (ReadDir) and file/directory metadata reads (Stat). This cache aims to eliminate +// redundant operations and improve performance by storing the results of these operations and +// reusing them when possible. Additionally, the cache performs readahead on ReadDir calls, +// proactively caching information about files and subdirectories to speed up future access. +// +// The cache maintains two primary maps: one for ReadDir results and another for Stat results. +// When a directory read or a stat operation is requested, the cache first checks if the result +// is already available. If it is, the cached result is returned immediately. If not, the +// operation is queued for execution, and the result is stored in the cache once the operation +// completes. In cases where the result is not immediately available, the caller may need to wait +// for the cache entry to be populated. However, because the queue is processed in order by a +// fixed number of worker goroutines, we are guaranteed that the required cache entry will be +// populated and available once the queue processes the corresponding task. +// +// The cache uses a worker pool to process the queued operations concurrently. This is +// implemented using a fixed number of worker goroutines that continually pull tasks from a +// queue and execute them. The queue itself is logically unbounded in the sense that it needs to +// accommodate all the new tasks that may be generated dynamically during the execution of ReadDir +// calls. Specifically, a single ReadDir call can add an unknown number of new Stat and ReadDir +// tasks to the queue because each directory entry may represent a file or subdirectory that +// requires further processing. +// +// For practical reasons, we are not using an unbounded queue but a channel with a maximum size +// of 10,000. This helps prevent excessive memory usage and ensures that the system remains +// responsive under load. If we encounter real examples of subtrees with more than 10,000 +// elements, we can consider addressing this limitation in the future. For now, this approach +// balances the need for readahead efficiency with practical constraints. +// +// It is crucial to note that each ReadDir and Stat call is executed only once. The result of a +// Stat call can be served from the cache if the information was already returned by an earlier +// ReadDir call. This helps to avoid redundant operations and ensures that the system remains +// efficient even under high load. + +const ( + kMaxQueueSize = 10_000 + + // Number of worker goroutines to process the queue. + // These workers share the same HTTP client and therefore the same rate limiter. + // If this number is increased, the rate limiter should be modified as well. + kNumCacheWorkers = 1 +) + +// queueFullError is returned when the queue is at capacity. +type queueFullError struct { + name string +} + +// Error returns the error message. +func (e queueFullError) Error() string { + return fmt.Sprintf("queue is at capacity (%d); cannot enqueue work for %q", kMaxQueueSize, e.name) +} + +// Common type for all cacheable calls. +type cacheEntry struct { + // Channel to signal that the operation has completed. + done chan struct{} + + // The (cleaned) name of the file or directory being operated on. + name string + + // Return values of the operation. + err error +} + +// String returns the path of the file or directory being operated on. +func (e *cacheEntry) String() string { + return e.name +} + +// Mark this entry as errored. +func (e *cacheEntry) markError(err error) { + e.err = err + close(e.done) +} + +// readDirEntry is the cache entry for a [ReadDir] call. +type readDirEntry struct { + cacheEntry + + // Return values of a [ReadDir] call. + entries []fs.DirEntry +} + +// Create a new readDirEntry. +func newReadDirEntry(name string) *readDirEntry { + return &readDirEntry{cacheEntry: cacheEntry{done: make(chan struct{}), name: name}} +} + +// Execute the operation and signal completion. +func (e *readDirEntry) execute(ctx context.Context, c *cache) { + t1 := time.Now() + e.entries, e.err = c.f.ReadDir(ctx, e.name) + t2 := time.Now() + log.Tracef(ctx, "readdir for %s took %f", e.name, t2.Sub(t1).Seconds()) + + // Finalize the read call by adding all directory entries to the stat cache. + c.completeReadDir(e.name, e.entries) + + // Signal that the operation has completed. + // The return value can now be used by routines waiting on it. + close(e.done) +} + +// Wait for the operation to complete and return the result. +func (e *readDirEntry) wait(ctx context.Context) ([]fs.DirEntry, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-e.done: + // Note: return a copy of the slice to prevent the caller from modifying the cache. + // The underlying elements are values (see [wsfsDirEntry]) so a shallow copy is sufficient. + return slices.Clone(e.entries), e.err + } +} + +// statEntry is the cache entry for a [Stat] call. +type statEntry struct { + cacheEntry + + // Return values of a [Stat] call. + info fs.FileInfo +} + +// Create a new stat entry. +func newStatEntry(name string) *statEntry { + return &statEntry{cacheEntry: cacheEntry{done: make(chan struct{}), name: name}} +} + +// Execute the operation and signal completion. +func (e *statEntry) execute(ctx context.Context, c *cache) { + t1 := time.Now() + e.info, e.err = c.f.Stat(ctx, e.name) + t2 := time.Now() + log.Tracef(ctx, "stat for %s took %f", e.name, t2.Sub(t1).Seconds()) + + // Signal that the operation has completed. + // The return value can now be used by routines waiting on it. + close(e.done) +} + +// Wait for the operation to complete and return the result. +func (e *statEntry) wait(ctx context.Context) (fs.FileInfo, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-e.done: + return e.info, e.err + } +} + +// Mark the stat entry as done. +func (e *statEntry) markDone(info fs.FileInfo, err error) { + e.info = info + e.err = err + close(e.done) +} + +// executable is the interface all cacheable calls must implement. +type executable interface { + fmt.Stringer + + execute(ctx context.Context, c *cache) +} + +// cache stores all entries for cacheable Workspace File System calls. +// We care about caching only [ReadDir] and [Stat] calls. +type cache struct { + f Filer + m sync.Mutex + + readDir map[string]*readDirEntry + stat map[string]*statEntry + + // Queue of operations to execute. + queue chan executable + + // For tracking the number of active goroutines. + wg sync.WaitGroup +} + +func newWorkspaceFilesReadaheadCache(f Filer) *cache { + c := &cache{ + f: f, + + readDir: make(map[string]*readDirEntry), + stat: make(map[string]*statEntry), + + queue: make(chan executable, kMaxQueueSize), + } + + ctx := context.Background() + for range kNumCacheWorkers { + c.wg.Add(1) + go c.work(ctx) + } + + return c +} + +// work until the queue is closed. +func (c *cache) work(ctx context.Context) { + defer c.wg.Done() + + for e := range c.queue { + e.execute(ctx, c) + } +} + +// enqueue adds an operation to the queue. +// If the context is canceled, an error is returned. +// If the queue is full, an error is returned. +// +// Its caller is holding the lock so it cannot block. +func (c *cache) enqueue(ctx context.Context, e executable) error { + select { + case <-ctx.Done(): + return ctx.Err() + case c.queue <- e: + return nil + default: + return queueFullError{e.String()} + } +} + +func (c *cache) completeReadDirForDir(name string, dirEntry fs.DirEntry) { + // Add to the stat cache if not already present. + if _, ok := c.stat[name]; !ok { + e := newStatEntry(name) + e.markDone(dirEntry.Info()) + c.stat[name] = e + } + + // Queue a [ReadDir] call for the directory if not already present. + if _, ok := c.readDir[name]; !ok { + // Create a new cache entry and queue the operation. + e := newReadDirEntry(name) + err := c.enqueue(context.Background(), e) + if err != nil { + e.markError(err) + } + + // Add the entry to the cache, even if has an error. + c.readDir[name] = e + } +} + +func (c *cache) completeReadDirForFile(name string, dirEntry fs.DirEntry) { + // Skip if this entry is already in the cache. + if _, ok := c.stat[name]; ok { + return + } + + // Create a new cache entry. + e := newStatEntry(name) + + // Depending on the object type, we either have to perform a real + // stat call, or we can use the [fs.DirEntry] info directly. + switch dirEntry.(wsfsDirEntry).ObjectType { + case workspace.ObjectTypeNotebook: + // Note: once the list API returns `repos_export_format` we can avoid this additional stat call. + // This is the only (?) case where this implementation is tied to the workspace filer. + + // Queue a [Stat] call for the file. + err := c.enqueue(context.Background(), e) + if err != nil { + e.markError(err) + } + default: + // Use the [fs.DirEntry] info directly. + e.markDone(dirEntry.Info()) + } + + // Add the entry to the cache, even if has an error. + c.stat[name] = e +} + +func (c *cache) completeReadDir(dir string, entries []fs.DirEntry) { + c.m.Lock() + defer c.m.Unlock() + + for _, e := range entries { + name := path.Join(dir, e.Name()) + + if e.IsDir() { + c.completeReadDirForDir(name, e) + } else { + c.completeReadDirForFile(name, e) + } + } +} + +// Cleanup closes the queue and waits for all goroutines to exit. +func (c *cache) Cleanup() { + close(c.queue) + c.wg.Wait() +} + +// Write passes through to the underlying Filer. +func (c *cache) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + return c.f.Write(ctx, name, reader, mode...) +} + +// Read passes through to the underlying Filer. +func (c *cache) Read(ctx context.Context, name string) (io.ReadCloser, error) { + return c.f.Read(ctx, name) +} + +// Delete passes through to the underlying Filer. +func (c *cache) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + return c.f.Delete(ctx, name, mode...) +} + +// Mkdir passes through to the underlying Filer. +func (c *cache) Mkdir(ctx context.Context, name string) error { + return c.f.Mkdir(ctx, name) +} + +// ReadDir returns the entries in a directory. +// If the directory is already in the cache, the cached value is returned. +func (c *cache) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + name = path.Clean(name) + + // Lock before R/W access to the cache. + c.m.Lock() + + // If the directory is already in the cache, wait for and return the cached value. + if e, ok := c.readDir[name]; ok { + c.m.Unlock() + return e.wait(ctx) + } + + // Otherwise, create a new cache entry and queue the operation. + e := newReadDirEntry(name) + err := c.enqueue(ctx, e) + if err != nil { + c.m.Unlock() + return nil, err + } + + c.readDir[name] = e + c.m.Unlock() + + // Wait for the operation to complete. + return e.wait(ctx) +} + +// statFromReadDir returns the file info for a file or directory. +// If the file info is already in the cache, the cached value is returned. +func (c *cache) statFromReadDir(ctx context.Context, name string, entry *readDirEntry) (fs.FileInfo, error) { + _, err := entry.wait(ctx) + if err != nil { + return nil, err + } + + // Upon completion of a [ReadDir] call, all directory entries are added to the stat cache and + // enqueue a [Stat] call if necessary (entries for notebooks are incomplete and require a + // real stat call). + // + // This means that the file or directory we're trying to stat, either + // + // - is present in the stat cache + // - doesn't exist. + // + c.m.Lock() + e, ok := c.stat[name] + c.m.Unlock() + if ok { + return e.wait(ctx) + } + + return nil, FileDoesNotExistError{name} +} + +// Stat returns the file info for a file or directory. +// If the file info is already in the cache, the cached value is returned. +func (c *cache) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + name = path.Clean(name) + + // Lock before R/W access to the cache. + c.m.Lock() + + // If the file info is already in the cache, wait for and return the cached value. + if e, ok := c.stat[name]; ok { + c.m.Unlock() + return e.wait(ctx) + } + + // If the parent directory is in the cache (or queued to be read), + // wait for it to complete to avoid redundant stat calls. + dir := path.Dir(name) + if dir != name { + if e, ok := c.readDir[dir]; ok { + c.m.Unlock() + return c.statFromReadDir(ctx, name, e) + } + } + + // Otherwise, create a new cache entry and queue the operation. + e := newStatEntry(name) + err := c.enqueue(ctx, e) + if err != nil { + c.m.Unlock() + return nil, err + } + + c.stat[name] = e + c.m.Unlock() + + // Wait for the operation to complete. + return e.wait(ctx) +} diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index d799c1f88..e911f4409 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -84,6 +84,10 @@ func (info wsfsFileInfo) Sys() any { return info.ObjectInfo } +func (info wsfsFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo { + return info.ObjectInfo +} + // UnmarshalJSON is a custom unmarshaller for the wsfsFileInfo struct. // It must be defined for this type because otherwise the implementation // of the embedded ObjectInfo type will be used. diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go index a872dcc65..d5d0ce554 100644 --- a/libs/filer/workspace_files_extensions_client.go +++ b/libs/filer/workspace_files_extensions_client.go @@ -162,10 +162,11 @@ func NewWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root strin return nil, err } + cache := newWorkspaceFilesReadaheadCache(filer) return &workspaceFilesExtensionsClient{ workspaceClient: w, - wsfs: filer, + wsfs: cache, root: root, }, nil } From 5b65358146627b63005e1ba9233bce603822dd84 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:17:59 +0530 Subject: [PATCH 20/88] Use local Terraform state only when lineage match (#1588) ## Changes DABs deployments should be isolated if `root_path` and workspace host are different. This PR fixes a bug where local terraform state gets piggybacked if the same cwd is used to deploy two isolated deployments for the same bundle target. This can happen if: 1. A user switches to a different identity on the same machine. 2. The workspace host URL the bundle/target points to is changed. 3. A user changes the `root_path` while doing bundle development. To solve this problem we rely on the lineage field available in the terraform state, which is a uuid identifying unique terraform deployments. There's a 1:1 mapping between a terraform deployment and a bundle deployment. For more details on how lineage works in terraform, see: https://developer.hashicorp.com/terraform/language/state/backends#manual-state-pull-push ## Tests Manually verified that changing the identity no longer results in the incorrect terraform state being used. Also, new unit tests are added. --- bundle/deploy/terraform/state_pull.go | 123 +++++++++----- bundle/deploy/terraform/state_pull_test.go | 185 ++++++++++++--------- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/deploy/terraform/state_test.go | 6 +- bundle/deploy/terraform/util.go | 33 ---- bundle/deploy/terraform/util_test.go | 33 ---- 6 files changed, 194 insertions(+), 188 deletions(-) diff --git a/bundle/deploy/terraform/state_pull.go b/bundle/deploy/terraform/state_pull.go index cc7d34274..9a5b91007 100644 --- a/bundle/deploy/terraform/state_pull.go +++ b/bundle/deploy/terraform/state_pull.go @@ -1,8 +1,8 @@ package terraform import ( - "bytes" "context" + "encoding/json" "errors" "io" "io/fs" @@ -12,10 +12,14 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) +type tfState struct { + Serial int64 `json:"serial"` + Lineage string `json:"lineage"` +} + type statePull struct { filerFactory deploy.FilerFactory } @@ -24,74 +28,105 @@ func (l *statePull) Name() string { return "terraform:state-pull" } -func (l *statePull) remoteState(ctx context.Context, f filer.Filer) (*bytes.Buffer, error) { - // Download state file from filer to local cache directory. - remote, err := f.Read(ctx, TerraformStateFileName) +func (l *statePull) remoteState(ctx context.Context, b *bundle.Bundle) (*tfState, []byte, error) { + f, err := l.filerFactory(b) if err != nil { - // On first deploy this state file doesn't yet exist. - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, err + return nil, nil, err } - defer remote.Close() + r, err := f.Read(ctx, TerraformStateFileName) + if err != nil { + return nil, nil, err + } + defer r.Close() - var buf bytes.Buffer - _, err = io.Copy(&buf, remote) + content, err := io.ReadAll(r) + if err != nil { + return nil, nil, err + } + + state := &tfState{} + err = json.Unmarshal(content, state) + if err != nil { + return nil, nil, err + } + + return state, content, nil +} + +func (l *statePull) localState(ctx context.Context, b *bundle.Bundle) (*tfState, error) { + dir, err := Dir(ctx, b) if err != nil { return nil, err } - return &buf, nil + content, err := os.ReadFile(filepath.Join(dir, TerraformStateFileName)) + if err != nil { + return nil, err + } + + state := &tfState{} + err = json.Unmarshal(content, state) + if err != nil { + return nil, err + } + + return state, nil } func (l *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - f, err := l.filerFactory(b) - if err != nil { - return diag.FromErr(err) - } - dir, err := Dir(ctx, b) if err != nil { return diag.FromErr(err) } - // Download state file from filer to local cache directory. - log.Infof(ctx, "Opening remote state file") - remote, err := l.remoteState(ctx, f) - if err != nil { - log.Infof(ctx, "Unable to open remote state file: %s", err) - return diag.FromErr(err) - } - if remote == nil { - log.Infof(ctx, "Remote state file does not exist") + localStatePath := filepath.Join(dir, TerraformStateFileName) + + // Case: Remote state file does not exist. In this case we fallback to using the + // local Terraform state. This allows users to change the "root_path" their bundle is + // configured with. + remoteState, remoteContent, err := l.remoteState(ctx, b) + if errors.Is(err, fs.ErrNotExist) { + log.Infof(ctx, "Remote state file does not exist. Using local Terraform state.") return nil } - - // Expect the state file to live under dir. - local, err := os.OpenFile(filepath.Join(dir, TerraformStateFileName), os.O_CREATE|os.O_RDWR, 0600) if err != nil { + return diag.Errorf("failed to read remote state file: %v", err) + } + + // Expected invariant: remote state file should have a lineage UUID. Error + // if that's not the case. + if remoteState.Lineage == "" { + return diag.Errorf("remote state file does not have a lineage") + } + + // Case: Local state file does not exist. In this case we should rely on the remote state file. + localState, err := l.localState(ctx, b) + if errors.Is(err, fs.ErrNotExist) { + log.Infof(ctx, "Local state file does not exist. Using remote Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) return diag.FromErr(err) } - defer local.Close() - - if !IsLocalStateStale(local, bytes.NewReader(remote.Bytes())) { - log.Infof(ctx, "Local state is the same or newer, ignoring remote state") - return nil + if err != nil { + return diag.Errorf("failed to read local state file: %v", err) } - // Truncating the file before writing - local.Truncate(0) - local.Seek(0, 0) - - // Write file to disk. - log.Infof(ctx, "Writing remote state file to local cache directory") - _, err = io.Copy(local, bytes.NewReader(remote.Bytes())) - if err != nil { + // If the lineage does not match, the Terraform state files do not correspond to the same deployment. + if localState.Lineage != remoteState.Lineage { + log.Infof(ctx, "Remote and local state lineages do not match. Using remote Terraform state. Invalidating local Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) return diag.FromErr(err) } + // If the remote state is newer than the local state, we should use the remote state. + if remoteState.Serial > localState.Serial { + log.Infof(ctx, "Remote state is newer than local state. Using remote Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) + return diag.FromErr(err) + } + + // default: local state is newer or equal to remote state in terms of serial sequence. + // It is also of the same lineage. Keep using the local state. return nil } diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index 26297bfcb..39937a3cc 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/mock" ) -func mockStateFilerForPull(t *testing.T, contents map[string]int, merr error) filer.Filer { +func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) filer.Filer { buf, err := json.Marshal(contents) assert.NoError(t, err) @@ -41,86 +41,123 @@ func statePullTestBundle(t *testing.T) *bundle.Bundle { } } -func TestStatePullLocalMissingRemoteMissing(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, nil, os.ErrNotExist)), - } +func TestStatePullLocalErrorWhenRemoteHasNoLineage(t *testing.T) { + m := &statePull{} - ctx := context.Background() - b := statePullTestBundle(t) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) + t.Run("no local state", func(t *testing.T) { + // setup remote state. + m.filerFactory = identityFiler(mockStateFilerForPull(t, map[string]any{"serial": 5}, nil)) - // Confirm that no local state file has been written. - _, err := os.Stat(localStateFile(t, ctx, b)) - assert.ErrorIs(t, err, fs.ErrNotExist) + ctx := context.Background() + b := statePullTestBundle(t) + diags := bundle.Apply(ctx, b, m) + assert.EqualError(t, diags.Error(), "remote state file does not have a lineage") + }) + + t.Run("local state with lineage", func(t *testing.T) { + // setup remote state. + m.filerFactory = identityFiler(mockStateFilerForPull(t, map[string]any{"serial": 5}, nil)) + + ctx := context.Background() + b := statePullTestBundle(t) + writeLocalState(t, ctx, b, map[string]any{"serial": 5, "lineage": "aaaa"}) + + diags := bundle.Apply(ctx, b, m) + assert.EqualError(t, diags.Error(), "remote state file does not have a lineage") + }) } -func TestStatePullLocalMissingRemotePresent(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5}, nil)), +func TestStatePullLocal(t *testing.T) { + tcases := []struct { + name string + + // remote state before applying the pull mutators + remote map[string]any + + // local state before applying the pull mutators + local map[string]any + + // expected local state after applying the pull mutators + expected map[string]any + }{ + { + name: "remote missing, local missing", + remote: nil, + local: nil, + expected: nil, + }, + { + name: "remote missing, local present", + remote: nil, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // fallback to local state, since remote state is missing. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local stale", + remote: map[string]any{"serial": 10, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // use remote, since remote is newer. + expected: map[string]any{"serial": float64(10), "lineage": "aaaa", "some_other_key": float64(123)}, + }, + { + name: "local equal", + remote: map[string]any{"serial": 5, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // use local state, since they are equal in terms of serial sequence. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local newer", + remote: map[string]any{"serial": 5, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 6, "lineage": "aaaa"}, + // use local state, since local is newer. + expected: map[string]any{"serial": float64(6), "lineage": "aaaa"}, + }, + { + name: "remote and local have different lineages", + remote: map[string]any{"serial": 5, "lineage": "aaaa"}, + local: map[string]any{"serial": 10, "lineage": "bbbb"}, + // use remote, since lineages do not match. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local is missing lineage", + remote: map[string]any{"serial": 5, "lineage": "aaaa"}, + local: map[string]any{"serial": 10}, + // use remote, since local does not have lineage. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, } - ctx := context.Background() - b := statePullTestBundle(t) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + m := &statePull{} + if tc.remote == nil { + // nil represents no remote state file. + m.filerFactory = identityFiler(mockStateFilerForPull(t, nil, os.ErrNotExist)) + } else { + m.filerFactory = identityFiler(mockStateFilerForPull(t, tc.remote, nil)) + } - // Confirm that the local state file has been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} + ctx := context.Background() + b := statePullTestBundle(t) + if tc.local != nil { + writeLocalState(t, ctx, b, tc.local) + } -func TestStatePullLocalStale(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5}, nil)), + diags := bundle.Apply(ctx, b, m) + assert.NoError(t, diags.Error()) + + if tc.expected == nil { + // nil represents no local state file is expected. + _, err := os.Stat(localStateFile(t, ctx, b)) + assert.ErrorIs(t, err, fs.ErrNotExist) + } else { + localState := readLocalState(t, ctx, b) + assert.Equal(t, tc.expected, localState) + + } + }) } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a stale local state file. - writeLocalState(t, ctx, b, map[string]int{"serial": 4}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} - -func TestStatePullLocalEqual(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5, "some_other_key": 123}, nil)), - } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a local state file with the same serial as the remote. - writeLocalState(t, ctx, b, map[string]int{"serial": 5}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has not been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} - -func TestStatePullLocalNewer(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5, "some_other_key": 123}, nil)), - } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a local state file with a newer serial as the remote. - writeLocalState(t, ctx, b, map[string]int{"serial": 6}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has not been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 6}, localState) } diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index e054773f3..ac74f345d 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -55,7 +55,7 @@ func TestStatePush(t *testing.T) { b := statePushTestBundle(t) // Write a stale local state file. - writeLocalState(t, ctx, b, map[string]int{"serial": 4}) + writeLocalState(t, ctx, b, map[string]any{"serial": 4}) diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } diff --git a/bundle/deploy/terraform/state_test.go b/bundle/deploy/terraform/state_test.go index ff3250625..73d7cb0de 100644 --- a/bundle/deploy/terraform/state_test.go +++ b/bundle/deploy/terraform/state_test.go @@ -26,19 +26,19 @@ func localStateFile(t *testing.T, ctx context.Context, b *bundle.Bundle) string return filepath.Join(dir, TerraformStateFileName) } -func readLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle) map[string]int { +func readLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle) map[string]any { f, err := os.Open(localStateFile(t, ctx, b)) require.NoError(t, err) defer f.Close() - var contents map[string]int + var contents map[string]any dec := json.NewDecoder(f) err = dec.Decode(&contents) require.NoError(t, err) return contents } -func writeLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle, contents map[string]int) { +func writeLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle, contents map[string]any) { f, err := os.Create(localStateFile(t, ctx, b)) require.NoError(t, err) defer f.Close() diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 1a8a83ac7..64d667b5f 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "io" "os" "path/filepath" @@ -22,10 +21,6 @@ type resourcesState struct { const SupportedStateVersion = 4 -type serialState struct { - Serial int `json:"serial"` -} - type stateResource struct { Type string `json:"type"` Name string `json:"name"` @@ -41,34 +36,6 @@ type stateInstanceAttributes struct { ID string `json:"id"` } -func IsLocalStateStale(local io.Reader, remote io.Reader) bool { - localState, err := loadState(local) - if err != nil { - return true - } - - remoteState, err := loadState(remote) - if err != nil { - return false - } - - return localState.Serial < remoteState.Serial -} - -func loadState(input io.Reader) (*serialState, error) { - content, err := io.ReadAll(input) - if err != nil { - return nil, err - } - var s serialState - err = json.Unmarshal(content, &s) - if err != nil { - return nil, err - } - - return &s, nil -} - func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) { cacheDir, err := Dir(ctx, b) if err != nil { diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go index 8949ebca8..251a7c256 100644 --- a/bundle/deploy/terraform/util_test.go +++ b/bundle/deploy/terraform/util_test.go @@ -2,48 +2,15 @@ package terraform import ( "context" - "fmt" "os" "path/filepath" - "strings" "testing" - "testing/iotest" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/stretchr/testify/assert" ) -func TestLocalStateIsNewer(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 4}`) - assert.False(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateIsOlder(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 6}`) - assert.True(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateIsTheSame(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 5}`) - assert.False(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateMarkStaleWhenFailsToLoad(t *testing.T) { - local := iotest.ErrReader(fmt.Errorf("Random error")) - remote := strings.NewReader(`{"serial": 5}`) - assert.True(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := iotest.ErrReader(fmt.Errorf("Random error")) - assert.False(t, IsLocalStateStale(local, remote)) -} - func TestParseResourcesStateWithNoFile(t *testing.T) { b := &bundle.Bundle{ RootPath: t.TempDir(), From 6953a5d5af7e64d905bce5d49917729454c79284 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 18 Jul 2024 16:17:42 +0200 Subject: [PATCH 21/88] Add read-only mode for extension aware workspace filer (#1609) ## Changes By default, construct a read/write instance. If constructed in read-only mode, the underlying filer is wrapped in a readahead cache. ## Tests * Filer integration tests pass. * Manual test that caching is enabled when running on WSFS. --- bundle/config/mutator/configure_wsfs.go | 2 +- .../workspace_files_extensions_client.go | 44 ++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/bundle/config/mutator/configure_wsfs.go b/bundle/config/mutator/configure_wsfs.go index 17af4828f..c7b764f00 100644 --- a/bundle/config/mutator/configure_wsfs.go +++ b/bundle/config/mutator/configure_wsfs.go @@ -39,7 +39,7 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno // If so, swap out vfs.Path instance of the sync root with one that // makes all Workspace File System interactions extension aware. p, err := vfs.NewFilerPath(ctx, root, func(path string) (filer.Filer, error) { - return filer.NewWorkspaceFilesExtensionsClient(b.WorkspaceClient(), path) + return filer.NewReadOnlyWorkspaceFilesExtensionsClient(b.WorkspaceClient(), path) }) if err != nil { return diag.FromErr(err) diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go index d5d0ce554..844e736b5 100644 --- a/libs/filer/workspace_files_extensions_client.go +++ b/libs/filer/workspace_files_extensions_client.go @@ -18,8 +18,9 @@ import ( type workspaceFilesExtensionsClient struct { workspaceClient *databricks.WorkspaceClient - wsfs Filer - root string + wsfs Filer + root string + readonly bool } var extensionsToLanguages = map[string]workspace.Language{ @@ -143,6 +144,14 @@ func (e DuplicatePathError) Error() string { return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName) } +type ReadOnlyError struct { + op string +} + +func (e ReadOnlyError) Error() string { + return fmt.Sprintf("failed to %s: filer is in read-only mode", e.op) +} + // This is a filer for the workspace file system that allows you to pretend the // workspace file system is a traditional file system. It allows you to list, read, write, // delete, and stat notebooks (and files in general) in the workspace, using their paths @@ -157,17 +166,30 @@ func (e DuplicatePathError) Error() string { // errors for namespace clashes (e.g. a file and a notebook or a directory and a notebook). // Thus users of these methods should be careful to avoid such clashes. func NewWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + return newWorkspaceFilesExtensionsClient(w, root, false) +} + +func NewReadOnlyWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + return newWorkspaceFilesExtensionsClient(w, root, true) +} + +func newWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string, readonly bool) (Filer, error) { filer, err := NewWorkspaceFilesClient(w, root) if err != nil { return nil, err } - cache := newWorkspaceFilesReadaheadCache(filer) + if readonly { + // Wrap in a readahead cache to avoid making unnecessary calls to the workspace. + filer = newWorkspaceFilesReadaheadCache(filer) + } + return &workspaceFilesExtensionsClient{ workspaceClient: w, - wsfs: cache, - root: root, + wsfs: filer, + root: root, + readonly: readonly, }, nil } @@ -214,6 +236,10 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin // (e.g. a file and a notebook or a directory and a notebook). Thus users of this // method should be careful to avoid such clashes. func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + if w.readonly { + return ReadOnlyError{"write"} + } + return w.wsfs.Write(ctx, name, reader, mode...) } @@ -247,6 +273,10 @@ func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) // Try to delete the file as a regular file. If the file is not found, try to delete it as a notebook. func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + if w.readonly { + return ReadOnlyError{"delete"} + } + err := w.wsfs.Delete(ctx, name, mode...) // If the file is not found, it might be a notebook. @@ -293,5 +323,9 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string) // (e.g. a file and a notebook or a directory and a notebook). Thus users of this // method should be careful to avoid such clashes. func (w *workspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error { + if w.readonly { + return ReadOnlyError{"mkdir"} + } + return w.wsfs.Mkdir(ctx, name) } From 2aeea5e384e72024f819015ed9e0c0a78a125cda Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 18 Jul 2024 16:57:31 +0200 Subject: [PATCH 22/88] Remove unused package bundle/deployer (#1607) ## Changes This has been superseded by individual mutators under `bundle/deploy/terraform`. ## Tests n/a --- bundle/deployer/deployer.go | 192 ------------------------------------ 1 file changed, 192 deletions(-) delete mode 100644 bundle/deployer/deployer.go diff --git a/bundle/deployer/deployer.go b/bundle/deployer/deployer.go deleted file mode 100644 index f4f718975..000000000 --- a/bundle/deployer/deployer.go +++ /dev/null @@ -1,192 +0,0 @@ -package deployer - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - - "github.com/databricks/cli/libs/locker" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" - "github.com/hashicorp/terraform-exec/tfexec" -) - -type DeploymentStatus int - -const ( - // Empty plan produced on terraform plan. No changes need to be applied - NoChanges DeploymentStatus = iota - - // Deployment failed. No databricks assets were deployed - Failed - - // Deployment failed/partially succeeded. failed to update remote terraform - // state file. - // The partially deployed resources are thus untracked and in most cases - // will need to be cleaned up manually - PartialButUntracked - - // Deployment failed/partially succeeded. Remote terraform state file is - // updated with any partially deployed resources - Partial - - // Deployment succeeded however the remote terraform state was not updated. - // The deployed resources are thus untracked and in most cases will need to - // be cleaned up manually - CompleteButUntracked - - // Deployment succeeeded with remote terraform state file updated - Complete -) - -// Deployer is a struct to deploy a DAB to a databricks workspace -// -// Here's a high level description of what a deploy looks like: -// -// 1. Client compiles the bundle configuration to a terraform HCL config file -// -// 2. Client tries to acquire a lock on the remote root of the project. -// -- If FAIL: print details about current holder of the deployment lock on -// remote root and terminate deployment -// -// 3. Client reads terraform state from remote root -// -// 4. Client applies the diff in terraform config to the databricks workspace -// -// 5. Client updates terraform state file in remote root -// -// 6. Client releases the deploy lock on remote root -type Deployer struct { - localRoot string - remoteRoot string - env string - locker *locker.Locker - wsc *databricks.WorkspaceClient -} - -func Create(ctx context.Context, env, localRoot, remoteRoot string, wsc *databricks.WorkspaceClient) (*Deployer, error) { - user, err := wsc.CurrentUser.Me(ctx) - if err != nil { - return nil, err - } - newLocker, err := locker.CreateLocker(user.UserName, remoteRoot, wsc) - if err != nil { - return nil, err - } - return &Deployer{ - localRoot: localRoot, - remoteRoot: remoteRoot, - env: env, - locker: newLocker, - wsc: wsc, - }, nil -} - -func (b *Deployer) DefaultTerraformRoot() string { - return filepath.Join(b.localRoot, ".databricks/bundle", b.env) -} - -func (b *Deployer) tfStateRemotePath() string { - // Note: remote paths are scoped to `remoteRoot` through the locker. Also see [Create]. - return ".bundle/terraform.tfstate" -} - -func (b *Deployer) tfStateLocalPath() string { - return filepath.Join(b.DefaultTerraformRoot(), "terraform.tfstate") -} - -func (d *Deployer) LoadTerraformState(ctx context.Context) error { - r, err := d.locker.Read(ctx, d.tfStateRemotePath()) - if errors.Is(err, fs.ErrNotExist) { - // If remote tf state is absent, use local tf state - return nil - } - if err != nil { - return err - } - defer r.Close() - err = os.MkdirAll(d.DefaultTerraformRoot(), os.ModeDir) - if err != nil { - return err - } - b, err := io.ReadAll(r) - if err != nil { - return err - } - return os.WriteFile(d.tfStateLocalPath(), b, os.ModePerm) -} - -func (b *Deployer) SaveTerraformState(ctx context.Context) error { - bytes, err := os.ReadFile(b.tfStateLocalPath()) - if err != nil { - return err - } - return b.locker.Write(ctx, b.tfStateRemotePath(), bytes) -} - -func (d *Deployer) Lock(ctx context.Context, isForced bool) error { - return d.locker.Lock(ctx, isForced) -} - -func (d *Deployer) Unlock(ctx context.Context) error { - return d.locker.Unlock(ctx) -} - -func (d *Deployer) ApplyTerraformConfig(ctx context.Context, configPath, terraformBinaryPath string, isForced bool) (DeploymentStatus, error) { - applyErr := d.Lock(ctx, isForced) - if applyErr != nil { - return Failed, applyErr - } - defer func() { - applyErr = d.Unlock(ctx) - if applyErr != nil { - log.Errorf(ctx, "failed to unlock deployment mutex: %s", applyErr) - } - }() - - applyErr = d.LoadTerraformState(ctx) - if applyErr != nil { - log.Debugf(ctx, "failed to load terraform state from workspace: %s", applyErr) - return Failed, applyErr - } - - tf, applyErr := tfexec.NewTerraform(configPath, terraformBinaryPath) - if applyErr != nil { - log.Debugf(ctx, "failed to construct terraform object: %s", applyErr) - return Failed, applyErr - } - - isPlanNotEmpty, applyErr := tf.Plan(ctx) - if applyErr != nil { - log.Debugf(ctx, "failed to compute terraform plan: %s", applyErr) - return Failed, applyErr - } - - if !isPlanNotEmpty { - log.Debugf(ctx, "terraform plan returned a empty diff") - return NoChanges, nil - } - - applyErr = tf.Apply(ctx) - // upload state even if apply fails to handle partial deployments - saveStateErr := d.SaveTerraformState(ctx) - - if applyErr != nil && saveStateErr != nil { - log.Errorf(ctx, "terraform apply failed: %s", applyErr) - log.Errorf(ctx, "failed to upload terraform state after partial terraform apply: %s", saveStateErr) - return PartialButUntracked, fmt.Errorf("deploymented failed: %s", applyErr) - } - if applyErr != nil { - log.Errorf(ctx, "terraform apply failed: %s", applyErr) - return Partial, fmt.Errorf("deploymented failed: %s", applyErr) - } - if saveStateErr != nil { - log.Errorf(ctx, "failed to upload terraform state after completing terraform apply: %s", saveStateErr) - return CompleteButUntracked, fmt.Errorf("failed to upload terraform state file: %s", saveStateErr) - } - return Complete, nil -} From c8ce18ffa19338d615957dcbb9e87d36090204b8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 18 Jul 2024 17:07:43 +0200 Subject: [PATCH 23/88] Release v0.224.0 (#1604) CLI: * Do not buffer files in memory when downloading ([#1599](https://github.com/databricks/cli/pull/1599)). Bundles: * Allow artifacts (JARs, wheels) to be uploaded to UC Volumes ([#1591](https://github.com/databricks/cli/pull/1591)). * Upgrade TF provider to 1.48.3 ([#1600](https://github.com/databricks/cli/pull/1600)). * Fixed job name normalisation for bundle generate ([#1601](https://github.com/databricks/cli/pull/1601)). Internal: * Add UUID to uniquely identify a deployment state ([#1595](https://github.com/databricks/cli/pull/1595)). * Track multiple locations associated with a `dyn.Value` ([#1510](https://github.com/databricks/cli/pull/1510)). * Attribute Terraform API requests the CLI ([#1598](https://github.com/databricks/cli/pull/1598)). * Implement readahead cache for Workspace API calls ([#1582](https://github.com/databricks/cli/pull/1582)). * Use local Terraform state only when lineage match ([#1588](https://github.com/databricks/cli/pull/1588)). * Add read-only mode for extension aware workspace filer ([#1609](https://github.com/databricks/cli/pull/1609)). Dependency updates: * Bump github.com/databricks/databricks-sdk-go from 0.43.0 to 0.43.2 ([#1594](https://github.com/databricks/cli/pull/1594)). --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb902e0b4..622519f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Version changelog +## 0.224.0 + +CLI: + * Do not buffer files in memory when downloading ([#1599](https://github.com/databricks/cli/pull/1599)). + +Bundles: + * Allow artifacts (JARs, wheels) to be uploaded to UC Volumes ([#1591](https://github.com/databricks/cli/pull/1591)). + * Upgrade TF provider to 1.48.3 ([#1600](https://github.com/databricks/cli/pull/1600)). + * Fixed job name normalisation for bundle generate ([#1601](https://github.com/databricks/cli/pull/1601)). + +Internal: + * Add UUID to uniquely identify a deployment state ([#1595](https://github.com/databricks/cli/pull/1595)). + * Track multiple locations associated with a `dyn.Value` ([#1510](https://github.com/databricks/cli/pull/1510)). + * Attribute Terraform API requests the CLI ([#1598](https://github.com/databricks/cli/pull/1598)). + * Implement readahead cache for Workspace API calls ([#1582](https://github.com/databricks/cli/pull/1582)). + * Add read-only mode for extension aware workspace filer ([#1609](https://github.com/databricks/cli/pull/1609)). + +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.43.0 to 0.43.2 ([#1594](https://github.com/databricks/cli/pull/1594)). + ## 0.223.2 Bundles: From 0448307b141eaf4488786ad2311b7bf76ac74ee1 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 19 Jul 2024 09:03:25 +0200 Subject: [PATCH 24/88] Add tests for the Workspace API readahead cache (#1605) ## Changes Backfill unit tests for #1582. ## Tests New tests pass. --- libs/filer/workspace_files_cache_test.go | 319 +++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 libs/filer/workspace_files_cache_test.go diff --git a/libs/filer/workspace_files_cache_test.go b/libs/filer/workspace_files_cache_test.go new file mode 100644 index 000000000..8983c5982 --- /dev/null +++ b/libs/filer/workspace_files_cache_test.go @@ -0,0 +1,319 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "testing" + + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" +) + +var errNotImplemented = fmt.Errorf("not implemented") + +type cacheTestFiler struct { + calls int + + readDir map[string][]fs.DirEntry + stat map[string]fs.FileInfo +} + +func (m *cacheTestFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...WriteMode) error { + return errNotImplemented +} + +func (m *cacheTestFiler) Read(ctx context.Context, path string) (io.ReadCloser, error) { + return nil, errNotImplemented +} + +func (m *cacheTestFiler) Delete(ctx context.Context, path string, mode ...DeleteMode) error { + return errNotImplemented +} + +func (m *cacheTestFiler) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) { + m.calls++ + if fi, ok := m.readDir[path]; ok { + delete(m.readDir, path) + return fi, nil + } + return nil, fs.ErrNotExist +} + +func (m *cacheTestFiler) Mkdir(ctx context.Context, path string) error { + return errNotImplemented +} + +func (m *cacheTestFiler) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + m.calls++ + if fi, ok := m.stat[name]; ok { + delete(m.stat, name) + return fi, nil + } + return nil, fs.ErrNotExist +} + +func TestWorkspaceFilesCache_ReadDirCache(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file2", + Size: 2, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First read dir should hit the filer, second should hit the cache. + for range 2 { + fi, err := cache.ReadDir(ctx, "dir1") + assert.NoError(t, err) + if assert.Len(t, fi, 2) { + assert.Equal(t, "file1", fi[0].Name()) + assert.Equal(t, "file2", fi[1].Name()) + } + } + + // Third stat should hit the filer, fourth should hit the cache. + for range 2 { + _, err := cache.ReadDir(ctx, "dir2") + assert.ErrorIs(t, err, fs.ErrNotExist) + } + + // Assert we only called the filer twice. + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirCacheIsolation(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First read dir should hit the filer, second should hit the cache. + entries, err := cache.ReadDir(ctx, "dir") + assert.NoError(t, err) + assert.Equal(t, "file", entries[0].Name()) + + // Modify the entry to check that mutations are not reflected in the cache. + entries[0] = wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "tainted", + }, + }, + } + + // Read the directory again to check that the cache is isolated. + entries, err = cache.ReadDir(ctx, "dir") + assert.NoError(t, err) + assert.Equal(t, "file", entries[0].Name()) +} + +func TestWorkspaceFilesCache_StatCache(t *testing.T) { + f := &cacheTestFiler{ + stat: map[string]fs.FileInfo{ + "file1": &wsfsFileInfo{ObjectInfo: workspace.ObjectInfo{Path: "file1", Size: 1}}, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First stat should hit the filer, second should hit the cache. + for range 2 { + fi, err := cache.Stat(ctx, "file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + } + + // Third stat should hit the filer, fourth should hit the cache. + for range 2 { + _, err := cache.Stat(ctx, "file2") + assert.ErrorIs(t, err, fs.ErrNotExist) + } + + // Assert we only called the filer twice. + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirPopulatesStatCache(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file2", + Size: 2, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "notebook1", + Size: 1, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: "this should not end up in the stat cache", + }, + }, + }, + }, + stat: map[string]fs.FileInfo{ + "dir1/notebook1": wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "notebook1", + Size: 1, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: workspace.ExportFormatJupyter, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // Issue read dir to populate the stat cache. + _, err := cache.ReadDir(ctx, "dir1") + assert.NoError(t, err) + + // Stat on a file in the directory should hit the cache. + fi, err := cache.Stat(ctx, "dir1/file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + + // If the containing directory has been read, absence is also inferred from the cache. + _, err = cache.Stat(ctx, "dir1/file3") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Stat on a notebook in the directory should have been performed in the background. + fi, err = cache.Stat(ctx, "dir1/notebook1") + if assert.NoError(t, err) { + assert.Equal(t, "notebook1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + assert.Equal(t, workspace.ExportFormatJupyter, fi.(wsfsFileInfo).ReposExportFormat) + } + + // Assert we called the filer twice (once for read dir, once for stat on the notebook). + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirTriggersReadahead(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "a": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "b1", + ObjectType: workspace.ObjectTypeDirectory, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "b2", + ObjectType: workspace.ObjectTypeDirectory, + }, + }, + }, + }, + "a/b1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + "a/b2": {}, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // Issue read dir to populate the stat cache. + _, err := cache.ReadDir(ctx, "a") + assert.NoError(t, err) + + // Stat on a directory in the directory should hit the cache. + fi, err := cache.Stat(ctx, "a/b1") + if assert.NoError(t, err) { + assert.Equal(t, "b1", fi.Name()) + assert.True(t, fi.IsDir()) + } + + // Stat on a file in a nested directory should hit the cache. + fi, err = cache.Stat(ctx, "a/b1/file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + + // Stat on a non-existing file in an empty nested directory should hit the cache. + _, err = cache.Stat(ctx, "a/b2/file2") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Assert we called the filer 3 times; once for each directory. + assert.Equal(t, 3, f.calls) +} From 15ca7fe62d522c5efc5fae73d911cbde03e116b2 Mon Sep 17 00:00:00 2001 From: Arpit Jasapara <87999496+arpitjasa-db@users.noreply.github.com> Date: Fri, 19 Jul 2024 04:38:20 -0700 Subject: [PATCH 25/88] Add UUID function to bundle template functions (#1612) ## Changes Add support for google/uuid.New() to DAB templates. This is needed to generate UUIDs in downstream templates like MLOps Stacks. ## Tests Unit tests. --- libs/template/helpers.go | 6 ++++++ libs/template/helpers_test.go | 17 +++++++++++++++++ libs/template/testdata/uuid/template/hello.tmpl | 1 + 3 files changed, 24 insertions(+) create mode 100644 libs/template/testdata/uuid/template/hello.tmpl diff --git a/libs/template/helpers.go b/libs/template/helpers.go index b3dea329e..1dfe74d73 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -14,6 +14,8 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/iam" + + "github.com/google/uuid" ) type ErrFail struct { @@ -51,6 +53,10 @@ func loadHelpers(ctx context.Context) template.FuncMap { "random_int": func(n int) int { return rand.Intn(n) }, + // Alias for https://pkg.go.dev/github.com/google/uuid#New. Returns, as a string, a UUID which is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC 4122. + "uuid": func() string { + return uuid.New().String() + }, // A key value pair. This is used with the map function to generate maps // to use inside a template "pair": func(k string, v any) pair { diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go index c0848c8d0..8cc7b928e 100644 --- a/libs/template/helpers_test.go +++ b/libs/template/helpers_test.go @@ -69,6 +69,23 @@ func TestTemplateRandIntFunction(t *testing.T) { assert.Empty(t, err) } +func TestTemplateUuidFunction(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ctx = root.SetWorkspaceClient(ctx, nil) + helpers := loadHelpers(ctx) + r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir) + require.NoError(t, err) + + err = r.walk() + assert.NoError(t, err) + + assert.Len(t, r.files, 1) + uuid := strings.TrimSpace(string(r.files[0].(*inMemoryFile).content)) + assert.Regexp(t, "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", uuid) +} + func TestTemplateUrlFunction(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() diff --git a/libs/template/testdata/uuid/template/hello.tmpl b/libs/template/testdata/uuid/template/hello.tmpl new file mode 100644 index 000000000..178c2a9c4 --- /dev/null +++ b/libs/template/testdata/uuid/template/hello.tmpl @@ -0,0 +1 @@ +{{print (uuid)}} From 52ca599cd5c87a4131d96863a90cef69da51e095 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 23 Jul 2024 18:15:02 +0200 Subject: [PATCH 26/88] Upgrade TF provider to 1.49.0 (#1617) ## Changes This includes a fix for model serving endpoints. See https://github.com/databricks/terraform-provider-databricks/pull/3690. ## Tests n/a --- bundle/internal/tf/codegen/schema/version.go | 2 +- .../internal/tf/schema/data_source_cluster.go | 232 +++++++++++++++++- .../internal/tf/schema/data_source_schema.go | 36 +++ .../internal/tf/schema/data_source_volume.go | 38 +++ bundle/internal/tf/schema/data_sources.go | 4 + .../internal/tf/schema/resource_dashboard.go | 21 ++ .../tf/schema/resource_permissions.go | 1 + .../tf/schema/resource_workspace_binding.go | 12 + bundle/internal/tf/schema/resources.go | 4 + bundle/internal/tf/schema/root.go | 2 +- 10 files changed, 339 insertions(+), 13 deletions(-) create mode 100644 bundle/internal/tf/schema/data_source_schema.go create mode 100644 bundle/internal/tf/schema/data_source_volume.go create mode 100644 bundle/internal/tf/schema/resource_dashboard.go create mode 100644 bundle/internal/tf/schema/resource_workspace_binding.go diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 63a4b1b78..aecb2736d 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.48.3" +const ProviderVersion = "1.49.0" diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index fff66dc93..94d67bbfa 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -10,7 +10,9 @@ type DataSourceClusterClusterInfoAutoscale struct { type DataSourceClusterClusterInfoAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -18,10 +20,16 @@ type DataSourceClusterClusterInfoAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type DataSourceClusterClusterInfoAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type DataSourceClusterClusterInfoClusterLogConfDbfs struct { @@ -49,12 +57,12 @@ type DataSourceClusterClusterInfoClusterLogStatus struct { } type DataSourceClusterClusterInfoDockerImageBasicAuth struct { - Password string `json:"password"` - Username string `json:"username"` + Password string `json:"password,omitempty"` + Username string `json:"username,omitempty"` } type DataSourceClusterClusterInfoDockerImage struct { - Url string `json:"url"` + Url string `json:"url,omitempty"` BasicAuth *DataSourceClusterClusterInfoDockerImageBasicAuth `json:"basic_auth,omitempty"` } @@ -139,12 +147,212 @@ type DataSourceClusterClusterInfoInitScripts struct { Workspace *DataSourceClusterClusterInfoInitScriptsWorkspace `json:"workspace,omitempty"` } +type DataSourceClusterClusterInfoSpecAutoscale struct { + MaxWorkers int `json:"max_workers,omitempty"` + MinWorkers int `json:"min_workers,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAwsAttributes struct { + Availability string `json:"availability,omitempty"` + EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` + EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` + EbsVolumeType string `json:"ebs_volume_type,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + SpotBidPricePercent int `json:"spot_bid_price_percent,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributes struct { + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConf struct { + Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { + MountOptions string `json:"mount_options,omitempty"` + ServerAddress string `json:"server_address"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfo struct { + LocalMountDirPath string `json:"local_mount_dir_path"` + RemoteMountDirPath string `json:"remote_mount_dir_path,omitempty"` + NetworkFilesystemInfo *DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo `json:"network_filesystem_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecDockerImageBasicAuth struct { + Password string `json:"password"` + Username string `json:"username"` +} + +type DataSourceClusterClusterInfoSpecDockerImage struct { + Url string `json:"url"` + BasicAuth *DataSourceClusterClusterInfoSpecDockerImageBasicAuth `json:"basic_auth,omitempty"` +} + +type DataSourceClusterClusterInfoSpecGcpAttributes struct { + Availability string `json:"availability,omitempty"` + BootDiskSize int `json:"boot_disk_size,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + LocalSsdCount int `json:"local_ssd_count,omitempty"` + UsePreemptibleExecutors bool `json:"use_preemptible_executors,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsAbfss struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsFile struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsGcs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsVolumes struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsWorkspace struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScripts struct { + Abfss *DataSourceClusterClusterInfoSpecInitScriptsAbfss `json:"abfss,omitempty"` + Dbfs *DataSourceClusterClusterInfoSpecInitScriptsDbfs `json:"dbfs,omitempty"` + File *DataSourceClusterClusterInfoSpecInitScriptsFile `json:"file,omitempty"` + Gcs *DataSourceClusterClusterInfoSpecInitScriptsGcs `json:"gcs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecInitScriptsS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoSpecInitScriptsVolumes `json:"volumes,omitempty"` + Workspace *DataSourceClusterClusterInfoSpecInitScriptsWorkspace `json:"workspace,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceClusterClusterInfoSpecLibraryCran `json:"cran,omitempty"` + Maven *DataSourceClusterClusterInfoSpecLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceClusterClusterInfoSpecLibraryPypi `json:"pypi,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadType struct { + Clients *DataSourceClusterClusterInfoSpecWorkloadTypeClients `json:"clients,omitempty"` +} + +type DataSourceClusterClusterInfoSpec struct { + ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + DataSecurityMode string `json:"data_security_mode,omitempty"` + DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` + DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` + IdempotencyToken string `json:"idempotency_token,omitempty"` + InstancePoolId string `json:"instance_pool_id,omitempty"` + NodeTypeId string `json:"node_type_id,omitempty"` + NumWorkers int `json:"num_workers,omitempty"` + PolicyId string `json:"policy_id,omitempty"` + RuntimeEngine string `json:"runtime_engine,omitempty"` + SingleUserName string `json:"single_user_name,omitempty"` + SparkConf map[string]string `json:"spark_conf,omitempty"` + SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` + SparkVersion string `json:"spark_version"` + SshPublicKeys []string `json:"ssh_public_keys,omitempty"` + Autoscale *DataSourceClusterClusterInfoSpecAutoscale `json:"autoscale,omitempty"` + AwsAttributes *DataSourceClusterClusterInfoSpecAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *DataSourceClusterClusterInfoSpecAzureAttributes `json:"azure_attributes,omitempty"` + ClusterLogConf *DataSourceClusterClusterInfoSpecClusterLogConf `json:"cluster_log_conf,omitempty"` + ClusterMountInfo []DataSourceClusterClusterInfoSpecClusterMountInfo `json:"cluster_mount_info,omitempty"` + DockerImage *DataSourceClusterClusterInfoSpecDockerImage `json:"docker_image,omitempty"` + GcpAttributes *DataSourceClusterClusterInfoSpecGcpAttributes `json:"gcp_attributes,omitempty"` + InitScripts []DataSourceClusterClusterInfoSpecInitScripts `json:"init_scripts,omitempty"` + Library []DataSourceClusterClusterInfoSpecLibrary `json:"library,omitempty"` + WorkloadType *DataSourceClusterClusterInfoSpecWorkloadType `json:"workload_type,omitempty"` +} + type DataSourceClusterClusterInfoTerminationReason struct { Code string `json:"code,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` Type string `json:"type,omitempty"` } +type DataSourceClusterClusterInfoWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoWorkloadType struct { + Clients *DataSourceClusterClusterInfoWorkloadTypeClients `json:"clients,omitempty"` +} + type DataSourceClusterClusterInfo struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterCores int `json:"cluster_cores,omitempty"` @@ -155,14 +363,14 @@ type DataSourceClusterClusterInfo struct { CreatorUserName string `json:"creator_user_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags"` + DefaultTags map[string]string `json:"default_tags,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` JdbcPort int `json:"jdbc_port,omitempty"` - LastActivityTime int `json:"last_activity_time,omitempty"` + LastRestartedTime int `json:"last_restarted_time,omitempty"` LastStateLossTime int `json:"last_state_loss_time,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` @@ -172,12 +380,12 @@ type DataSourceClusterClusterInfo struct { SparkConf map[string]string `json:"spark_conf,omitempty"` SparkContextId int `json:"spark_context_id,omitempty"` SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` + SparkVersion string `json:"spark_version,omitempty"` SshPublicKeys []string `json:"ssh_public_keys,omitempty"` StartTime int `json:"start_time,omitempty"` - State string `json:"state"` + State string `json:"state,omitempty"` StateMessage string `json:"state_message,omitempty"` - TerminateTime int `json:"terminate_time,omitempty"` + TerminatedTime int `json:"terminated_time,omitempty"` Autoscale *DataSourceClusterClusterInfoAutoscale `json:"autoscale,omitempty"` AwsAttributes *DataSourceClusterClusterInfoAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *DataSourceClusterClusterInfoAzureAttributes `json:"azure_attributes,omitempty"` @@ -188,7 +396,9 @@ type DataSourceClusterClusterInfo struct { Executors []DataSourceClusterClusterInfoExecutors `json:"executors,omitempty"` GcpAttributes *DataSourceClusterClusterInfoGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []DataSourceClusterClusterInfoInitScripts `json:"init_scripts,omitempty"` + Spec *DataSourceClusterClusterInfoSpec `json:"spec,omitempty"` TerminationReason *DataSourceClusterClusterInfoTerminationReason `json:"termination_reason,omitempty"` + WorkloadType *DataSourceClusterClusterInfoWorkloadType `json:"workload_type,omitempty"` } type DataSourceCluster struct { diff --git a/bundle/internal/tf/schema/data_source_schema.go b/bundle/internal/tf/schema/data_source_schema.go new file mode 100644 index 000000000..9d778cc88 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_schema.go @@ -0,0 +1,36 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag struct { + InheritedFromName string `json:"inherited_from_name,omitempty"` + InheritedFromType string `json:"inherited_from_type,omitempty"` + Value string `json:"value"` +} + +type DataSourceSchemaSchemaInfo struct { + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + CatalogType string `json:"catalog_type,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + SchemaId string `json:"schema_id,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + EffectivePredictiveOptimizationFlag *DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"` +} + +type DataSourceSchema struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + SchemaInfo *DataSourceSchemaSchemaInfo `json:"schema_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_volume.go b/bundle/internal/tf/schema/data_source_volume.go new file mode 100644 index 000000000..67e6100f6 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_volume.go @@ -0,0 +1,38 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails struct { + Algorithm string `json:"algorithm,omitempty"` + AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"` +} + +type DataSourceVolumeVolumeInfoEncryptionDetails struct { + SseEncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"` +} + +type DataSourceVolumeVolumeInfo struct { + AccessPoint string `json:"access_point,omitempty"` + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + VolumeId string `json:"volume_id,omitempty"` + VolumeType string `json:"volume_type,omitempty"` + EncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetails `json:"encryption_details,omitempty"` +} + +type DataSourceVolume struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + VolumeInfo *DataSourceVolumeVolumeInfo `json:"volume_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index b68df2b40..4ac78613f 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -36,6 +36,7 @@ type DataSources struct { Notebook map[string]any `json:"databricks_notebook,omitempty"` NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` Schemas map[string]any `json:"databricks_schemas,omitempty"` ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` @@ -50,6 +51,7 @@ type DataSources struct { Tables map[string]any `json:"databricks_tables,omitempty"` User map[string]any `json:"databricks_user,omitempty"` Views map[string]any `json:"databricks_views,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` Volumes map[string]any `json:"databricks_volumes,omitempty"` Zones map[string]any `json:"databricks_zones,omitempty"` } @@ -89,6 +91,7 @@ func NewDataSources() *DataSources { Notebook: make(map[string]any), NotebookPaths: make(map[string]any), Pipelines: make(map[string]any), + Schema: make(map[string]any), Schemas: make(map[string]any), ServicePrincipal: make(map[string]any), ServicePrincipals: make(map[string]any), @@ -103,6 +106,7 @@ func NewDataSources() *DataSources { Tables: make(map[string]any), User: make(map[string]any), Views: make(map[string]any), + Volume: make(map[string]any), Volumes: make(map[string]any), Zones: make(map[string]any), } diff --git a/bundle/internal/tf/schema/resource_dashboard.go b/bundle/internal/tf/schema/resource_dashboard.go new file mode 100644 index 000000000..0c2fa4a0f --- /dev/null +++ b/bundle/internal/tf/schema/resource_dashboard.go @@ -0,0 +1,21 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceDashboard struct { + CreateTime string `json:"create_time,omitempty"` + DashboardChangeDetected bool `json:"dashboard_change_detected,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DisplayName string `json:"display_name"` + EmbedCredentials bool `json:"embed_credentials,omitempty"` + Etag string `json:"etag,omitempty"` + FilePath string `json:"file_path,omitempty"` + Id string `json:"id,omitempty"` + LifecycleState string `json:"lifecycle_state,omitempty"` + Md5 string `json:"md5,omitempty"` + ParentPath string `json:"parent_path"` + Path string `json:"path,omitempty"` + SerializedDashboard string `json:"serialized_dashboard,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + WarehouseId string `json:"warehouse_id"` +} diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index 5d8df11e7..ee94a1a8f 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -13,6 +13,7 @@ type ResourcePermissions struct { Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` DirectoryId string `json:"directory_id,omitempty"` DirectoryPath string `json:"directory_path,omitempty"` ExperimentId string `json:"experiment_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_workspace_binding.go b/bundle/internal/tf/schema/resource_workspace_binding.go new file mode 100644 index 000000000..f0be7a41f --- /dev/null +++ b/bundle/internal/tf/schema/resource_workspace_binding.go @@ -0,0 +1,12 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceWorkspaceBinding struct { + BindingType string `json:"binding_type,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Id string `json:"id,omitempty"` + SecurableName string `json:"securable_name,omitempty"` + SecurableType string `json:"securable_type,omitempty"` + WorkspaceId int `json:"workspace_id,omitempty"` +} diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79d71a65f..79c1b32b5 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -16,6 +16,7 @@ type Resources struct { ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` Connection map[string]any `json:"databricks_connection,omitempty"` + Dashboard map[string]any `json:"databricks_dashboard,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -96,6 +97,7 @@ type Resources struct { VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } @@ -115,6 +117,7 @@ func NewResources() *Resources { ClusterPolicy: make(map[string]any), ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), Connection: make(map[string]any), + Dashboard: make(map[string]any), DbfsFile: make(map[string]any), DefaultNamespaceSetting: make(map[string]any), Directory: make(map[string]any), @@ -195,6 +198,7 @@ func NewResources() *Resources { VectorSearchEndpoint: make(map[string]any), VectorSearchIndex: make(map[string]any), Volume: make(map[string]any), + WorkspaceBinding: make(map[string]any), WorkspaceConf: make(map[string]any), WorkspaceFile: make(map[string]any), } diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index a79e998cf..8401d8dac 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.48.3" +const ProviderVersion = "1.49.0" func NewRoot() *Root { return &Root{ From 4bf88b4209ee3bbaabcfac0b817e6c98cdecf328 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:50:11 +0530 Subject: [PATCH 27/88] Support multiple locations for diagnostics (#1610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This PR changes `diag.Diagnostics` to allow including multiple locations associated with the diagnostic message. The diagnostics that now return multiple locations with this PR are: 1. Warning for unknown keys in config. 2. Use of experimental.run_as 3. Accidental sync.exludes that exclude all files. ## Tests Existing unit tests pass. New unit test case to assert on error message when multiple locations are included. Example output: ``` ➜ bundle-playground-2 ~/cli2/cli/cli bundle validate Warning: You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC. at experimental.use_legacy_run_as in resources.yml:10:22 databricks.yml:13:22 Name: fix run_if Target: default Workspace: User: shreyas.goenka@databricks.com Path: /Users/shreyas.goenka@databricks.com/.bundle/fix run_if/default Found 1 warning ``` --- .../mutator/python/python_diagnostics.go | 16 +- .../mutator/python/python_diagnostics_test.go | 10 +- .../mutator/python/python_mutator_test.go | 13 +- bundle/config/mutator/run_as.go | 8 +- bundle/config/root.go | 11 + bundle/config/validate/files_to_sync.go | 6 +- .../validate/job_cluster_key_defined.go | 8 +- bundle/config/validate/validate.go | 4 + .../config/validate/validate_sync_patterns.go | 9 +- bundle/render/render_text_output.go | 26 +- bundle/render/render_text_output_test.go | 91 +++---- .../sync_include_exclude_no_matches_test.go | 9 +- libs/diag/diagnostic.go | 6 +- libs/dyn/convert/normalize.go | 53 +++-- libs/dyn/convert/normalize_test.go | 224 +++++++++--------- 15 files changed, 276 insertions(+), 218 deletions(-) diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index b8efc9ef7..96baa5093 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -55,12 +55,18 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { return nil, fmt.Errorf("failed to parse path: %s", err) } + var locations []dyn.Location + location := convertPythonLocation(parsedLine.Location) + if location != (dyn.Location{}) { + locations = append(locations, location) + } + diag := diag.Diagnostic{ - Severity: severity, - Summary: parsedLine.Summary, - Detail: parsedLine.Detail, - Location: convertPythonLocation(parsedLine.Location), - Path: path, + Severity: severity, + Summary: parsedLine.Summary, + Detail: parsedLine.Detail, + Locations: locations, + Path: path, } diags = diags.Append(diag) diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 7b66e2537..09d9f93bd 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -39,10 +39,12 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Location: dyn.Location{ - File: "src/examples/file.py", - Line: 1, - Column: 2, + Locations: []dyn.Location{ + { + File: "src/examples/file.py", + Line: 1, + Column: 2, + }, }, }, }, diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 588589831..fbe835f92 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -97,11 +97,14 @@ func TestPythonMutator_load(t *testing.T) { assert.Equal(t, 1, len(diags)) assert.Equal(t, "job doesn't have any tasks", diags[0].Summary) - assert.Equal(t, dyn.Location{ - File: "src/examples/file.py", - Line: 10, - Column: 5, - }, diags[0].Location) + assert.Equal(t, []dyn.Location{ + { + File: "src/examples/file.py", + Line: 10, + Column: 5, + }, + }, diags[0].Locations) + } func TestPythonMutator_load_disallowed(t *testing.T) { diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index d344a988a..168918d0d 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -178,10 +178,10 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { setRunAsForJobs(b) return diag.Diagnostics{ { - Severity: diag.Warning, - Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Path: dyn.MustPathFromString("experimental.use_legacy_run_as"), - Location: b.Config.GetLocation("experimental.use_legacy_run_as"), + Severity: diag.Warning, + Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", + Path: dyn.MustPathFromString("experimental.use_legacy_run_as"), + Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } } diff --git a/bundle/config/root.go b/bundle/config/root.go index 594a9105f..4239a34d0 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -524,6 +524,17 @@ func (r Root) GetLocation(path string) dyn.Location { return v.Location() } +// Get all locations of the configuration value at the specified path. We need both +// this function and it's singular version (GetLocation) because some diagnostics just need +// the primary location and some need all locations associated with a configuration value. +func (r Root) GetLocations(path string) []dyn.Location { + v, err := dyn.Get(r.value, path) + if err != nil { + return []dyn.Location{} + } + return v.Locations() +} + // Value returns the dynamic configuration value of the root object. This value // is the source of truth and is kept in sync with values in the typed configuration. func (r Root) Value() dyn.Value { diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index d53e38243..ae6bfef1a 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -45,8 +45,10 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag. diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", - Location: loc.Location(), - Path: loc.Path(), + // Show all locations where sync.exclude is defined, since merging + // sync.exclude is additive. + Locations: loc.Locations(), + Path: loc.Path(), }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 37ed3f417..168303d83 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -41,8 +42,11 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("job_cluster_key %s is not defined", task.JobClusterKey), - Location: loc.Location(), - Path: loc.Path(), + // Show only the location where the job_cluster_key is defined. + // Other associated locations are not relevant since they are + // overridden during merging. + Locations: []dyn.Location{loc.Location()}, + Path: loc.Path(), }) } } diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index af7e984a1..b4da0bc05 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -20,6 +20,10 @@ func (l location) Location() dyn.Location { return l.rb.Config().GetLocation(l.path) } +func (l location) Locations() []dyn.Location { + return l.rb.Config().GetLocations(l.path) +} + func (l location) Path() dyn.Path { return dyn.MustPathFromString(l.path) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index a04c10776..f3655ca94 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/fileset" "golang.org/x/sync/errgroup" ) @@ -64,10 +65,10 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di loc := location{path: fmt.Sprintf("%s[%d]", path, index), rb: rb} mu.Lock() diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("Pattern %s does not match any files", p), - Location: loc.Location(), - Path: loc.Path(), + Severity: diag.Warning, + Summary: fmt.Sprintf("Pattern %s does not match any files", p), + Locations: []dyn.Location{loc.Location()}, + Path: loc.Path(), }) mu.Unlock() } diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 439ae6132..2ef6b2656 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -32,8 +32,8 @@ const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} {{- if .Path.String }} {{ "at " }}{{ .Path.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -46,8 +46,8 @@ const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} {{- if .Path.String }} {{ "at " }}{{ .Path.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -141,12 +141,18 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) t = warningT } - // Make file relative to bundle root - if d.Location.File != "" && b != nil { - out, err := filepath.Rel(b.RootPath, d.Location.File) - // if we can't relativize the path, just use path as-is - if err == nil { - d.Location.File = out + for i := range d.Locations { + if b == nil { + break + } + + // Make location relative to bundle root + if d.Locations[i].File != "" { + out, err := filepath.Rel(b.RootPath, d.Locations[i].File) + // if we can't relativize the path, just use path as-is + if err == nil { + d.Locations[i].File = out + } } } diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index b7aec8864..81e276199 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -88,34 +88,22 @@ func TestRenderTextOutput(t *testing.T) { bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (1)", - Detail: "detail (1)", - Location: dyn.Location{ - File: "foo.py", - Line: 1, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (2)", - Detail: "detail (2)", - Location: dyn.Location{ - File: "foo.py", - Line: 2, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning (3)", - Detail: "detail (3)", - Location: dyn.Location{ - File: "foo.py", - Line: 3, - Column: 1, - }, + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, }, opts: RenderOptions{RenderSummaryTable: true}, @@ -174,24 +162,16 @@ func TestRenderTextOutput(t *testing.T) { bundle: nil, diags: diag.Diagnostics{ diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (1)", - Detail: "detail (1)", - Location: dyn.Location{ - File: "foo.py", - Line: 1, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning (2)", - Detail: "detail (2)", - Location: dyn.Location{ - File: "foo.py", - Line: 3, - Column: 1, - }, + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, }, opts: RenderOptions{RenderSummaryTable: false}, @@ -252,17 +232,42 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Summary: "failed to load xxx", Detail: "'name' is required", - Location: dyn.Location{ + Locations: []dyn.Location{{ File: "foo.yaml", Line: 1, - Column: 2, - }, + Column: 2}}, }, }, expected: "Error: failed to load xxx\n" + " in foo.yaml:1:2\n\n" + "'name' is required\n\n", }, + { + name: "error with multiple source locations", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "failed to load xxx", + Detail: "'name' is required", + Locations: []dyn.Location{ + { + File: "foo.yaml", + Line: 1, + Column: 2, + }, + { + File: "bar.yaml", + Line: 3, + Column: 4, + }, + }, + }, + }, + expected: "Error: failed to load xxx\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "'name' is required\n\n", + }, { name: "error with path", diags: diag.Diagnostics{ diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 94cedbaa6..5f4fa47ce 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,11 +22,13 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[0].Severity, diag.Warning) require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") - require.Equal(t, diags[0].Location.File, filepath.Join("sync", "override", "databricks.yml")) - require.Equal(t, diags[0].Location.Line, 17) - require.Equal(t, diags[0].Location.Column, 11) require.Equal(t, diags[0].Path.String(), "sync.exclude[0]") + assert.Len(t, diags[0].Locations, 1) + require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) + require.Equal(t, diags[0].Locations[0].Line, 17) + require.Equal(t, diags[0].Locations[0].Column, 11) + summaries := []string{ fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")), fmt.Sprintf("Pattern %s does not match any files", filepath.Join("tests", "*")), diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index 621527551..305089d22 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -17,9 +17,9 @@ type Diagnostic struct { // This may be multiple lines and may be nil. Detail string - // Location is a source code location associated with the diagnostic message. - // It may be zero if there is no associated location. - Location dyn.Location + // Locations are the source code locations associated with the diagnostic message. + // It may be empty if there are no associated locations. + Locations []dyn.Location // Path is a path to the value in a configuration tree that the diagnostic is associated with. // It may be nil if there is no associated path. diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 246c97eaf..bf5756e7f 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -65,19 +65,19 @@ func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen [] func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected a %s value, found null", expected), + Locations: []dyn.Location{src.Location()}, + Path: path, } } func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), + Locations: []dyn.Location{src.Location()}, + Path: path, } } @@ -98,8 +98,9 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), - Location: pk.Location(), - Path: path, + // Show all locations the unknown field is defined at. + Locations: pk.Locations(), + Path: path, }) } continue @@ -320,10 +321,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn out = int64(src.MustFloat()) if src.MustFloat() != float64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), + Locations: []dyn.Location{src.Location()}, + Path: path, }) } case dyn.KindString: @@ -336,10 +337,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Path: path, }) } case dyn.KindNil: @@ -363,10 +364,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d out = float64(src.MustInt()) if src.MustInt() != int64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), + Locations: []dyn.Location{src.Location()}, + Path: path, }) } case dyn.KindString: @@ -379,10 +380,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Path: path, }) } case dyn.KindNil: diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 452ed4eb1..df9a1a9a5 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -40,10 +40,10 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.NewPath(dyn.Key("bar")), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -58,23 +58,33 @@ func TestNormalizeStructUnknownField(t *testing.T) { } var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) + + m := dyn.NewMapping() + m.Set(dyn.V("foo"), dyn.V("val-foo")) + // Set the unknown field, with location information. + m.Set(dyn.NewValue("bar", []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }), dyn.V("var-bar")) + + vin := dyn.V(m) vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ Severity: diag.Warning, Summary: `unknown field: bar`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + // Assert location of the unknown field is included in the diagnostic. + Locations: []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }, + Path: dyn.EmptyPath, }, err[0]) // The field that can be mapped to the struct field is retained. assert.Equal(t, map[string]any{ - "foo": "bar", + "foo": "val-foo", }, vout.AsAny()) } @@ -100,10 +110,10 @@ func TestNormalizeStructError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Get("foo").Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -245,10 +255,10 @@ func TestNormalizeStructRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -262,10 +272,10 @@ func TestNormalizeStructIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -291,10 +301,10 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.NewPath(dyn.Key("bar")), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -317,10 +327,10 @@ func TestNormalizeMapError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -372,10 +382,10 @@ func TestNormalizeMapRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -385,10 +395,10 @@ func TestNormalizeMapIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -415,10 +425,10 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Index(2)), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.NewPath(dyn.Index(2)), }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -439,10 +449,10 @@ func TestNormalizeSliceError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -494,10 +504,10 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -507,10 +517,10 @@ func TestNormalizeSliceIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found int`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -528,10 +538,10 @@ func TestNormalizeStringNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a string value, found null`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -565,10 +575,10 @@ func TestNormalizeStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.EmptyPath, }, err[0]) } @@ -586,10 +596,10 @@ func TestNormalizeBoolNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a bool value, found null`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -628,10 +638,10 @@ func TestNormalizeBoolFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found string`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -641,10 +651,10 @@ func TestNormalizeBoolError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.EmptyPath, }, err[0]) } @@ -662,10 +672,10 @@ func TestNormalizeIntNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a int value, found null`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -683,10 +693,10 @@ func TestNormalizeIntFromFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "1.5" as integer due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -712,10 +722,10 @@ func TestNormalizeIntFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as an integer`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -725,10 +735,10 @@ func TestNormalizeIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected int, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.EmptyPath, }, err[0]) } @@ -746,10 +756,10 @@ func TestNormalizeFloatNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a float value, found null`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -771,10 +781,10 @@ func TestNormalizeFloatFromIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -800,10 +810,10 @@ func TestNormalizeFloatFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as a floating point number`, + Locations: []dyn.Location{vin.Location()}, + Path: dyn.EmptyPath, }, err[0]) } @@ -813,10 +823,10 @@ func TestNormalizeFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected float, found map`, + Locations: []dyn.Location{{}}, + Path: dyn.EmptyPath, }, err[0]) } From 39fc86e83b473384ec55c4fee4eae0ac0cd8bd3b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 24 Jul 2024 11:13:49 +0200 Subject: [PATCH 28/88] Split artifact cleanup into prepare step before build (#1618) ## Changes Now prepare stage which does cleanup is execute once before every build, so artifacts built into the same folder are correctly kept Fixes workaround 2 from this issue #1602 ## Tests Added unit test --- bundle/artifacts/artifacts.go | 16 ++++++ bundle/artifacts/build.go | 19 ------- bundle/artifacts/prepare.go | 57 +++++++++++++++++++ bundle/artifacts/upload_test.go | 7 ++- bundle/artifacts/whl/build.go | 9 +-- bundle/artifacts/whl/prepare.go | 48 ++++++++++++++++ bundle/phases/build.go | 1 + .../python_wheel_multiple/.gitignore | 3 + .../python_wheel_multiple/bundle.yml | 25 ++++++++ .../my_test_code/setup.py | 15 +++++ .../my_test_code/setup2.py | 15 +++++ .../my_test_code/src/__init__.py | 2 + .../my_test_code/src/__main__.py | 16 ++++++ bundle/tests/python_wheel_test.go | 17 ++++++ 14 files changed, 223 insertions(+), 27 deletions(-) create mode 100644 bundle/artifacts/prepare.go create mode 100644 bundle/artifacts/whl/prepare.go create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/.gitignore create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/bundle.yml create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py create mode 100644 bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index 15565cd60..3060d08d9 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts/whl" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" @@ -29,6 +30,10 @@ var buildMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactTy var uploadMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{} +var prepareMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{ + config.ArtifactPythonWheel: whl.Prepare, +} + func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { mutatorFactory, ok := buildMutators[t] if !ok { @@ -47,6 +52,17 @@ func getUploadMutator(t config.ArtifactType, name string) bundle.Mutator { return mutatorFactory(name) } +func getPrepareMutator(t config.ArtifactType, name string) bundle.Mutator { + mutatorFactory, ok := prepareMutators[t] + if !ok { + mutatorFactory = func(_ string) bundle.Mutator { + return mutator.NoOp() + } + } + + return mutatorFactory(name) +} + // Basic Build defines a general build mutator which builds artifact based on artifact.BuildCommand type basicBuild struct { name string diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index c8c3bf67c..47c2f24d8 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -35,15 +35,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("artifact doesn't exist: %s", m.name) } - // Check if source paths are absolute, if not, make them absolute - for k := range artifact.Files { - f := &artifact.Files[k] - if !filepath.IsAbs(f.Source) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - f.Source = filepath.Join(dirPath, f.Source) - } - } - // Skip building if build command is not specified or infered if artifact.BuildCommand == "" { // If no build command was specified or infered and there is no @@ -58,16 +49,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diags } - // If artifact path is not provided, use bundle root dir - if artifact.Path == "" { - artifact.Path = b.RootPath - } - - if !filepath.IsAbs(artifact.Path) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - artifact.Path = filepath.Join(dirPath, artifact.Path) - } - diags := bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) if diags.HasError() { return diags diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go new file mode 100644 index 000000000..493e8f7a8 --- /dev/null +++ b/bundle/artifacts/prepare.go @@ -0,0 +1,57 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +func PrepareAll() bundle.Mutator { + return &all{ + name: "Prepare", + fn: prepareArtifactByName, + } +} + +type prepare struct { + name string +} + +func prepareArtifactByName(name string) (bundle.Mutator, error) { + return &prepare{name}, nil +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + // Check if source paths are absolute, if not, make them absolute + for k := range artifact.Files { + f := &artifact.Files[k] + if !filepath.IsAbs(f.Source) { + dirPath := filepath.Dir(artifact.ConfigFilePath) + f.Source = filepath.Join(dirPath, f.Source) + } + } + + // If artifact path is not provided, use bundle root dir + if artifact.Path == "" { + artifact.Path = b.RootPath + } + + if !filepath.IsAbs(artifact.Path) { + dirPath := filepath.Dir(artifact.ConfigFilePath) + artifact.Path = filepath.Join(dirPath, artifact.Path) + } + + return bundle.Apply(ctx, b, getPrepareMutator(artifact.Type, m.name)) +} diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go index cf08843a7..a71610b03 100644 --- a/bundle/artifacts/upload_test.go +++ b/bundle/artifacts/upload_test.go @@ -63,7 +63,12 @@ func TestExpandGlobFilesSource(t *testing.T) { return &noop{} } - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) + pm := &prepare{"test"} + prepareMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { + return &noop{} + } + + diags := bundle.Apply(context.Background(), b, bundle.Seq(pm, bm, u)) require.NoError(t, diags.Error()) require.Equal(t, 2, len(b.Config.Artifacts["test"].Files)) diff --git a/bundle/artifacts/whl/build.go b/bundle/artifacts/whl/build.go index 992ade297..18d4b8ede 100644 --- a/bundle/artifacts/whl/build.go +++ b/bundle/artifacts/whl/build.go @@ -3,7 +3,6 @@ package whl import ( "context" "fmt" - "os" "path/filepath" "github.com/databricks/cli/bundle" @@ -36,18 +35,14 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Building %s...", m.name)) - dir := artifact.Path - - distPath := filepath.Join(dir, "dist") - os.RemoveAll(distPath) - python.CleanupWheelFolder(dir) - out, err := artifact.Build(ctx) if err != nil { return diag.Errorf("build failed %s, error: %v, output: %s", m.name, err, out) } log.Infof(ctx, "Build succeeded") + dir := artifact.Path + distPath := filepath.Join(artifact.Path, "dist") wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") if len(wheels) == 0 { return diag.Errorf("cannot find built wheel in %s for package %s", dir, m.name) diff --git a/bundle/artifacts/whl/prepare.go b/bundle/artifacts/whl/prepare.go new file mode 100644 index 000000000..7284b11ec --- /dev/null +++ b/bundle/artifacts/whl/prepare.go @@ -0,0 +1,48 @@ +package whl + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/python" +) + +type prepare struct { + name string +} + +func Prepare(name string) bundle.Mutator { + return &prepare{ + name: name, + } +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.whl.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + dir := artifact.Path + + distPath := filepath.Join(dir, "dist") + + // If we have multiple artifacts con figured, prepare will be called multiple times + // The first time we will remove the folders, other times will be no-op. + err := os.RemoveAll(distPath) + if err != nil { + log.Infof(ctx, "Failed to remove dist folder: %v", err) + } + python.CleanupWheelFolder(dir) + + return nil +} diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 362d23be1..3ddc6b181 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -16,6 +16,7 @@ func Build() bundle.Mutator { scripts.Execute(config.ScriptPreBuild), artifacts.DetectPackages(), artifacts.InferMissingProperties(), + artifacts.PrepareAll(), artifacts.BuildAll(), scripts.Execute(config.ScriptPostBuild), mutator.ResolveVariableReferences( diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml new file mode 100644 index 000000000..6964c58a4 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml @@ -0,0 +1,25 @@ +bundle: + name: python-wheel + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + build: "python3 setup.py bdist_wheel" + my_test_code_2: + type: whl + path: "./my_test_code" + build: "python3 setup2.py bdist_wheel" + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py new file mode 100644 index 000000000..0bd871dd3 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py new file mode 100644 index 000000000..424bec9f1 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code_2", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py new file mode 100644 index 000000000..909f1f322 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py new file mode 100644 index 000000000..73d045afb --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 8d0036a7b..52b3d6e07 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -96,3 +96,20 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } + +func TestPythonWheelBuildMultiple(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") + require.NoError(t, err) + require.Equal(t, 2, len(matches)) + + match := libraries.ValidateLocalLibrariesExist() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} From e6241e196fe2ae37bcf6709243eabdeeb94f261a Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:32:19 +0530 Subject: [PATCH 29/88] Move to a single prompt during bundle destroy (#1583) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes Right now we ask users for two confirmations when destroying a bundle. One to destroy the resources and one to delete the files. This PR consolidates the two prompts into one. ## Tests Manually Destroying a bundle with no resources: ``` ➜ bundle-playground git:(master) ✗ cli bundle destroy All files and directories at the following location will be deleted: /Users/shreyas.goenka@databricks.com/.bundle/bundle-playground/default Would you like to proceed? [y/n]: y No resources to destroy Updating deployment state... Deleting files... Destroy complete! ``` Destroying a bundle with no remote state: ``` ➜ bundle-playground git:(master) ✗ cli bundle destroy No active deployment found to destroy! ``` When a user cancells a deployment: ``` ➜ bundle-playground git:(master) ✗ cli bundle destroy The following resources will be deleted: delete job job_1 delete job job_2 delete pipeline foo All files and directories at the following location will be deleted: /Users/shreyas.goenka@databricks.com/.bundle/bundle-playground/default Would you like to proceed? [y/n]: n Destroy cancelled! ``` When a user destroys resources: ``` ➜ bundle-playground git:(master) ✗ cli bundle destroy The following resources will be deleted: delete job job_1 delete job job_2 delete pipeline foo All files and directories at the following location will be deleted: /Users/shreyas.goenka@databricks.com/.bundle/bundle-playground/default Would you like to proceed? [y/n]: y Updating deployment state... Deleting files... Destroy complete! ``` --- bundle/deploy/files/delete.go | 22 +------- bundle/deploy/terraform/destroy.go | 84 ++---------------------------- bundle/deploy/terraform/plan.go | 11 ++-- bundle/phases/destroy.go | 69 ++++++++++++++++++++++-- libs/terraform/plan.go | 39 ++++++++++++-- 5 files changed, 108 insertions(+), 117 deletions(-) diff --git a/bundle/deploy/files/delete.go b/bundle/deploy/files/delete.go index 133971449..bb28c2722 100644 --- a/bundle/deploy/files/delete.go +++ b/bundle/deploy/files/delete.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/sync" "github.com/databricks/databricks-sdk-go/service/workspace" - "github.com/fatih/color" ) type delete struct{} @@ -22,24 +21,7 @@ func (m *delete) Name() string { } func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // Do not delete files if terraform destroy was not consented - if !b.Plan.IsEmpty && !b.Plan.ConfirmApply { - return nil - } - - cmdio.LogString(ctx, "Starting deletion of remote bundle files") - cmdio.LogString(ctx, fmt.Sprintf("Bundle remote directory is %s", b.Config.Workspace.RootPath)) - - red := color.New(color.FgRed).SprintFunc() - if !b.AutoApprove { - proceed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\n%s and all files in it will be %s Proceed?", b.Config.Workspace.RootPath, red("deleted permanently!"))) - if err != nil { - return diag.FromErr(err) - } - if !proceed { - return nil - } - } + cmdio.LogString(ctx, "Deleting files...") err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{ Path: b.Config.Workspace.RootPath, @@ -54,8 +36,6 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if err != nil { return diag.FromErr(err) } - - cmdio.LogString(ctx, "Successfully deleted files!") return nil } diff --git a/bundle/deploy/terraform/destroy.go b/bundle/deploy/terraform/destroy.go index 16f074a22..9c63a0b37 100644 --- a/bundle/deploy/terraform/destroy.go +++ b/bundle/deploy/terraform/destroy.go @@ -2,61 +2,13 @@ package terraform import ( "context" - "fmt" - "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/fatih/color" + "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" - tfjson "github.com/hashicorp/terraform-json" ) -type PlanResourceChange struct { - ResourceType string `json:"resource_type"` - Action string `json:"action"` - ResourceName string `json:"resource_name"` -} - -func (c *PlanResourceChange) String() string { - result := strings.Builder{} - switch c.Action { - case "delete": - result.WriteString(" delete ") - default: - result.WriteString(c.Action + " ") - } - switch c.ResourceType { - case "databricks_job": - result.WriteString("job ") - case "databricks_pipeline": - result.WriteString("pipeline ") - default: - result.WriteString(c.ResourceType + " ") - } - result.WriteString(c.ResourceName) - return result.String() -} - -func (c *PlanResourceChange) IsInplaceSupported() bool { - return false -} - -func logDestroyPlan(ctx context.Context, changes []*tfjson.ResourceChange) error { - cmdio.LogString(ctx, "The following resources will be removed:") - for _, c := range changes { - if c.Change.Actions.Delete() { - cmdio.Log(ctx, &PlanResourceChange{ - ResourceType: c.Type, - Action: "delete", - ResourceName: c.Name, - }) - } - } - return nil -} - type destroy struct{} func (w *destroy) Name() string { @@ -66,7 +18,7 @@ func (w *destroy) Name() string { func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // return early if plan is empty if b.Plan.IsEmpty { - cmdio.LogString(ctx, "No resources to destroy in plan. Skipping destroy!") + log.Debugf(ctx, "No resources to destroy in plan. Skipping destroy.") return nil } @@ -75,45 +27,15 @@ func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diag.Errorf("terraform not initialized") } - // read plan file - plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) - if err != nil { - return diag.FromErr(err) - } - - // print the resources that will be destroyed - err = logDestroyPlan(ctx, plan.ResourceChanges) - if err != nil { - return diag.FromErr(err) - } - - // Ask for confirmation, if needed - if !b.Plan.ConfirmApply { - red := color.New(color.FgRed).SprintFunc() - b.Plan.ConfirmApply, err = cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nThis will permanently %s resources! Proceed?", red("destroy"))) - if err != nil { - return diag.FromErr(err) - } - } - - // return if confirmation was not provided - if !b.Plan.ConfirmApply { - return nil - } - if b.Plan.Path == "" { return diag.Errorf("no plan found") } - cmdio.LogString(ctx, "Starting to destroy resources") - // Apply terraform according to the computed destroy plan - err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) + err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { return diag.Errorf("terraform destroy: %v", err) } - - cmdio.LogString(ctx, "Successfully destroyed resources!") return nil } diff --git a/bundle/deploy/terraform/plan.go b/bundle/deploy/terraform/plan.go index 50e0f78ca..72f0b49a8 100644 --- a/bundle/deploy/terraform/plan.go +++ b/bundle/deploy/terraform/plan.go @@ -6,8 +6,8 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/terraform" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -33,8 +33,6 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Starting plan computation") - err := tf.Init(ctx, tfexec.Upgrade(true)) if err != nil { return diag.Errorf("terraform init: %v", err) @@ -55,12 +53,11 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Set plan in main bundle struct for downstream mutators b.Plan = &terraform.Plan{ - Path: planPath, - ConfirmApply: b.AutoApprove, - IsEmpty: !notEmpty, + Path: planPath, + IsEmpty: !notEmpty, } - cmdio.LogString(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) + log.Debugf(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) return nil } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index f1beace84..bd99af789 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -3,13 +3,18 @@ package phases import ( "context" "errors" + "fmt" "net/http" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + terraformlib "github.com/databricks/cli/libs/terraform" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -26,8 +31,63 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { return true, err } +func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + deleteActions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + if rc.Change.Actions.Delete() { + deleteActions = append(deleteActions, terraformlib.Action{ + Action: terraformlib.ActionTypeDelete, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + } + + if len(deleteActions) > 0 { + cmdio.LogString(ctx, "The following resources will be deleted:") + for _, a := range deleteActions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + + } + + cmdio.LogString(ctx, fmt.Sprintf("All files and directories at the following location will be deleted: %s", b.Config.Workspace.RootPath)) + cmdio.LogString(ctx, "") + + if b.AutoApprove { + return true, nil + } + + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The destroy phase deletes artifacts and resources. func Destroy() bundle.Mutator { + // Core destructive mutators for destroy. These require informed user consent. + destroyCore := bundle.Seq( + terraform.Destroy(), + terraform.StatePush(), + files.Delete(), + bundle.LogString("Destroy complete!"), + ) + destroyMutator := bundle.Seq( lock.Acquire(), bundle.Defer( @@ -36,13 +96,14 @@ func Destroy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("destroy")), - terraform.Destroy(), - terraform.StatePush(), - files.Delete(), + bundle.If( + approvalForDestroy, + destroyCore, + bundle.LogString("Destroy cancelled!"), + ), ), lock.Release(lock.GoalDestroy), ), - bundle.LogString("Destroy complete!"), ) return newPhase( diff --git a/libs/terraform/plan.go b/libs/terraform/plan.go index 22fea6206..36383cc24 100644 --- a/libs/terraform/plan.go +++ b/libs/terraform/plan.go @@ -1,13 +1,44 @@ package terraform +import "strings" + type Plan struct { // Path to the plan Path string - // Holds whether the user can consented to destruction. Either by interactive - // confirmation or by passing a command line flag - ConfirmApply bool - // If true, the plan is empty and applying it will not do anything IsEmpty bool } + +type Action struct { + // Type and name of the resource + ResourceType string `json:"resource_type"` + ResourceName string `json:"resource_name"` + + Action ActionType `json:"action"` +} + +func (a Action) String() string { + // terraform resources have the databricks_ prefix, which is not needed. + rtype := strings.TrimPrefix(a.ResourceType, "databricks_") + return strings.Join([]string{" ", string(a.Action), rtype, a.ResourceName}, " ") +} + +func (c Action) IsInplaceSupported() bool { + return false +} + +// These enum values correspond to action types defined in the tfjson library. +// "recreate" maps to the tfjson.Actions.Replace() function. +// "update" maps to tfjson.Actions.Update() and so on. source: +// https://github.com/hashicorp/terraform-json/blob/0104004301ca8e7046d089cdc2e2db2179d225be/action.go#L14 +type ActionType string + +const ( + ActionTypeCreate ActionType = "create" + ActionTypeDelete ActionType = "delete" + ActionTypeUpdate ActionType = "update" + ActionTypeNoOp ActionType = "no-op" + ActionTypeRead ActionType = "read" + ActionTypeRecreate ActionType = "recreate" +) From 9dbb58e821cb0b98c343755f0647007cba9c2570 Mon Sep 17 00:00:00 2001 From: Cor Date: Thu, 25 Jul 2024 10:51:37 +0200 Subject: [PATCH 30/88] Update Python dependencies before install when upgrading a labs project (#1624) The install script might require the up-to-date Python dependencies, explained in more detail in the referenced issue below Fixes #1623 ## Tests ! Need support with testing ! --- cmd/labs/project/installer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 92dfe9e7c..39ed9a966 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -132,14 +132,14 @@ func (i *installer) Upgrade(ctx context.Context) error { if err != nil { return fmt.Errorf("record version: %w", err) } - err = i.runInstallHook(ctx) - if err != nil { - return fmt.Errorf("installer: %w", err) - } err = i.installPythonDependencies(ctx, ".") if err != nil { return fmt.Errorf("python dependencies: %w", err) } + err = i.runInstallHook(ctx) + if err != nil { + return fmt.Errorf("installer: %w", err) + } return nil } From 90aaf2d20f625b178d47e67366bd77134e19f100 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 25 Jul 2024 16:18:49 +0200 Subject: [PATCH 31/88] Upgrade TF provider to 1.49.1 (#1626) ## Changes Upgrade TF provider to 1.49.1 --- bundle/internal/tf/codegen/schema/version.go | 2 +- bundle/internal/tf/schema/config.go | 61 +++-- .../internal/tf/schema/data_source_cluster.go | 232 +----------------- .../schema/data_source_external_location.go | 1 - bundle/internal/tf/schema/data_source_job.go | 52 ++-- .../schema/data_source_storage_credential.go | 1 - bundle/internal/tf/schema/data_sources.go | 198 ++++++++------- .../tf/schema/resource_external_location.go | 1 - bundle/internal/tf/schema/resource_job.go | 56 ++--- .../schema/resource_metastore_data_access.go | 1 - .../tf/schema/resource_mws_workspaces.go | 1 - .../tf/schema/resource_online_table.go | 9 +- .../tf/schema/resource_permissions.go | 1 - .../tf/schema/resource_storage_credential.go | 2 - .../tf/schema/resource_system_schema.go | 1 - bundle/internal/tf/schema/resources.go | 4 - bundle/internal/tf/schema/root.go | 2 +- 17 files changed, 174 insertions(+), 451 deletions(-) diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index aecb2736d..39d4f66c1 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.49.0" +const ProviderVersion = "1.49.1" diff --git a/bundle/internal/tf/schema/config.go b/bundle/internal/tf/schema/config.go index e807cdc53..d24d57339 100644 --- a/bundle/internal/tf/schema/config.go +++ b/bundle/internal/tf/schema/config.go @@ -3,36 +3,33 @@ package schema type Config struct { - AccountId string `json:"account_id,omitempty"` - ActionsIdTokenRequestToken string `json:"actions_id_token_request_token,omitempty"` - ActionsIdTokenRequestUrl string `json:"actions_id_token_request_url,omitempty"` - AuthType string `json:"auth_type,omitempty"` - AzureClientId string `json:"azure_client_id,omitempty"` - AzureClientSecret string `json:"azure_client_secret,omitempty"` - AzureEnvironment string `json:"azure_environment,omitempty"` - AzureLoginAppId string `json:"azure_login_app_id,omitempty"` - AzureTenantId string `json:"azure_tenant_id,omitempty"` - AzureUseMsi bool `json:"azure_use_msi,omitempty"` - AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` - ClientId string `json:"client_id,omitempty"` - ClientSecret string `json:"client_secret,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ConfigFile string `json:"config_file,omitempty"` - DatabricksCliPath string `json:"databricks_cli_path,omitempty"` - DebugHeaders bool `json:"debug_headers,omitempty"` - DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` - GoogleCredentials string `json:"google_credentials,omitempty"` - GoogleServiceAccount string `json:"google_service_account,omitempty"` - Host string `json:"host,omitempty"` - HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` - MetadataServiceUrl string `json:"metadata_service_url,omitempty"` - Password string `json:"password,omitempty"` - Profile string `json:"profile,omitempty"` - RateLimit int `json:"rate_limit,omitempty"` - RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` - ServerlessComputeId string `json:"serverless_compute_id,omitempty"` - SkipVerify bool `json:"skip_verify,omitempty"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + AccountId string `json:"account_id,omitempty"` + AuthType string `json:"auth_type,omitempty"` + AzureClientId string `json:"azure_client_id,omitempty"` + AzureClientSecret string `json:"azure_client_secret,omitempty"` + AzureEnvironment string `json:"azure_environment,omitempty"` + AzureLoginAppId string `json:"azure_login_app_id,omitempty"` + AzureTenantId string `json:"azure_tenant_id,omitempty"` + AzureUseMsi bool `json:"azure_use_msi,omitempty"` + AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` + ClientId string `json:"client_id,omitempty"` + ClientSecret string `json:"client_secret,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ConfigFile string `json:"config_file,omitempty"` + DatabricksCliPath string `json:"databricks_cli_path,omitempty"` + DebugHeaders bool `json:"debug_headers,omitempty"` + DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` + GoogleCredentials string `json:"google_credentials,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + Host string `json:"host,omitempty"` + HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` + MetadataServiceUrl string `json:"metadata_service_url,omitempty"` + Password string `json:"password,omitempty"` + Profile string `json:"profile,omitempty"` + RateLimit int `json:"rate_limit,omitempty"` + RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + WarehouseId string `json:"warehouse_id,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index 94d67bbfa..fff66dc93 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -10,9 +10,7 @@ type DataSourceClusterClusterInfoAutoscale struct { type DataSourceClusterClusterInfoAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` - EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` - EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -20,16 +18,10 @@ type DataSourceClusterClusterInfoAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } -type DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo struct { - LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` - LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` -} - type DataSourceClusterClusterInfoAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` - LogAnalyticsInfo *DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` } type DataSourceClusterClusterInfoClusterLogConfDbfs struct { @@ -57,12 +49,12 @@ type DataSourceClusterClusterInfoClusterLogStatus struct { } type DataSourceClusterClusterInfoDockerImageBasicAuth struct { - Password string `json:"password,omitempty"` - Username string `json:"username,omitempty"` + Password string `json:"password"` + Username string `json:"username"` } type DataSourceClusterClusterInfoDockerImage struct { - Url string `json:"url,omitempty"` + Url string `json:"url"` BasicAuth *DataSourceClusterClusterInfoDockerImageBasicAuth `json:"basic_auth,omitempty"` } @@ -147,212 +139,12 @@ type DataSourceClusterClusterInfoInitScripts struct { Workspace *DataSourceClusterClusterInfoInitScriptsWorkspace `json:"workspace,omitempty"` } -type DataSourceClusterClusterInfoSpecAutoscale struct { - MaxWorkers int `json:"max_workers,omitempty"` - MinWorkers int `json:"min_workers,omitempty"` -} - -type DataSourceClusterClusterInfoSpecAwsAttributes struct { - Availability string `json:"availability,omitempty"` - EbsVolumeCount int `json:"ebs_volume_count,omitempty"` - EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` - EbsVolumeSize int `json:"ebs_volume_size,omitempty"` - EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` - EbsVolumeType string `json:"ebs_volume_type,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - InstanceProfileArn string `json:"instance_profile_arn,omitempty"` - SpotBidPricePercent int `json:"spot_bid_price_percent,omitempty"` - ZoneId string `json:"zone_id,omitempty"` -} - -type DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo struct { - LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` - LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` -} - -type DataSourceClusterClusterInfoSpecAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` - LogAnalyticsInfo *DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` -} - -type DataSourceClusterClusterInfoSpecClusterLogConfDbfs struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { - CannedAcl string `json:"canned_acl,omitempty"` - Destination string `json:"destination"` - EnableEncryption bool `json:"enable_encryption,omitempty"` - EncryptionType string `json:"encryption_type,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - KmsKey string `json:"kms_key,omitempty"` - Region string `json:"region,omitempty"` -} - -type DataSourceClusterClusterInfoSpecClusterLogConf struct { - Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` - S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` -} - -type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { - MountOptions string `json:"mount_options,omitempty"` - ServerAddress string `json:"server_address"` -} - -type DataSourceClusterClusterInfoSpecClusterMountInfo struct { - LocalMountDirPath string `json:"local_mount_dir_path"` - RemoteMountDirPath string `json:"remote_mount_dir_path,omitempty"` - NetworkFilesystemInfo *DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo `json:"network_filesystem_info,omitempty"` -} - -type DataSourceClusterClusterInfoSpecDockerImageBasicAuth struct { - Password string `json:"password"` - Username string `json:"username"` -} - -type DataSourceClusterClusterInfoSpecDockerImage struct { - Url string `json:"url"` - BasicAuth *DataSourceClusterClusterInfoSpecDockerImageBasicAuth `json:"basic_auth,omitempty"` -} - -type DataSourceClusterClusterInfoSpecGcpAttributes struct { - Availability string `json:"availability,omitempty"` - BootDiskSize int `json:"boot_disk_size,omitempty"` - GoogleServiceAccount string `json:"google_service_account,omitempty"` - LocalSsdCount int `json:"local_ssd_count,omitempty"` - UsePreemptibleExecutors bool `json:"use_preemptible_executors,omitempty"` - ZoneId string `json:"zone_id,omitempty"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsAbfss struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsDbfs struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsFile struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsGcs struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsS3 struct { - CannedAcl string `json:"canned_acl,omitempty"` - Destination string `json:"destination"` - EnableEncryption bool `json:"enable_encryption,omitempty"` - EncryptionType string `json:"encryption_type,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - KmsKey string `json:"kms_key,omitempty"` - Region string `json:"region,omitempty"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsVolumes struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScriptsWorkspace struct { - Destination string `json:"destination"` -} - -type DataSourceClusterClusterInfoSpecInitScripts struct { - Abfss *DataSourceClusterClusterInfoSpecInitScriptsAbfss `json:"abfss,omitempty"` - Dbfs *DataSourceClusterClusterInfoSpecInitScriptsDbfs `json:"dbfs,omitempty"` - File *DataSourceClusterClusterInfoSpecInitScriptsFile `json:"file,omitempty"` - Gcs *DataSourceClusterClusterInfoSpecInitScriptsGcs `json:"gcs,omitempty"` - S3 *DataSourceClusterClusterInfoSpecInitScriptsS3 `json:"s3,omitempty"` - Volumes *DataSourceClusterClusterInfoSpecInitScriptsVolumes `json:"volumes,omitempty"` - Workspace *DataSourceClusterClusterInfoSpecInitScriptsWorkspace `json:"workspace,omitempty"` -} - -type DataSourceClusterClusterInfoSpecLibraryCran struct { - Package string `json:"package"` - Repo string `json:"repo,omitempty"` -} - -type DataSourceClusterClusterInfoSpecLibraryMaven struct { - Coordinates string `json:"coordinates"` - Exclusions []string `json:"exclusions,omitempty"` - Repo string `json:"repo,omitempty"` -} - -type DataSourceClusterClusterInfoSpecLibraryPypi struct { - Package string `json:"package"` - Repo string `json:"repo,omitempty"` -} - -type DataSourceClusterClusterInfoSpecLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Requirements string `json:"requirements,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *DataSourceClusterClusterInfoSpecLibraryCran `json:"cran,omitempty"` - Maven *DataSourceClusterClusterInfoSpecLibraryMaven `json:"maven,omitempty"` - Pypi *DataSourceClusterClusterInfoSpecLibraryPypi `json:"pypi,omitempty"` -} - -type DataSourceClusterClusterInfoSpecWorkloadTypeClients struct { - Jobs bool `json:"jobs,omitempty"` - Notebooks bool `json:"notebooks,omitempty"` -} - -type DataSourceClusterClusterInfoSpecWorkloadType struct { - Clients *DataSourceClusterClusterInfoSpecWorkloadTypeClients `json:"clients,omitempty"` -} - -type DataSourceClusterClusterInfoSpec struct { - ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ClusterName string `json:"cluster_name,omitempty"` - CustomTags map[string]string `json:"custom_tags,omitempty"` - DataSecurityMode string `json:"data_security_mode,omitempty"` - DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` - DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` - EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` - EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` - IdempotencyToken string `json:"idempotency_token,omitempty"` - InstancePoolId string `json:"instance_pool_id,omitempty"` - NodeTypeId string `json:"node_type_id,omitempty"` - NumWorkers int `json:"num_workers,omitempty"` - PolicyId string `json:"policy_id,omitempty"` - RuntimeEngine string `json:"runtime_engine,omitempty"` - SingleUserName string `json:"single_user_name,omitempty"` - SparkConf map[string]string `json:"spark_conf,omitempty"` - SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` - SshPublicKeys []string `json:"ssh_public_keys,omitempty"` - Autoscale *DataSourceClusterClusterInfoSpecAutoscale `json:"autoscale,omitempty"` - AwsAttributes *DataSourceClusterClusterInfoSpecAwsAttributes `json:"aws_attributes,omitempty"` - AzureAttributes *DataSourceClusterClusterInfoSpecAzureAttributes `json:"azure_attributes,omitempty"` - ClusterLogConf *DataSourceClusterClusterInfoSpecClusterLogConf `json:"cluster_log_conf,omitempty"` - ClusterMountInfo []DataSourceClusterClusterInfoSpecClusterMountInfo `json:"cluster_mount_info,omitempty"` - DockerImage *DataSourceClusterClusterInfoSpecDockerImage `json:"docker_image,omitempty"` - GcpAttributes *DataSourceClusterClusterInfoSpecGcpAttributes `json:"gcp_attributes,omitempty"` - InitScripts []DataSourceClusterClusterInfoSpecInitScripts `json:"init_scripts,omitempty"` - Library []DataSourceClusterClusterInfoSpecLibrary `json:"library,omitempty"` - WorkloadType *DataSourceClusterClusterInfoSpecWorkloadType `json:"workload_type,omitempty"` -} - type DataSourceClusterClusterInfoTerminationReason struct { Code string `json:"code,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` Type string `json:"type,omitempty"` } -type DataSourceClusterClusterInfoWorkloadTypeClients struct { - Jobs bool `json:"jobs,omitempty"` - Notebooks bool `json:"notebooks,omitempty"` -} - -type DataSourceClusterClusterInfoWorkloadType struct { - Clients *DataSourceClusterClusterInfoWorkloadTypeClients `json:"clients,omitempty"` -} - type DataSourceClusterClusterInfo struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterCores int `json:"cluster_cores,omitempty"` @@ -363,14 +155,14 @@ type DataSourceClusterClusterInfo struct { CreatorUserName string `json:"creator_user_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags,omitempty"` + DefaultTags map[string]string `json:"default_tags"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` JdbcPort int `json:"jdbc_port,omitempty"` - LastRestartedTime int `json:"last_restarted_time,omitempty"` + LastActivityTime int `json:"last_activity_time,omitempty"` LastStateLossTime int `json:"last_state_loss_time,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` @@ -380,12 +172,12 @@ type DataSourceClusterClusterInfo struct { SparkConf map[string]string `json:"spark_conf,omitempty"` SparkContextId int `json:"spark_context_id,omitempty"` SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version,omitempty"` + SparkVersion string `json:"spark_version"` SshPublicKeys []string `json:"ssh_public_keys,omitempty"` StartTime int `json:"start_time,omitempty"` - State string `json:"state,omitempty"` + State string `json:"state"` StateMessage string `json:"state_message,omitempty"` - TerminatedTime int `json:"terminated_time,omitempty"` + TerminateTime int `json:"terminate_time,omitempty"` Autoscale *DataSourceClusterClusterInfoAutoscale `json:"autoscale,omitempty"` AwsAttributes *DataSourceClusterClusterInfoAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *DataSourceClusterClusterInfoAzureAttributes `json:"azure_attributes,omitempty"` @@ -396,9 +188,7 @@ type DataSourceClusterClusterInfo struct { Executors []DataSourceClusterClusterInfoExecutors `json:"executors,omitempty"` GcpAttributes *DataSourceClusterClusterInfoGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []DataSourceClusterClusterInfoInitScripts `json:"init_scripts,omitempty"` - Spec *DataSourceClusterClusterInfoSpec `json:"spec,omitempty"` TerminationReason *DataSourceClusterClusterInfoTerminationReason `json:"termination_reason,omitempty"` - WorkloadType *DataSourceClusterClusterInfoWorkloadType `json:"workload_type,omitempty"` } type DataSourceCluster struct { diff --git a/bundle/internal/tf/schema/data_source_external_location.go b/bundle/internal/tf/schema/data_source_external_location.go index a3e78cbd3..0fea6e529 100644 --- a/bundle/internal/tf/schema/data_source_external_location.go +++ b/bundle/internal/tf/schema/data_source_external_location.go @@ -19,7 +19,6 @@ type DataSourceExternalLocationExternalLocationInfo struct { CreatedBy string `json:"created_by,omitempty"` CredentialId string `json:"credential_id,omitempty"` CredentialName string `json:"credential_name,omitempty"` - IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go index 91806d670..e5ec5afb7 100644 --- a/bundle/internal/tf/schema/data_source_job.go +++ b/bundle/internal/tf/schema/data_source_job.go @@ -26,7 +26,6 @@ type DataSourceJobJobSettingsSettingsEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -56,9 +55,9 @@ type DataSourceJobJobSettingsSettingsGitSource struct { } type DataSourceJobJobSettingsSettingsHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type DataSourceJobJobSettingsSettingsHealth struct { @@ -223,7 +222,7 @@ type DataSourceJobJobSettingsSettingsJobClusterNewCluster struct { } type DataSourceJobJobSettingsSettingsJobCluster struct { - JobClusterKey string `json:"job_cluster_key"` + JobClusterKey string `json:"job_cluster_key,omitempty"` NewCluster *DataSourceJobJobSettingsSettingsJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -501,7 +500,6 @@ type DataSourceJobJobSettingsSettingsTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -531,14 +529,13 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskEmailNotifications struc OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealth struct { @@ -808,7 +805,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id"` + WarehouseId string `json:"warehouse_id,omitempty"` Alert *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -827,10 +824,6 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSt Id string `json:"id"` } -type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -839,7 +832,6 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotifications str OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -852,7 +844,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key"` + TaskKey string `json:"task_key,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` @@ -880,9 +872,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTask struct { } type DataSourceJobJobSettingsSettingsTaskHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type DataSourceJobJobSettingsSettingsTaskHealth struct { @@ -1152,7 +1144,7 @@ type DataSourceJobJobSettingsSettingsTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id"` + WarehouseId string `json:"warehouse_id,omitempty"` Alert *DataSourceJobJobSettingsSettingsTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskSqlTaskFile `json:"file,omitempty"` @@ -1171,10 +1163,6 @@ type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } -type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1183,7 +1171,6 @@ type DataSourceJobJobSettingsSettingsTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -1196,7 +1183,7 @@ type DataSourceJobJobSettingsSettingsTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key"` + TaskKey string `json:"task_key,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskDbtTask `json:"dbt_task,omitempty"` @@ -1224,11 +1211,6 @@ type DataSourceJobJobSettingsSettingsTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } -type DataSourceJobJobSettingsSettingsTriggerPeriodic struct { - Interval int `json:"interval"` - Unit string `json:"unit"` -} - type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1239,7 +1221,6 @@ type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { type DataSourceJobJobSettingsSettingsTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *DataSourceJobJobSettingsSettingsTriggerFileArrival `json:"file_arrival,omitempty"` - Periodic *DataSourceJobJobSettingsSettingsTriggerPeriodic `json:"periodic,omitempty"` TableUpdate *DataSourceJobJobSettingsSettingsTriggerTableUpdate `json:"table_update,omitempty"` } @@ -1255,10 +1236,6 @@ type DataSourceJobJobSettingsSettingsWebhookNotificationsOnStart struct { Id string `json:"id"` } -type DataSourceJobJobSettingsSettingsWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type DataSourceJobJobSettingsSettingsWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1267,7 +1244,6 @@ type DataSourceJobJobSettingsSettingsWebhookNotifications struct { OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_storage_credential.go b/bundle/internal/tf/schema/data_source_storage_credential.go index bf58f2726..c7045d445 100644 --- a/bundle/internal/tf/schema/data_source_storage_credential.go +++ b/bundle/internal/tf/schema/data_source_storage_credential.go @@ -36,7 +36,6 @@ type DataSourceStorageCredentialStorageCredentialInfo struct { CreatedAt int `json:"created_at,omitempty"` CreatedBy string `json:"created_by,omitempty"` Id string `json:"id,omitempty"` - IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 4ac78613f..c32483db0 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -3,111 +3,105 @@ package schema type DataSources struct { - AwsAssumeRolePolicy map[string]any `json:"databricks_aws_assume_role_policy,omitempty"` - AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"` - AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"` - AwsUnityCatalogAssumeRolePolicy map[string]any `json:"databricks_aws_unity_catalog_assume_role_policy,omitempty"` - AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"` - Catalog map[string]any `json:"databricks_catalog,omitempty"` - Catalogs map[string]any `json:"databricks_catalogs,omitempty"` - Cluster map[string]any `json:"databricks_cluster,omitempty"` - ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` - Clusters map[string]any `json:"databricks_clusters,omitempty"` - CurrentConfig map[string]any `json:"databricks_current_config,omitempty"` - CurrentMetastore map[string]any `json:"databricks_current_metastore,omitempty"` - CurrentUser map[string]any `json:"databricks_current_user,omitempty"` - DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` - DbfsFilePaths map[string]any `json:"databricks_dbfs_file_paths,omitempty"` - Directory map[string]any `json:"databricks_directory,omitempty"` - ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` - ExternalLocations map[string]any `json:"databricks_external_locations,omitempty"` - Group map[string]any `json:"databricks_group,omitempty"` - InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` - InstanceProfiles map[string]any `json:"databricks_instance_profiles,omitempty"` - Job map[string]any `json:"databricks_job,omitempty"` - Jobs map[string]any `json:"databricks_jobs,omitempty"` - Metastore map[string]any `json:"databricks_metastore,omitempty"` - Metastores map[string]any `json:"databricks_metastores,omitempty"` - MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` - MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` - MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` - MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` - NodeType map[string]any `json:"databricks_node_type,omitempty"` - Notebook map[string]any `json:"databricks_notebook,omitempty"` - NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` - Pipelines map[string]any `json:"databricks_pipelines,omitempty"` - Schema map[string]any `json:"databricks_schema,omitempty"` - Schemas map[string]any `json:"databricks_schemas,omitempty"` - ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` - ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` - Share map[string]any `json:"databricks_share,omitempty"` - Shares map[string]any `json:"databricks_shares,omitempty"` - SparkVersion map[string]any `json:"databricks_spark_version,omitempty"` - SqlWarehouse map[string]any `json:"databricks_sql_warehouse,omitempty"` - SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"` - StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` - StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"` - Table map[string]any `json:"databricks_table,omitempty"` - Tables map[string]any `json:"databricks_tables,omitempty"` - User map[string]any `json:"databricks_user,omitempty"` - Views map[string]any `json:"databricks_views,omitempty"` - Volume map[string]any `json:"databricks_volume,omitempty"` - Volumes map[string]any `json:"databricks_volumes,omitempty"` - Zones map[string]any `json:"databricks_zones,omitempty"` + AwsAssumeRolePolicy map[string]any `json:"databricks_aws_assume_role_policy,omitempty"` + AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"` + AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"` + AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"` + Catalog map[string]any `json:"databricks_catalog,omitempty"` + Catalogs map[string]any `json:"databricks_catalogs,omitempty"` + Cluster map[string]any `json:"databricks_cluster,omitempty"` + ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` + Clusters map[string]any `json:"databricks_clusters,omitempty"` + CurrentConfig map[string]any `json:"databricks_current_config,omitempty"` + CurrentMetastore map[string]any `json:"databricks_current_metastore,omitempty"` + CurrentUser map[string]any `json:"databricks_current_user,omitempty"` + DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` + DbfsFilePaths map[string]any `json:"databricks_dbfs_file_paths,omitempty"` + Directory map[string]any `json:"databricks_directory,omitempty"` + ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` + ExternalLocations map[string]any `json:"databricks_external_locations,omitempty"` + Group map[string]any `json:"databricks_group,omitempty"` + InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` + InstanceProfiles map[string]any `json:"databricks_instance_profiles,omitempty"` + Job map[string]any `json:"databricks_job,omitempty"` + Jobs map[string]any `json:"databricks_jobs,omitempty"` + Metastore map[string]any `json:"databricks_metastore,omitempty"` + Metastores map[string]any `json:"databricks_metastores,omitempty"` + MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` + MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` + MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` + NodeType map[string]any `json:"databricks_node_type,omitempty"` + Notebook map[string]any `json:"databricks_notebook,omitempty"` + NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` + Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + Schemas map[string]any `json:"databricks_schemas,omitempty"` + ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` + ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` + Share map[string]any `json:"databricks_share,omitempty"` + Shares map[string]any `json:"databricks_shares,omitempty"` + SparkVersion map[string]any `json:"databricks_spark_version,omitempty"` + SqlWarehouse map[string]any `json:"databricks_sql_warehouse,omitempty"` + SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"` + StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` + StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"` + Table map[string]any `json:"databricks_table,omitempty"` + Tables map[string]any `json:"databricks_tables,omitempty"` + User map[string]any `json:"databricks_user,omitempty"` + Views map[string]any `json:"databricks_views,omitempty"` + Volumes map[string]any `json:"databricks_volumes,omitempty"` + Zones map[string]any `json:"databricks_zones,omitempty"` } func NewDataSources() *DataSources { return &DataSources{ - AwsAssumeRolePolicy: make(map[string]any), - AwsBucketPolicy: make(map[string]any), - AwsCrossaccountPolicy: make(map[string]any), - AwsUnityCatalogAssumeRolePolicy: make(map[string]any), - AwsUnityCatalogPolicy: make(map[string]any), - Catalog: make(map[string]any), - Catalogs: make(map[string]any), - Cluster: make(map[string]any), - ClusterPolicy: make(map[string]any), - Clusters: make(map[string]any), - CurrentConfig: make(map[string]any), - CurrentMetastore: make(map[string]any), - CurrentUser: make(map[string]any), - DbfsFile: make(map[string]any), - DbfsFilePaths: make(map[string]any), - Directory: make(map[string]any), - ExternalLocation: make(map[string]any), - ExternalLocations: make(map[string]any), - Group: make(map[string]any), - InstancePool: make(map[string]any), - InstanceProfiles: make(map[string]any), - Job: make(map[string]any), - Jobs: make(map[string]any), - Metastore: make(map[string]any), - Metastores: make(map[string]any), - MlflowExperiment: make(map[string]any), - MlflowModel: make(map[string]any), - MwsCredentials: make(map[string]any), - MwsWorkspaces: make(map[string]any), - NodeType: make(map[string]any), - Notebook: make(map[string]any), - NotebookPaths: make(map[string]any), - Pipelines: make(map[string]any), - Schema: make(map[string]any), - Schemas: make(map[string]any), - ServicePrincipal: make(map[string]any), - ServicePrincipals: make(map[string]any), - Share: make(map[string]any), - Shares: make(map[string]any), - SparkVersion: make(map[string]any), - SqlWarehouse: make(map[string]any), - SqlWarehouses: make(map[string]any), - StorageCredential: make(map[string]any), - StorageCredentials: make(map[string]any), - Table: make(map[string]any), - Tables: make(map[string]any), - User: make(map[string]any), - Views: make(map[string]any), - Volume: make(map[string]any), - Volumes: make(map[string]any), - Zones: make(map[string]any), + AwsAssumeRolePolicy: make(map[string]any), + AwsBucketPolicy: make(map[string]any), + AwsCrossaccountPolicy: make(map[string]any), + AwsUnityCatalogPolicy: make(map[string]any), + Catalog: make(map[string]any), + Catalogs: make(map[string]any), + Cluster: make(map[string]any), + ClusterPolicy: make(map[string]any), + Clusters: make(map[string]any), + CurrentConfig: make(map[string]any), + CurrentMetastore: make(map[string]any), + CurrentUser: make(map[string]any), + DbfsFile: make(map[string]any), + DbfsFilePaths: make(map[string]any), + Directory: make(map[string]any), + ExternalLocation: make(map[string]any), + ExternalLocations: make(map[string]any), + Group: make(map[string]any), + InstancePool: make(map[string]any), + InstanceProfiles: make(map[string]any), + Job: make(map[string]any), + Jobs: make(map[string]any), + Metastore: make(map[string]any), + Metastores: make(map[string]any), + MlflowExperiment: make(map[string]any), + MlflowModel: make(map[string]any), + MwsCredentials: make(map[string]any), + MwsWorkspaces: make(map[string]any), + NodeType: make(map[string]any), + Notebook: make(map[string]any), + NotebookPaths: make(map[string]any), + Pipelines: make(map[string]any), + Schemas: make(map[string]any), + ServicePrincipal: make(map[string]any), + ServicePrincipals: make(map[string]any), + Share: make(map[string]any), + Shares: make(map[string]any), + SparkVersion: make(map[string]any), + SqlWarehouse: make(map[string]any), + SqlWarehouses: make(map[string]any), + StorageCredential: make(map[string]any), + StorageCredentials: make(map[string]any), + Table: make(map[string]any), + Tables: make(map[string]any), + User: make(map[string]any), + Views: make(map[string]any), + Volumes: make(map[string]any), + Zones: make(map[string]any), } } diff --git a/bundle/internal/tf/schema/resource_external_location.go b/bundle/internal/tf/schema/resource_external_location.go index da28271bc..af64c677c 100644 --- a/bundle/internal/tf/schema/resource_external_location.go +++ b/bundle/internal/tf/schema/resource_external_location.go @@ -18,7 +18,6 @@ type ResourceExternalLocation struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` - IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 42b648b0f..6e624ad8a 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -26,7 +26,6 @@ type ResourceJobEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -61,9 +60,9 @@ type ResourceJobGitSource struct { } type ResourceJobHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type ResourceJobHealth struct { @@ -230,6 +229,7 @@ type ResourceJobJobClusterNewClusterWorkloadType struct { type ResourceJobJobClusterNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -262,7 +262,7 @@ type ResourceJobJobClusterNewCluster struct { } type ResourceJobJobCluster struct { - JobClusterKey string `json:"job_cluster_key"` + JobClusterKey string `json:"job_cluster_key,omitempty"` NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -452,6 +452,7 @@ type ResourceJobNewClusterWorkloadType struct { type ResourceJobNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -574,7 +575,6 @@ type ResourceJobTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -604,14 +604,13 @@ type ResourceJobTaskForEachTaskTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } type ResourceJobTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type ResourceJobTaskForEachTaskTaskHealth struct { @@ -804,6 +803,7 @@ type ResourceJobTaskForEachTaskTaskNewClusterWorkloadType struct { type ResourceJobTaskForEachTaskTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -927,7 +927,7 @@ type ResourceJobTaskForEachTaskTaskSqlTaskQuery struct { type ResourceJobTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id"` + WarehouseId string `json:"warehouse_id,omitempty"` Alert *ResourceJobTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -946,10 +946,6 @@ type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } -type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -958,7 +954,6 @@ type ResourceJobTaskForEachTaskTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -972,7 +967,7 @@ type ResourceJobTaskForEachTaskTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key"` + TaskKey string `json:"task_key,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` @@ -1000,9 +995,9 @@ type ResourceJobTaskForEachTask struct { } type ResourceJobTaskHealthRules struct { - Metric string `json:"metric"` - Op string `json:"op"` - Value int `json:"value"` + Metric string `json:"metric,omitempty"` + Op string `json:"op,omitempty"` + Value int `json:"value,omitempty"` } type ResourceJobTaskHealth struct { @@ -1195,6 +1190,7 @@ type ResourceJobTaskNewClusterWorkloadType struct { type ResourceJobTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -1318,7 +1314,7 @@ type ResourceJobTaskSqlTaskQuery struct { type ResourceJobTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id"` + WarehouseId string `json:"warehouse_id,omitempty"` Alert *ResourceJobTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskSqlTaskFile `json:"file,omitempty"` @@ -1337,10 +1333,6 @@ type ResourceJobTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } -type ResourceJobTaskWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type ResourceJobTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1349,7 +1341,6 @@ type ResourceJobTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []ResourceJobTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -1363,7 +1354,7 @@ type ResourceJobTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key"` + TaskKey string `json:"task_key,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"` DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"` @@ -1391,11 +1382,6 @@ type ResourceJobTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } -type ResourceJobTriggerPeriodic struct { - Interval int `json:"interval"` - Unit string `json:"unit"` -} - type ResourceJobTriggerTable struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1413,7 +1399,6 @@ type ResourceJobTriggerTableUpdate struct { type ResourceJobTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"` - Periodic *ResourceJobTriggerPeriodic `json:"periodic,omitempty"` Table *ResourceJobTriggerTable `json:"table,omitempty"` TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"` } @@ -1430,10 +1415,6 @@ type ResourceJobWebhookNotificationsOnStart struct { Id string `json:"id"` } -type ResourceJobWebhookNotificationsOnStreamingBacklogExceeded struct { - Id string `json:"id"` -} - type ResourceJobWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1442,7 +1423,6 @@ type ResourceJobWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobWebhookNotificationsOnStart `json:"on_start,omitempty"` - OnStreamingBacklogExceeded []ResourceJobWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_metastore_data_access.go b/bundle/internal/tf/schema/resource_metastore_data_access.go index 2e2ff4eb4..155730055 100644 --- a/bundle/internal/tf/schema/resource_metastore_data_access.go +++ b/bundle/internal/tf/schema/resource_metastore_data_access.go @@ -37,7 +37,6 @@ type ResourceMetastoreDataAccess struct { ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` IsDefault bool `json:"is_default,omitempty"` - IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_mws_workspaces.go b/bundle/internal/tf/schema/resource_mws_workspaces.go index 6c053cb84..21d1ce428 100644 --- a/bundle/internal/tf/schema/resource_mws_workspaces.go +++ b/bundle/internal/tf/schema/resource_mws_workspaces.go @@ -43,7 +43,6 @@ type ResourceMwsWorkspaces struct { CustomTags map[string]string `json:"custom_tags,omitempty"` CustomerManagedKeyId string `json:"customer_managed_key_id,omitempty"` DeploymentName string `json:"deployment_name,omitempty"` - GcpWorkspaceSa string `json:"gcp_workspace_sa,omitempty"` Id string `json:"id,omitempty"` IsNoPublicIpEnabled bool `json:"is_no_public_ip_enabled,omitempty"` Location string `json:"location,omitempty"` diff --git a/bundle/internal/tf/schema/resource_online_table.go b/bundle/internal/tf/schema/resource_online_table.go index de671eade..af8a348d3 100644 --- a/bundle/internal/tf/schema/resource_online_table.go +++ b/bundle/internal/tf/schema/resource_online_table.go @@ -19,9 +19,8 @@ type ResourceOnlineTableSpec struct { } type ResourceOnlineTable struct { - Id string `json:"id,omitempty"` - Name string `json:"name"` - Status []any `json:"status,omitempty"` - TableServingUrl string `json:"table_serving_url,omitempty"` - Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + Status []any `json:"status,omitempty"` + Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index ee94a1a8f..5d8df11e7 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -13,7 +13,6 @@ type ResourcePermissions struct { Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` - DashboardId string `json:"dashboard_id,omitempty"` DirectoryId string `json:"directory_id,omitempty"` DirectoryPath string `json:"directory_path,omitempty"` ExperimentId string `json:"experiment_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_storage_credential.go b/bundle/internal/tf/schema/resource_storage_credential.go index 1c62cf8df..3d4a501ea 100644 --- a/bundle/internal/tf/schema/resource_storage_credential.go +++ b/bundle/internal/tf/schema/resource_storage_credential.go @@ -36,13 +36,11 @@ type ResourceStorageCredential struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` - IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` ReadOnly bool `json:"read_only,omitempty"` SkipValidation bool `json:"skip_validation,omitempty"` - StorageCredentialId string `json:"storage_credential_id,omitempty"` AwsIamRole *ResourceStorageCredentialAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceStorageCredentialAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceStorageCredentialAzureServicePrincipal `json:"azure_service_principal,omitempty"` diff --git a/bundle/internal/tf/schema/resource_system_schema.go b/bundle/internal/tf/schema/resource_system_schema.go index fe5b128d6..09a86103a 100644 --- a/bundle/internal/tf/schema/resource_system_schema.go +++ b/bundle/internal/tf/schema/resource_system_schema.go @@ -3,7 +3,6 @@ package schema type ResourceSystemSchema struct { - FullName string `json:"full_name,omitempty"` Id string `json:"id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Schema string `json:"schema,omitempty"` diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79c1b32b5..79d71a65f 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -16,7 +16,6 @@ type Resources struct { ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` Connection map[string]any `json:"databricks_connection,omitempty"` - Dashboard map[string]any `json:"databricks_dashboard,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -97,7 +96,6 @@ type Resources struct { VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` Volume map[string]any `json:"databricks_volume,omitempty"` - WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } @@ -117,7 +115,6 @@ func NewResources() *Resources { ClusterPolicy: make(map[string]any), ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), Connection: make(map[string]any), - Dashboard: make(map[string]any), DbfsFile: make(map[string]any), DefaultNamespaceSetting: make(map[string]any), Directory: make(map[string]any), @@ -198,7 +195,6 @@ func NewResources() *Resources { VectorSearchEndpoint: make(map[string]any), VectorSearchIndex: make(map[string]any), Volume: make(map[string]any), - WorkspaceBinding: make(map[string]any), WorkspaceConf: make(map[string]any), WorkspaceFile: make(map[string]any), } diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 8401d8dac..171128350 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.49.0" +const ProviderVersion = "1.49.1" func NewRoot() *Root { return &Root{ From 37b9df96e6606d0655f1e574d4eebb72eedd9cde Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:46:27 +0530 Subject: [PATCH 32/88] Support multiple paths for diagnostics (#1616) ## Changes Some diagnostics can have multiple paths associated with them. For instance, ensuring that unique resource keys are used across all resources. This PR extends `diag.Diagnostic` to accept multiple paths. This PR is symmetrical to https://github.com/databricks/cli/pull/1610/files ## Tests Unit tests --- .../mutator/python/python_diagnostics.go | 6 ++- .../mutator/python/python_diagnostics_test.go | 2 +- bundle/config/mutator/run_as.go | 2 +- bundle/config/validate/files_to_sync.go | 3 +- .../validate/job_cluster_key_defined.go | 2 +- .../config/validate/validate_sync_patterns.go | 2 +- bundle/render/render_text_output.go | 8 +-- bundle/render/render_text_output_test.go | 23 +++++++- .../sync_include_exclude_no_matches_test.go | 4 +- libs/diag/diagnostic.go | 6 +-- libs/dyn/convert/normalize.go | 14 ++--- libs/dyn/convert/normalize_test.go | 52 +++++++++---------- 12 files changed, 76 insertions(+), 48 deletions(-) diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index 96baa5093..12822065b 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -54,6 +54,10 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { if err != nil { return nil, fmt.Errorf("failed to parse path: %s", err) } + var paths []dyn.Path + if path != nil { + paths = []dyn.Path{path} + } var locations []dyn.Location location := convertPythonLocation(parsedLine.Location) @@ -66,7 +70,7 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { Summary: parsedLine.Summary, Detail: parsedLine.Detail, Locations: locations, - Path: path, + Paths: paths, } diags = diags.Append(diag) diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 09d9f93bd..b73b0f73c 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -56,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Path: dyn.MustPathFromString("resources.jobs.job0.name"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.job0.name")}, }, }, }, diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index 168918d0d..423bc38e2 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -180,7 +180,7 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { { Severity: diag.Warning, Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Path: dyn.MustPathFromString("experimental.use_legacy_run_as"), + Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index ae6bfef1a..7cdad772a 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func FilesToSync() bundle.ReadOnlyMutator { @@ -48,7 +49,7 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag. // Show all locations where sync.exclude is defined, since merging // sync.exclude is additive. Locations: loc.Locations(), - Path: loc.Path(), + Paths: []dyn.Path{loc.Path()}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 168303d83..368c3edb1 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -46,7 +46,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund // Other associated locations are not relevant since they are // overridden during merging. Locations: []dyn.Location{loc.Location()}, - Path: loc.Path(), + Paths: []dyn.Path{loc.Path()}, }) } } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index f3655ca94..573077b66 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -68,7 +68,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di Severity: diag.Warning, Summary: fmt.Sprintf("Pattern %s does not match any files", p), Locations: []dyn.Location{loc.Location()}, - Path: loc.Path(), + Paths: []dyn.Path{loc.Path()}, }) mu.Unlock() } diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 2ef6b2656..ea0b9a944 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -29,8 +29,8 @@ var renderFuncMap = template.FuncMap{ } const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} {{- range $index, $element := .Locations }} {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} @@ -43,8 +43,8 @@ const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} ` const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} {{- range $index, $element := .Locations }} {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 81e276199..976f86e79 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -275,7 +275,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Path: dyn.MustPathFromString("resources.jobs.xxx"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.xxx")}, }, }, expected: "Error: failed to load xxx\n" + @@ -283,6 +283,27 @@ func TestRenderDiagnostics(t *testing.T) { "\n" + "'name' is required\n\n", }, + { + name: "error with multiple paths", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Detail: "'name' is required", + Summary: "failed to load xxx", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + dyn.MustPathFromString("resources.jobs.zzz"), + }, + }, + }, + expected: "Error: failed to load xxx\n" + + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " resources.jobs.zzz\n" + + "\n" + + "'name' is required\n\n", + }, } for _, tc := range testCases { diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 5f4fa47ce..23f99b3a7 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -22,7 +22,9 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[0].Severity, diag.Warning) require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") - require.Equal(t, diags[0].Path.String(), "sync.exclude[0]") + + require.Len(t, diags[0].Paths, 1) + require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]") assert.Len(t, diags[0].Locations, 1) require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index 305089d22..93334c067 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -21,9 +21,9 @@ type Diagnostic struct { // It may be empty if there are no associated locations. Locations []dyn.Location - // Path is a path to the value in a configuration tree that the diagnostic is associated with. - // It may be nil if there is no associated path. - Path dyn.Path + // Paths are paths to the values in the configuration tree that the diagnostic is associated with. + // It may be nil if there are no associated paths. + Paths []dyn.Path } // Errorf creates a new error diagnostic. diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index bf5756e7f..c80a914f1 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -68,7 +68,7 @@ func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnosti Severity: diag.Warning, Summary: fmt.Sprintf("expected a %s value, found null", expected), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, } } @@ -77,7 +77,7 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost Severity: diag.Warning, Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, } } @@ -100,7 +100,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), // Show all locations the unknown field is defined at. Locations: pk.Locations(), - Path: path, + Paths: []dyn.Path{path}, }) } continue @@ -324,7 +324,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -340,7 +340,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: @@ -367,7 +367,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -383,7 +383,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d Severity: diag.Warning, Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), Locations: []dyn.Location{src.Location()}, - Path: path, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index df9a1a9a5..c2256615e 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -43,7 +43,7 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Path: dyn.NewPath(dyn.Key("bar")), + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -79,7 +79,7 @@ func TestNormalizeStructUnknownField(t *testing.T) { {File: "hello.yaml", Line: 1, Column: 1}, {File: "world.yaml", Line: 2, Column: 2}, }, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) // The field that can be mapped to the struct field is retained. @@ -113,7 +113,7 @@ func TestNormalizeStructError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Get("foo").Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -258,7 +258,7 @@ func TestNormalizeStructRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -275,7 +275,7 @@ func TestNormalizeStructIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -304,7 +304,7 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Path: dyn.NewPath(dyn.Key("bar")), + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -330,7 +330,7 @@ func TestNormalizeMapError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -385,7 +385,7 @@ func TestNormalizeMapRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -398,7 +398,7 @@ func TestNormalizeMapIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected map, found int`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -428,7 +428,7 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Path: dyn.NewPath(dyn.Index(2)), + Paths: []dyn.Path{dyn.NewPath(dyn.Index(2))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -452,7 +452,7 @@ func TestNormalizeSliceError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -507,7 +507,7 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -520,7 +520,7 @@ func TestNormalizeSliceIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected sequence, found int`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -541,7 +541,7 @@ func TestNormalizeStringNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a string value, found null`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -578,7 +578,7 @@ func TestNormalizeStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected string, found map`, Locations: []dyn.Location{{}}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -599,7 +599,7 @@ func TestNormalizeBoolNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a bool value, found null`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -641,7 +641,7 @@ func TestNormalizeBoolFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found string`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -654,7 +654,7 @@ func TestNormalizeBoolError(t *testing.T) { Severity: diag.Warning, Summary: `expected bool, found map`, Locations: []dyn.Location{{}}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -675,7 +675,7 @@ func TestNormalizeIntNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a int value, found null`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -696,7 +696,7 @@ func TestNormalizeIntFromFloatError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "1.5" as integer due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -725,7 +725,7 @@ func TestNormalizeIntFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as an integer`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -738,7 +738,7 @@ func TestNormalizeIntError(t *testing.T) { Severity: diag.Warning, Summary: `expected int, found map`, Locations: []dyn.Location{{}}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -759,7 +759,7 @@ func TestNormalizeFloatNil(t *testing.T) { Severity: diag.Warning, Summary: `expected a float value, found null`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -784,7 +784,7 @@ func TestNormalizeFloatFromIntError(t *testing.T) { Severity: diag.Warning, Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -813,7 +813,7 @@ func TestNormalizeFloatFromStringError(t *testing.T) { Severity: diag.Warning, Summary: `cannot parse "abc" as a floating point number`, Locations: []dyn.Location{vin.Location()}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -826,7 +826,7 @@ func TestNormalizeFloatError(t *testing.T) { Severity: diag.Warning, Summary: `expected float, found map`, Locations: []dyn.Location{{}}, - Path: dyn.EmptyPath, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } From 383d580917a8459eeebd1ce54d95e65afaa13aa7 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 26 Jul 2024 11:33:36 +0200 Subject: [PATCH 33/88] Release v0.224.1 (#1627) Bundles: * Add UUID function to bundle template functions ([#1612](https://github.com/databricks/cli/pull/1612)). * Upgrade TF provider to 1.49.0 ([#1617](https://github.com/databricks/cli/pull/1617)). * Upgrade TF provider to 1.49.1 ([#1626](https://github.com/databricks/cli/pull/1626)). * Support multiple locations for diagnostics ([#1610](https://github.com/databricks/cli/pull/1610)). * Split artifact cleanup into prepare step before build ([#1618](https://github.com/databricks/cli/pull/1618)). * Move to a single prompt during bundle destroy ([#1583](https://github.com/databricks/cli/pull/1583)). Internal: * Add tests for the Workspace API readahead cache ([#1605](https://github.com/databricks/cli/pull/1605)). * Update Python dependencies before install when upgrading a labs project ([#1624](https://github.com/databricks/cli/pull/1624)). --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 622519f23..b272377ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Version changelog +## 0.224.1 + +Bundles: + * Add UUID function to bundle template functions ([#1612](https://github.com/databricks/cli/pull/1612)). + * Upgrade TF provider to 1.49.0 ([#1617](https://github.com/databricks/cli/pull/1617)). + * Upgrade TF provider to 1.49.1 ([#1626](https://github.com/databricks/cli/pull/1626)). + * Support multiple locations for diagnostics ([#1610](https://github.com/databricks/cli/pull/1610)). + * Split artifact cleanup into prepare step before build ([#1618](https://github.com/databricks/cli/pull/1618)). + * Move to a single prompt during bundle destroy ([#1583](https://github.com/databricks/cli/pull/1583)). + +Internal: + * Add tests for the Workspace API readahead cache ([#1605](https://github.com/databricks/cli/pull/1605)). + * Update Python dependencies before install when upgrading a labs project ([#1624](https://github.com/databricks/cli/pull/1624)). + + + ## 0.224.0 CLI: From a52b188e9952c5b99bb59589fc8469c71983fa26 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 29 Jul 2024 18:34:02 +0530 Subject: [PATCH 34/88] Use dynamic walking to validate unique resource keys (#1614) ## Changes This PR: 1. Uses dynamic walking (via the `dyn.MapByPattern` func) to validate no two resources have the same resource key. The allows us to remove this validation at merge time. 2. Modifies `dyn.Mapping` to always return a sorted slice of pairs. This makes traversal functions like `dyn.Walk` or `dyn.MapByPattern` deterministic. ## Tests Unit tests. Also manually. --- bundle/config/mutator/mutator.go | 5 + bundle/config/resources.go | 120 --------------- bundle/config/resources_test.go | 120 --------------- bundle/config/root.go | 11 -- bundle/config/root_test.go | 16 -- .../config/validate/unique_resource_keys.go | 116 +++++++++++++++ bundle/tests/conflicting_resource_ids_test.go | 42 ------ .../databricks.yml | 8 +- .../resources1.yml} | 4 + .../resources2.yml | 8 + .../databricks.yml | 2 +- .../resources.yml | 0 .../databricks.yml | 3 + .../resources.yml | 4 + .../databricks.yml | 0 .../resources1.yml | 0 .../resources2.yml | 0 .../databricks.yml | 18 +++ .../databricks.yml | 0 bundle/tests/validate_test.go | 139 ++++++++++++++++++ libs/dyn/mapping.go | 3 +- 21 files changed, 304 insertions(+), 315 deletions(-) create mode 100644 bundle/config/validate/unique_resource_keys.go delete mode 100644 bundle/tests/conflicting_resource_ids_test.go rename bundle/tests/{conflicting_resource_ids/no_subconfigurations => validate/duplicate_resource_name_in_multiple_locations}/databricks.yml (53%) rename bundle/tests/{conflicting_resource_ids/one_subconfiguration/resources.yml => validate/duplicate_resource_name_in_multiple_locations/resources1.yml} (59%) create mode 100644 bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml rename bundle/tests/{conflicting_resource_ids/one_subconfiguration => validate/duplicate_resource_name_in_subconfiguration}/databricks.yml (84%) rename bundle/{config/testdata => tests/validate}/duplicate_resource_name_in_subconfiguration/resources.yml (100%) rename bundle/{config/testdata/duplicate_resource_name_in_subconfiguration => tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job}/databricks.yml (76%) create mode 100644 bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml rename bundle/tests/{conflicting_resource_ids/two_subconfigurations => validate/duplicate_resource_names_in_different_subconfiguations}/databricks.yml (100%) rename bundle/tests/{conflicting_resource_ids/two_subconfigurations => validate/duplicate_resource_names_in_different_subconfiguations}/resources1.yml (100%) rename bundle/tests/{conflicting_resource_ids/two_subconfigurations => validate/duplicate_resource_names_in_different_subconfiguations}/resources2.yml (100%) create mode 100644 bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml rename bundle/{config/testdata/duplicate_resource_names_in_root => tests/validate/duplicate_resource_names_in_root_job_and_pipeline}/databricks.yml (100%) create mode 100644 bundle/tests/validate_test.go diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 52f85eeb8..0458beff4 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/scripts" ) @@ -26,5 +27,9 @@ func DefaultMutators() []bundle.Mutator { DefineDefaultTarget(), LoadGitDetails(), pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad), + + // Note: This mutator must run before the target overrides are merged. + // See the mutator for more details. + validate.UniqueResourceKeys(), } } diff --git a/bundle/config/resources.go b/bundle/config/resources.go index f70052ec0..062e38ed5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -20,126 +20,6 @@ type Resources struct { QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` } -type UniqueResourceIdTracker struct { - Type map[string]string - ConfigPath map[string]string -} - -// verifies merging is safe by checking no duplicate identifiers exist -func (r *Resources) VerifySafeMerge(other *Resources) error { - rootTracker, err := r.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - otherTracker, err := other.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - for k := range otherTracker.Type { - if _, ok := rootTracker.Type[k]; ok { - return fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - rootTracker.Type[k], - rootTracker.ConfigPath[k], - otherTracker.Type[k], - otherTracker.ConfigPath[k], - ) - } - } - return nil -} - -// This function verifies there are no duplicate names used for the resource definations -func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker, error) { - tracker := &UniqueResourceIdTracker{ - Type: make(map[string]string), - ConfigPath: make(map[string]string), - } - for k := range r.Jobs { - tracker.Type[k] = "job" - tracker.ConfigPath[k] = r.Jobs[k].ConfigFilePath - } - for k := range r.Pipelines { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "pipeline", - r.Pipelines[k].ConfigFilePath, - ) - } - tracker.Type[k] = "pipeline" - tracker.ConfigPath[k] = r.Pipelines[k].ConfigFilePath - } - for k := range r.Models { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_model", - r.Models[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_model" - tracker.ConfigPath[k] = r.Models[k].ConfigFilePath - } - for k := range r.Experiments { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_experiment", - r.Experiments[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_experiment" - tracker.ConfigPath[k] = r.Experiments[k].ConfigFilePath - } - for k := range r.ModelServingEndpoints { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "model_serving_endpoint", - r.ModelServingEndpoints[k].ConfigFilePath, - ) - } - tracker.Type[k] = "model_serving_endpoint" - tracker.ConfigPath[k] = r.ModelServingEndpoints[k].ConfigFilePath - } - for k := range r.RegisteredModels { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "registered_model", - r.RegisteredModels[k].ConfigFilePath, - ) - } - tracker.Type[k] = "registered_model" - tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath - } - for k := range r.QualityMonitors { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "quality_monitor", - r.QualityMonitors[k].ConfigFilePath, - ) - } - tracker.Type[k] = "quality_monitor" - tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath - } - return tracker, nil -} - type resource struct { resource ConfigResource resource_type string diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 7415029b1..6860d73da 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -5,129 +5,9 @@ import ( "reflect" "testing" - "github.com/databricks/cli/bundle/config/paths" - "github.com/databricks/cli/bundle/config/resources" "github.com/stretchr/testify/assert" ) -func TestVerifyUniqueResourceIdentifiers(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - Experiments: map[string]*resources.MlflowExperiment{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - _, err := r.VerifyUniqueResourceIdentifiers() - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, mlflow_experiment at foo2.yml)") -} - -func TestVerifySafeMerge(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Pipelines: map[string]*resources.Pipeline{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, pipeline at foo2.yml)") -} - -func TestVerifySafeMergeForSameResourceType(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, job at foo2.yml)") -} - -func TestVerifySafeMergeForRegisteredModels(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)") -} - // This test ensures that all resources have a custom marshaller and unmarshaller. // This is required because DABs resources map to Databricks APIs, and they do so // by embedding the corresponding Go SDK structs. diff --git a/bundle/config/root.go b/bundle/config/root.go index 4239a34d0..cace22156 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -100,11 +100,6 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { if err != nil { return nil, diag.Errorf("failed to load %s: %v", path, err) } - - _, err = r.Resources.VerifyUniqueResourceIdentifiers() - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } return &r, diags } @@ -281,12 +276,6 @@ func (r *Root) InitializeVariables(vars []string) error { } func (r *Root) Merge(other *Root) error { - // Check for safe merge, protecting against duplicate resource identifiers - err := r.Resources.VerifySafeMerge(&other.Resources) - if err != nil { - return err - } - // Merge dynamic configuration values. return r.Mutate(func(root dyn.Value) (dyn.Value, error) { return merge.Merge(root, other.value) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index aed670d6c..c95e6e86c 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -30,22 +30,6 @@ func TestRootLoad(t *testing.T) { assert.Equal(t, "basic", root.Bundle.Name) } -func TestDuplicateIdOnLoadReturnsError(t *testing.T) { - _, diags := Load("./testdata/duplicate_resource_names_in_root/databricks.yml") - assert.ErrorContains(t, diags.Error(), "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)") -} - -func TestDuplicateIdOnMergeReturnsError(t *testing.T) { - root, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml") - require.NoError(t, diags.Error()) - - other, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/resources.yml") - require.NoError(t, diags.Error()) - - err := root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") -} - func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go new file mode 100644 index 000000000..d6212b0ac --- /dev/null +++ b/bundle/config/validate/unique_resource_keys.go @@ -0,0 +1,116 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "sort" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// This mutator validates that: +// +// 1. Each resource key is unique across different resource types. No two resources +// of the same type can have the same key. This is because command like "bundle run" +// rely on the resource key to identify the resource to run. +// Eg: jobs.foo and pipelines.foo are not allowed simultaneously. +// +// 2. Each resource definition is contained within a single file, and is not spread +// across multiple files. Note: This is not applicable to resource configuration +// defined in a target override. That is why this mutator MUST run before the target +// overrides are merged. +func UniqueResourceKeys() bundle.Mutator { + return &uniqueResourceKeys{} +} + +type uniqueResourceKeys struct{} + +func (m *uniqueResourceKeys) Name() string { + return "validate:unique_resource_keys" +} + +func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + type metadata struct { + locations []dyn.Location + paths []dyn.Path + } + + // Maps of resource key to the paths and locations the resource is defined at. + resourceMetadata := map[string]*metadata{} + + rv := b.Config.Value().Get("resources") + + // return early if no resources are defined or the resources block is empty. + if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { + return diags + } + + // Gather the paths and locations of all resources. + _, err := dyn.MapByPattern( + rv, + dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource. Eg: "my_job" for jobs.my_job. + k := p[1].Key() + + m, ok := resourceMetadata[k] + if !ok { + m = &metadata{ + paths: []dyn.Path{}, + locations: []dyn.Location{}, + } + } + + // dyn.Path under the hood is a slice. The code that walks the configuration + // tree uses the same underlying slice to track the path as it walks + // the tree. So, we need to clone it here. + m.paths = append(m.paths, slices.Clone(p)) + m.locations = append(m.locations, v.Locations()...) + + resourceMetadata[k] = m + return v, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + for k, v := range resourceMetadata { + if len(v.locations) <= 1 { + continue + } + + // Sort the locations and paths for consistent error messages. This helps + // with unit testing. + sort.Slice(v.locations, func(i, j int) bool { + l1 := v.locations[i] + l2 := v.locations[j] + + if l1.File != l2.File { + return l1.File < l2.File + } + if l1.Line != l2.Line { + return l1.Line < l2.Line + } + return l1.Column < l2.Column + }) + sort.Slice(v.paths, func(i, j int) bool { + return v.paths[i].String() < v.paths[j].String() + }) + + // If there are multiple resources with the same key, report an error. + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("multiple resources have been defined with the same key: %s", k), + Locations: v.locations, + Paths: v.paths, + }) + } + + return diags +} diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go deleted file mode 100644 index e7f0aa28f..000000000 --- a/bundle/tests/conflicting_resource_ids_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package config_tests - -import ( - "context" - "fmt" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConflictingResourceIdsNoSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/no_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) -} - -func TestConflictingResourceIdsOneSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/one_subconfiguration") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml") - resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) -} - -func TestConflictingResourceIdsTwoSubconfigs(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/two_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - resources1ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources1.yml") - resources2ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources2.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", resources1ConfigPath, resources2ConfigPath)) -} diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml similarity index 53% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml index 1e9aa10b1..ebb1f9005 100644 --- a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml @@ -4,10 +4,10 @@ bundle: workspace: profile: test +include: + - ./*.yml + resources: jobs: foo: - name: job foo - pipelines: - foo: - name: pipeline foo + name: job foo 1 diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml similarity index 59% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml index c3dcb6e2f..deb81caa1 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml @@ -1,4 +1,8 @@ resources: + jobs: + foo: + name: job foo 2 + pipelines: foo: name: pipeline foo diff --git a/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml new file mode 100644 index 000000000..4e0a342b3 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + name: job foo 3 + + experiments: + foo: + name: experiment foo diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml similarity index 84% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml index ea4dec2e1..5bec67483 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml @@ -5,7 +5,7 @@ workspace: profile: test include: - - "*.yml" + - ./resources.yml resources: jobs: diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml similarity index 76% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml index a81602920..5bec67483 100644 --- a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml @@ -4,6 +4,9 @@ bundle: workspace: profile: test +include: + - ./resources.yml + resources: jobs: foo: diff --git a/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml new file mode 100644 index 000000000..83fb75735 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + name: job foo 2 diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml diff --git a/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml new file mode 100644 index 000000000..d286f1049 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test + +workspace: + profile: test + +resources: + jobs: + foo: + name: job foo + bar: + name: job bar + pipelines: + baz: + name: pipeline baz + experiments: + foo: + name: experiment foo diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go new file mode 100644 index 000000000..9cd7c201b --- /dev/null +++ b/bundle/tests/validate_test.go @@ -0,0 +1,139 @@ +package config_tests + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateUniqueResourceIdentifiers(t *testing.T) { + tcases := []struct { + name string + diagnostics diag.Diagnostics + }{ + { + name: "duplicate_resource_names_in_root_job_and_pipeline", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 13, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_root_job_and_experiment", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 18, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration_job_and_job", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_different_subconfiguations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_multiple_locations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 8, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./validate/"+tc.name) + require.NoError(t, err) + + // The UniqueResourceKeys mutator is run as part of the Load phase. + diags := bundle.Apply(ctx, b, phases.Load()) + assert.Equal(t, tc.diagnostics, diags) + }) + } +} diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go index 668f57ecc..f9f2d2e97 100644 --- a/libs/dyn/mapping.go +++ b/libs/dyn/mapping.go @@ -46,7 +46,8 @@ func newMappingFromGoMap(vin map[string]Value) Mapping { return m } -// Pairs returns all the key-value pairs in the Mapping. +// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by +// their key in lexicographic order. func (m Mapping) Pairs() []Pair { return m.pairs } From ecba875fe5728bf647de57576b557c6f08591688 Mon Sep 17 00:00:00 2001 From: Alex Moschos <166370939+alexmos-db@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:13:05 +0200 Subject: [PATCH 35/88] Regenerate TF schema (#1635) ## Changes - Regenerate TF schema for CLI. Due to an issue the previous generation missed some TF changes. --- bundle/internal/tf/schema/config.go | 61 ++--- .../internal/tf/schema/data_source_cluster.go | 232 +++++++++++++++++- .../schema/data_source_external_location.go | 1 + bundle/internal/tf/schema/data_source_job.go | 52 ++-- .../schema/data_source_storage_credential.go | 1 + bundle/internal/tf/schema/data_sources.go | 198 +++++++-------- .../tf/schema/resource_external_location.go | 1 + bundle/internal/tf/schema/resource_job.go | 56 +++-- .../schema/resource_metastore_data_access.go | 1 + .../tf/schema/resource_mws_workspaces.go | 1 + .../tf/schema/resource_online_table.go | 9 +- .../tf/schema/resource_permissions.go | 1 + .../tf/schema/resource_storage_credential.go | 2 + .../tf/schema/resource_system_schema.go | 1 + bundle/internal/tf/schema/resources.go | 4 + 15 files changed, 449 insertions(+), 172 deletions(-) diff --git a/bundle/internal/tf/schema/config.go b/bundle/internal/tf/schema/config.go index d24d57339..e807cdc53 100644 --- a/bundle/internal/tf/schema/config.go +++ b/bundle/internal/tf/schema/config.go @@ -3,33 +3,36 @@ package schema type Config struct { - AccountId string `json:"account_id,omitempty"` - AuthType string `json:"auth_type,omitempty"` - AzureClientId string `json:"azure_client_id,omitempty"` - AzureClientSecret string `json:"azure_client_secret,omitempty"` - AzureEnvironment string `json:"azure_environment,omitempty"` - AzureLoginAppId string `json:"azure_login_app_id,omitempty"` - AzureTenantId string `json:"azure_tenant_id,omitempty"` - AzureUseMsi bool `json:"azure_use_msi,omitempty"` - AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` - ClientId string `json:"client_id,omitempty"` - ClientSecret string `json:"client_secret,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ConfigFile string `json:"config_file,omitempty"` - DatabricksCliPath string `json:"databricks_cli_path,omitempty"` - DebugHeaders bool `json:"debug_headers,omitempty"` - DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` - GoogleCredentials string `json:"google_credentials,omitempty"` - GoogleServiceAccount string `json:"google_service_account,omitempty"` - Host string `json:"host,omitempty"` - HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` - MetadataServiceUrl string `json:"metadata_service_url,omitempty"` - Password string `json:"password,omitempty"` - Profile string `json:"profile,omitempty"` - RateLimit int `json:"rate_limit,omitempty"` - RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` - SkipVerify bool `json:"skip_verify,omitempty"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + AccountId string `json:"account_id,omitempty"` + ActionsIdTokenRequestToken string `json:"actions_id_token_request_token,omitempty"` + ActionsIdTokenRequestUrl string `json:"actions_id_token_request_url,omitempty"` + AuthType string `json:"auth_type,omitempty"` + AzureClientId string `json:"azure_client_id,omitempty"` + AzureClientSecret string `json:"azure_client_secret,omitempty"` + AzureEnvironment string `json:"azure_environment,omitempty"` + AzureLoginAppId string `json:"azure_login_app_id,omitempty"` + AzureTenantId string `json:"azure_tenant_id,omitempty"` + AzureUseMsi bool `json:"azure_use_msi,omitempty"` + AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` + ClientId string `json:"client_id,omitempty"` + ClientSecret string `json:"client_secret,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ConfigFile string `json:"config_file,omitempty"` + DatabricksCliPath string `json:"databricks_cli_path,omitempty"` + DebugHeaders bool `json:"debug_headers,omitempty"` + DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` + GoogleCredentials string `json:"google_credentials,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + Host string `json:"host,omitempty"` + HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` + MetadataServiceUrl string `json:"metadata_service_url,omitempty"` + Password string `json:"password,omitempty"` + Profile string `json:"profile,omitempty"` + RateLimit int `json:"rate_limit,omitempty"` + RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` + ServerlessComputeId string `json:"serverless_compute_id,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + WarehouseId string `json:"warehouse_id,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index fff66dc93..94d67bbfa 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -10,7 +10,9 @@ type DataSourceClusterClusterInfoAutoscale struct { type DataSourceClusterClusterInfoAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -18,10 +20,16 @@ type DataSourceClusterClusterInfoAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type DataSourceClusterClusterInfoAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type DataSourceClusterClusterInfoClusterLogConfDbfs struct { @@ -49,12 +57,12 @@ type DataSourceClusterClusterInfoClusterLogStatus struct { } type DataSourceClusterClusterInfoDockerImageBasicAuth struct { - Password string `json:"password"` - Username string `json:"username"` + Password string `json:"password,omitempty"` + Username string `json:"username,omitempty"` } type DataSourceClusterClusterInfoDockerImage struct { - Url string `json:"url"` + Url string `json:"url,omitempty"` BasicAuth *DataSourceClusterClusterInfoDockerImageBasicAuth `json:"basic_auth,omitempty"` } @@ -139,12 +147,212 @@ type DataSourceClusterClusterInfoInitScripts struct { Workspace *DataSourceClusterClusterInfoInitScriptsWorkspace `json:"workspace,omitempty"` } +type DataSourceClusterClusterInfoSpecAutoscale struct { + MaxWorkers int `json:"max_workers,omitempty"` + MinWorkers int `json:"min_workers,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAwsAttributes struct { + Availability string `json:"availability,omitempty"` + EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` + EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` + EbsVolumeType string `json:"ebs_volume_type,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + SpotBidPricePercent int `json:"spot_bid_price_percent,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributes struct { + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConf struct { + Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { + MountOptions string `json:"mount_options,omitempty"` + ServerAddress string `json:"server_address"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfo struct { + LocalMountDirPath string `json:"local_mount_dir_path"` + RemoteMountDirPath string `json:"remote_mount_dir_path,omitempty"` + NetworkFilesystemInfo *DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo `json:"network_filesystem_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecDockerImageBasicAuth struct { + Password string `json:"password"` + Username string `json:"username"` +} + +type DataSourceClusterClusterInfoSpecDockerImage struct { + Url string `json:"url"` + BasicAuth *DataSourceClusterClusterInfoSpecDockerImageBasicAuth `json:"basic_auth,omitempty"` +} + +type DataSourceClusterClusterInfoSpecGcpAttributes struct { + Availability string `json:"availability,omitempty"` + BootDiskSize int `json:"boot_disk_size,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + LocalSsdCount int `json:"local_ssd_count,omitempty"` + UsePreemptibleExecutors bool `json:"use_preemptible_executors,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsAbfss struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsFile struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsGcs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsVolumes struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsWorkspace struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScripts struct { + Abfss *DataSourceClusterClusterInfoSpecInitScriptsAbfss `json:"abfss,omitempty"` + Dbfs *DataSourceClusterClusterInfoSpecInitScriptsDbfs `json:"dbfs,omitempty"` + File *DataSourceClusterClusterInfoSpecInitScriptsFile `json:"file,omitempty"` + Gcs *DataSourceClusterClusterInfoSpecInitScriptsGcs `json:"gcs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecInitScriptsS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoSpecInitScriptsVolumes `json:"volumes,omitempty"` + Workspace *DataSourceClusterClusterInfoSpecInitScriptsWorkspace `json:"workspace,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceClusterClusterInfoSpecLibraryCran `json:"cran,omitempty"` + Maven *DataSourceClusterClusterInfoSpecLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceClusterClusterInfoSpecLibraryPypi `json:"pypi,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadType struct { + Clients *DataSourceClusterClusterInfoSpecWorkloadTypeClients `json:"clients,omitempty"` +} + +type DataSourceClusterClusterInfoSpec struct { + ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + DataSecurityMode string `json:"data_security_mode,omitempty"` + DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` + DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` + IdempotencyToken string `json:"idempotency_token,omitempty"` + InstancePoolId string `json:"instance_pool_id,omitempty"` + NodeTypeId string `json:"node_type_id,omitempty"` + NumWorkers int `json:"num_workers,omitempty"` + PolicyId string `json:"policy_id,omitempty"` + RuntimeEngine string `json:"runtime_engine,omitempty"` + SingleUserName string `json:"single_user_name,omitempty"` + SparkConf map[string]string `json:"spark_conf,omitempty"` + SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` + SparkVersion string `json:"spark_version"` + SshPublicKeys []string `json:"ssh_public_keys,omitempty"` + Autoscale *DataSourceClusterClusterInfoSpecAutoscale `json:"autoscale,omitempty"` + AwsAttributes *DataSourceClusterClusterInfoSpecAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *DataSourceClusterClusterInfoSpecAzureAttributes `json:"azure_attributes,omitempty"` + ClusterLogConf *DataSourceClusterClusterInfoSpecClusterLogConf `json:"cluster_log_conf,omitempty"` + ClusterMountInfo []DataSourceClusterClusterInfoSpecClusterMountInfo `json:"cluster_mount_info,omitempty"` + DockerImage *DataSourceClusterClusterInfoSpecDockerImage `json:"docker_image,omitempty"` + GcpAttributes *DataSourceClusterClusterInfoSpecGcpAttributes `json:"gcp_attributes,omitempty"` + InitScripts []DataSourceClusterClusterInfoSpecInitScripts `json:"init_scripts,omitempty"` + Library []DataSourceClusterClusterInfoSpecLibrary `json:"library,omitempty"` + WorkloadType *DataSourceClusterClusterInfoSpecWorkloadType `json:"workload_type,omitempty"` +} + type DataSourceClusterClusterInfoTerminationReason struct { Code string `json:"code,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` Type string `json:"type,omitempty"` } +type DataSourceClusterClusterInfoWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoWorkloadType struct { + Clients *DataSourceClusterClusterInfoWorkloadTypeClients `json:"clients,omitempty"` +} + type DataSourceClusterClusterInfo struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterCores int `json:"cluster_cores,omitempty"` @@ -155,14 +363,14 @@ type DataSourceClusterClusterInfo struct { CreatorUserName string `json:"creator_user_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags"` + DefaultTags map[string]string `json:"default_tags,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` JdbcPort int `json:"jdbc_port,omitempty"` - LastActivityTime int `json:"last_activity_time,omitempty"` + LastRestartedTime int `json:"last_restarted_time,omitempty"` LastStateLossTime int `json:"last_state_loss_time,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` @@ -172,12 +380,12 @@ type DataSourceClusterClusterInfo struct { SparkConf map[string]string `json:"spark_conf,omitempty"` SparkContextId int `json:"spark_context_id,omitempty"` SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` + SparkVersion string `json:"spark_version,omitempty"` SshPublicKeys []string `json:"ssh_public_keys,omitempty"` StartTime int `json:"start_time,omitempty"` - State string `json:"state"` + State string `json:"state,omitempty"` StateMessage string `json:"state_message,omitempty"` - TerminateTime int `json:"terminate_time,omitempty"` + TerminatedTime int `json:"terminated_time,omitempty"` Autoscale *DataSourceClusterClusterInfoAutoscale `json:"autoscale,omitempty"` AwsAttributes *DataSourceClusterClusterInfoAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *DataSourceClusterClusterInfoAzureAttributes `json:"azure_attributes,omitempty"` @@ -188,7 +396,9 @@ type DataSourceClusterClusterInfo struct { Executors []DataSourceClusterClusterInfoExecutors `json:"executors,omitempty"` GcpAttributes *DataSourceClusterClusterInfoGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []DataSourceClusterClusterInfoInitScripts `json:"init_scripts,omitempty"` + Spec *DataSourceClusterClusterInfoSpec `json:"spec,omitempty"` TerminationReason *DataSourceClusterClusterInfoTerminationReason `json:"termination_reason,omitempty"` + WorkloadType *DataSourceClusterClusterInfoWorkloadType `json:"workload_type,omitempty"` } type DataSourceCluster struct { diff --git a/bundle/internal/tf/schema/data_source_external_location.go b/bundle/internal/tf/schema/data_source_external_location.go index 0fea6e529..a3e78cbd3 100644 --- a/bundle/internal/tf/schema/data_source_external_location.go +++ b/bundle/internal/tf/schema/data_source_external_location.go @@ -19,6 +19,7 @@ type DataSourceExternalLocationExternalLocationInfo struct { CreatedBy string `json:"created_by,omitempty"` CredentialId string `json:"credential_id,omitempty"` CredentialName string `json:"credential_name,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go index e5ec5afb7..91806d670 100644 --- a/bundle/internal/tf/schema/data_source_job.go +++ b/bundle/internal/tf/schema/data_source_job.go @@ -26,6 +26,7 @@ type DataSourceJobJobSettingsSettingsEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -55,9 +56,9 @@ type DataSourceJobJobSettingsSettingsGitSource struct { } type DataSourceJobJobSettingsSettingsHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsHealth struct { @@ -222,7 +223,7 @@ type DataSourceJobJobSettingsSettingsJobClusterNewCluster struct { } type DataSourceJobJobSettingsSettingsJobCluster struct { - JobClusterKey string `json:"job_cluster_key,omitempty"` + JobClusterKey string `json:"job_cluster_key"` NewCluster *DataSourceJobJobSettingsSettingsJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -500,6 +501,7 @@ type DataSourceJobJobSettingsSettingsTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -529,13 +531,14 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskEmailNotifications struc OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealth struct { @@ -805,7 +808,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -824,6 +827,10 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSt Id string `json:"id"` } +type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -832,6 +839,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotifications str OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsTaskForEachTaskTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -844,7 +852,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` @@ -872,9 +880,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTask struct { } type DataSourceJobJobSettingsSettingsTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsTaskHealth struct { @@ -1144,7 +1152,7 @@ type DataSourceJobJobSettingsSettingsTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *DataSourceJobJobSettingsSettingsTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskSqlTaskFile `json:"file,omitempty"` @@ -1163,6 +1171,10 @@ type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } +type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1171,6 +1183,7 @@ type DataSourceJobJobSettingsSettingsTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -1183,7 +1196,7 @@ type DataSourceJobJobSettingsSettingsTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskDbtTask `json:"dbt_task,omitempty"` @@ -1211,6 +1224,11 @@ type DataSourceJobJobSettingsSettingsTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } +type DataSourceJobJobSettingsSettingsTriggerPeriodic struct { + Interval int `json:"interval"` + Unit string `json:"unit"` +} + type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1221,6 +1239,7 @@ type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { type DataSourceJobJobSettingsSettingsTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *DataSourceJobJobSettingsSettingsTriggerFileArrival `json:"file_arrival,omitempty"` + Periodic *DataSourceJobJobSettingsSettingsTriggerPeriodic `json:"periodic,omitempty"` TableUpdate *DataSourceJobJobSettingsSettingsTriggerTableUpdate `json:"table_update,omitempty"` } @@ -1236,6 +1255,10 @@ type DataSourceJobJobSettingsSettingsWebhookNotificationsOnStart struct { Id string `json:"id"` } +type DataSourceJobJobSettingsSettingsWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type DataSourceJobJobSettingsSettingsWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1244,6 +1267,7 @@ type DataSourceJobJobSettingsSettingsWebhookNotifications struct { OnDurationWarningThresholdExceeded []DataSourceJobJobSettingsSettingsWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []DataSourceJobJobSettingsSettingsWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []DataSourceJobJobSettingsSettingsWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []DataSourceJobJobSettingsSettingsWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []DataSourceJobJobSettingsSettingsWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_storage_credential.go b/bundle/internal/tf/schema/data_source_storage_credential.go index c7045d445..bf58f2726 100644 --- a/bundle/internal/tf/schema/data_source_storage_credential.go +++ b/bundle/internal/tf/schema/data_source_storage_credential.go @@ -36,6 +36,7 @@ type DataSourceStorageCredentialStorageCredentialInfo struct { CreatedAt int `json:"created_at,omitempty"` CreatedBy string `json:"created_by,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index c32483db0..4ac78613f 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -3,105 +3,111 @@ package schema type DataSources struct { - AwsAssumeRolePolicy map[string]any `json:"databricks_aws_assume_role_policy,omitempty"` - AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"` - AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"` - AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"` - Catalog map[string]any `json:"databricks_catalog,omitempty"` - Catalogs map[string]any `json:"databricks_catalogs,omitempty"` - Cluster map[string]any `json:"databricks_cluster,omitempty"` - ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` - Clusters map[string]any `json:"databricks_clusters,omitempty"` - CurrentConfig map[string]any `json:"databricks_current_config,omitempty"` - CurrentMetastore map[string]any `json:"databricks_current_metastore,omitempty"` - CurrentUser map[string]any `json:"databricks_current_user,omitempty"` - DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` - DbfsFilePaths map[string]any `json:"databricks_dbfs_file_paths,omitempty"` - Directory map[string]any `json:"databricks_directory,omitempty"` - ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` - ExternalLocations map[string]any `json:"databricks_external_locations,omitempty"` - Group map[string]any `json:"databricks_group,omitempty"` - InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` - InstanceProfiles map[string]any `json:"databricks_instance_profiles,omitempty"` - Job map[string]any `json:"databricks_job,omitempty"` - Jobs map[string]any `json:"databricks_jobs,omitempty"` - Metastore map[string]any `json:"databricks_metastore,omitempty"` - Metastores map[string]any `json:"databricks_metastores,omitempty"` - MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` - MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` - MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` - MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` - NodeType map[string]any `json:"databricks_node_type,omitempty"` - Notebook map[string]any `json:"databricks_notebook,omitempty"` - NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` - Pipelines map[string]any `json:"databricks_pipelines,omitempty"` - Schemas map[string]any `json:"databricks_schemas,omitempty"` - ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` - ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` - Share map[string]any `json:"databricks_share,omitempty"` - Shares map[string]any `json:"databricks_shares,omitempty"` - SparkVersion map[string]any `json:"databricks_spark_version,omitempty"` - SqlWarehouse map[string]any `json:"databricks_sql_warehouse,omitempty"` - SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"` - StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` - StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"` - Table map[string]any `json:"databricks_table,omitempty"` - Tables map[string]any `json:"databricks_tables,omitempty"` - User map[string]any `json:"databricks_user,omitempty"` - Views map[string]any `json:"databricks_views,omitempty"` - Volumes map[string]any `json:"databricks_volumes,omitempty"` - Zones map[string]any `json:"databricks_zones,omitempty"` + AwsAssumeRolePolicy map[string]any `json:"databricks_aws_assume_role_policy,omitempty"` + AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"` + AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"` + AwsUnityCatalogAssumeRolePolicy map[string]any `json:"databricks_aws_unity_catalog_assume_role_policy,omitempty"` + AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"` + Catalog map[string]any `json:"databricks_catalog,omitempty"` + Catalogs map[string]any `json:"databricks_catalogs,omitempty"` + Cluster map[string]any `json:"databricks_cluster,omitempty"` + ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` + Clusters map[string]any `json:"databricks_clusters,omitempty"` + CurrentConfig map[string]any `json:"databricks_current_config,omitempty"` + CurrentMetastore map[string]any `json:"databricks_current_metastore,omitempty"` + CurrentUser map[string]any `json:"databricks_current_user,omitempty"` + DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` + DbfsFilePaths map[string]any `json:"databricks_dbfs_file_paths,omitempty"` + Directory map[string]any `json:"databricks_directory,omitempty"` + ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` + ExternalLocations map[string]any `json:"databricks_external_locations,omitempty"` + Group map[string]any `json:"databricks_group,omitempty"` + InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` + InstanceProfiles map[string]any `json:"databricks_instance_profiles,omitempty"` + Job map[string]any `json:"databricks_job,omitempty"` + Jobs map[string]any `json:"databricks_jobs,omitempty"` + Metastore map[string]any `json:"databricks_metastore,omitempty"` + Metastores map[string]any `json:"databricks_metastores,omitempty"` + MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` + MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` + MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` + NodeType map[string]any `json:"databricks_node_type,omitempty"` + Notebook map[string]any `json:"databricks_notebook,omitempty"` + NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` + Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` + Schemas map[string]any `json:"databricks_schemas,omitempty"` + ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` + ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` + Share map[string]any `json:"databricks_share,omitempty"` + Shares map[string]any `json:"databricks_shares,omitempty"` + SparkVersion map[string]any `json:"databricks_spark_version,omitempty"` + SqlWarehouse map[string]any `json:"databricks_sql_warehouse,omitempty"` + SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"` + StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` + StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"` + Table map[string]any `json:"databricks_table,omitempty"` + Tables map[string]any `json:"databricks_tables,omitempty"` + User map[string]any `json:"databricks_user,omitempty"` + Views map[string]any `json:"databricks_views,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` + Volumes map[string]any `json:"databricks_volumes,omitempty"` + Zones map[string]any `json:"databricks_zones,omitempty"` } func NewDataSources() *DataSources { return &DataSources{ - AwsAssumeRolePolicy: make(map[string]any), - AwsBucketPolicy: make(map[string]any), - AwsCrossaccountPolicy: make(map[string]any), - AwsUnityCatalogPolicy: make(map[string]any), - Catalog: make(map[string]any), - Catalogs: make(map[string]any), - Cluster: make(map[string]any), - ClusterPolicy: make(map[string]any), - Clusters: make(map[string]any), - CurrentConfig: make(map[string]any), - CurrentMetastore: make(map[string]any), - CurrentUser: make(map[string]any), - DbfsFile: make(map[string]any), - DbfsFilePaths: make(map[string]any), - Directory: make(map[string]any), - ExternalLocation: make(map[string]any), - ExternalLocations: make(map[string]any), - Group: make(map[string]any), - InstancePool: make(map[string]any), - InstanceProfiles: make(map[string]any), - Job: make(map[string]any), - Jobs: make(map[string]any), - Metastore: make(map[string]any), - Metastores: make(map[string]any), - MlflowExperiment: make(map[string]any), - MlflowModel: make(map[string]any), - MwsCredentials: make(map[string]any), - MwsWorkspaces: make(map[string]any), - NodeType: make(map[string]any), - Notebook: make(map[string]any), - NotebookPaths: make(map[string]any), - Pipelines: make(map[string]any), - Schemas: make(map[string]any), - ServicePrincipal: make(map[string]any), - ServicePrincipals: make(map[string]any), - Share: make(map[string]any), - Shares: make(map[string]any), - SparkVersion: make(map[string]any), - SqlWarehouse: make(map[string]any), - SqlWarehouses: make(map[string]any), - StorageCredential: make(map[string]any), - StorageCredentials: make(map[string]any), - Table: make(map[string]any), - Tables: make(map[string]any), - User: make(map[string]any), - Views: make(map[string]any), - Volumes: make(map[string]any), - Zones: make(map[string]any), + AwsAssumeRolePolicy: make(map[string]any), + AwsBucketPolicy: make(map[string]any), + AwsCrossaccountPolicy: make(map[string]any), + AwsUnityCatalogAssumeRolePolicy: make(map[string]any), + AwsUnityCatalogPolicy: make(map[string]any), + Catalog: make(map[string]any), + Catalogs: make(map[string]any), + Cluster: make(map[string]any), + ClusterPolicy: make(map[string]any), + Clusters: make(map[string]any), + CurrentConfig: make(map[string]any), + CurrentMetastore: make(map[string]any), + CurrentUser: make(map[string]any), + DbfsFile: make(map[string]any), + DbfsFilePaths: make(map[string]any), + Directory: make(map[string]any), + ExternalLocation: make(map[string]any), + ExternalLocations: make(map[string]any), + Group: make(map[string]any), + InstancePool: make(map[string]any), + InstanceProfiles: make(map[string]any), + Job: make(map[string]any), + Jobs: make(map[string]any), + Metastore: make(map[string]any), + Metastores: make(map[string]any), + MlflowExperiment: make(map[string]any), + MlflowModel: make(map[string]any), + MwsCredentials: make(map[string]any), + MwsWorkspaces: make(map[string]any), + NodeType: make(map[string]any), + Notebook: make(map[string]any), + NotebookPaths: make(map[string]any), + Pipelines: make(map[string]any), + Schema: make(map[string]any), + Schemas: make(map[string]any), + ServicePrincipal: make(map[string]any), + ServicePrincipals: make(map[string]any), + Share: make(map[string]any), + Shares: make(map[string]any), + SparkVersion: make(map[string]any), + SqlWarehouse: make(map[string]any), + SqlWarehouses: make(map[string]any), + StorageCredential: make(map[string]any), + StorageCredentials: make(map[string]any), + Table: make(map[string]any), + Tables: make(map[string]any), + User: make(map[string]any), + Views: make(map[string]any), + Volume: make(map[string]any), + Volumes: make(map[string]any), + Zones: make(map[string]any), } } diff --git a/bundle/internal/tf/schema/resource_external_location.go b/bundle/internal/tf/schema/resource_external_location.go index af64c677c..da28271bc 100644 --- a/bundle/internal/tf/schema/resource_external_location.go +++ b/bundle/internal/tf/schema/resource_external_location.go @@ -18,6 +18,7 @@ type ResourceExternalLocation struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 6e624ad8a..42b648b0f 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -26,6 +26,7 @@ type ResourceJobEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -60,9 +61,9 @@ type ResourceJobGitSource struct { } type ResourceJobHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobHealth struct { @@ -229,7 +230,6 @@ type ResourceJobJobClusterNewClusterWorkloadType struct { type ResourceJobJobClusterNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -262,7 +262,7 @@ type ResourceJobJobClusterNewCluster struct { } type ResourceJobJobCluster struct { - JobClusterKey string `json:"job_cluster_key,omitempty"` + JobClusterKey string `json:"job_cluster_key"` NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -452,7 +452,6 @@ type ResourceJobNewClusterWorkloadType struct { type ResourceJobNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -575,6 +574,7 @@ type ResourceJobTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } @@ -604,13 +604,14 @@ type ResourceJobTaskForEachTaskTaskEmailNotifications struct { OnDurationWarningThresholdExceeded []string `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []string `json:"on_failure,omitempty"` OnStart []string `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []string `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []string `json:"on_success,omitempty"` } type ResourceJobTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobTaskForEachTaskTaskHealth struct { @@ -803,7 +804,6 @@ type ResourceJobTaskForEachTaskTaskNewClusterWorkloadType struct { type ResourceJobTaskForEachTaskTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -927,7 +927,7 @@ type ResourceJobTaskForEachTaskTaskSqlTaskQuery struct { type ResourceJobTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *ResourceJobTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -946,6 +946,10 @@ type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } +type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type ResourceJobTaskForEachTaskTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -954,6 +958,7 @@ type ResourceJobTaskForEachTaskTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobTaskForEachTaskTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -967,7 +972,7 @@ type ResourceJobTaskForEachTaskTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` @@ -995,9 +1000,9 @@ type ResourceJobTaskForEachTask struct { } type ResourceJobTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobTaskHealth struct { @@ -1190,7 +1195,6 @@ type ResourceJobTaskNewClusterWorkloadType struct { type ResourceJobTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -1314,7 +1318,7 @@ type ResourceJobTaskSqlTaskQuery struct { type ResourceJobTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *ResourceJobTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskSqlTaskFile `json:"file,omitempty"` @@ -1333,6 +1337,10 @@ type ResourceJobTaskWebhookNotificationsOnStart struct { Id string `json:"id"` } +type ResourceJobTaskWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type ResourceJobTaskWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1341,6 +1349,7 @@ type ResourceJobTaskWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobTaskWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobTaskWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobTaskWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []ResourceJobTaskWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobTaskWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } @@ -1354,7 +1363,7 @@ type ResourceJobTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"` DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"` @@ -1382,6 +1391,11 @@ type ResourceJobTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } +type ResourceJobTriggerPeriodic struct { + Interval int `json:"interval"` + Unit string `json:"unit"` +} + type ResourceJobTriggerTable struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1399,6 +1413,7 @@ type ResourceJobTriggerTableUpdate struct { type ResourceJobTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"` + Periodic *ResourceJobTriggerPeriodic `json:"periodic,omitempty"` Table *ResourceJobTriggerTable `json:"table,omitempty"` TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"` } @@ -1415,6 +1430,10 @@ type ResourceJobWebhookNotificationsOnStart struct { Id string `json:"id"` } +type ResourceJobWebhookNotificationsOnStreamingBacklogExceeded struct { + Id string `json:"id"` +} + type ResourceJobWebhookNotificationsOnSuccess struct { Id string `json:"id"` } @@ -1423,6 +1442,7 @@ type ResourceJobWebhookNotifications struct { OnDurationWarningThresholdExceeded []ResourceJobWebhookNotificationsOnDurationWarningThresholdExceeded `json:"on_duration_warning_threshold_exceeded,omitempty"` OnFailure []ResourceJobWebhookNotificationsOnFailure `json:"on_failure,omitempty"` OnStart []ResourceJobWebhookNotificationsOnStart `json:"on_start,omitempty"` + OnStreamingBacklogExceeded []ResourceJobWebhookNotificationsOnStreamingBacklogExceeded `json:"on_streaming_backlog_exceeded,omitempty"` OnSuccess []ResourceJobWebhookNotificationsOnSuccess `json:"on_success,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_metastore_data_access.go b/bundle/internal/tf/schema/resource_metastore_data_access.go index 155730055..2e2ff4eb4 100644 --- a/bundle/internal/tf/schema/resource_metastore_data_access.go +++ b/bundle/internal/tf/schema/resource_metastore_data_access.go @@ -37,6 +37,7 @@ type ResourceMetastoreDataAccess struct { ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` IsDefault bool `json:"is_default,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_mws_workspaces.go b/bundle/internal/tf/schema/resource_mws_workspaces.go index 21d1ce428..6c053cb84 100644 --- a/bundle/internal/tf/schema/resource_mws_workspaces.go +++ b/bundle/internal/tf/schema/resource_mws_workspaces.go @@ -43,6 +43,7 @@ type ResourceMwsWorkspaces struct { CustomTags map[string]string `json:"custom_tags,omitempty"` CustomerManagedKeyId string `json:"customer_managed_key_id,omitempty"` DeploymentName string `json:"deployment_name,omitempty"` + GcpWorkspaceSa string `json:"gcp_workspace_sa,omitempty"` Id string `json:"id,omitempty"` IsNoPublicIpEnabled bool `json:"is_no_public_ip_enabled,omitempty"` Location string `json:"location,omitempty"` diff --git a/bundle/internal/tf/schema/resource_online_table.go b/bundle/internal/tf/schema/resource_online_table.go index af8a348d3..de671eade 100644 --- a/bundle/internal/tf/schema/resource_online_table.go +++ b/bundle/internal/tf/schema/resource_online_table.go @@ -19,8 +19,9 @@ type ResourceOnlineTableSpec struct { } type ResourceOnlineTable struct { - Id string `json:"id,omitempty"` - Name string `json:"name"` - Status []any `json:"status,omitempty"` - Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + Status []any `json:"status,omitempty"` + TableServingUrl string `json:"table_serving_url,omitempty"` + Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index 5d8df11e7..ee94a1a8f 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -13,6 +13,7 @@ type ResourcePermissions struct { Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` DirectoryId string `json:"directory_id,omitempty"` DirectoryPath string `json:"directory_path,omitempty"` ExperimentId string `json:"experiment_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_storage_credential.go b/bundle/internal/tf/schema/resource_storage_credential.go index 3d4a501ea..1c62cf8df 100644 --- a/bundle/internal/tf/schema/resource_storage_credential.go +++ b/bundle/internal/tf/schema/resource_storage_credential.go @@ -36,11 +36,13 @@ type ResourceStorageCredential struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` ReadOnly bool `json:"read_only,omitempty"` SkipValidation bool `json:"skip_validation,omitempty"` + StorageCredentialId string `json:"storage_credential_id,omitempty"` AwsIamRole *ResourceStorageCredentialAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceStorageCredentialAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceStorageCredentialAzureServicePrincipal `json:"azure_service_principal,omitempty"` diff --git a/bundle/internal/tf/schema/resource_system_schema.go b/bundle/internal/tf/schema/resource_system_schema.go index 09a86103a..fe5b128d6 100644 --- a/bundle/internal/tf/schema/resource_system_schema.go +++ b/bundle/internal/tf/schema/resource_system_schema.go @@ -3,6 +3,7 @@ package schema type ResourceSystemSchema struct { + FullName string `json:"full_name,omitempty"` Id string `json:"id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Schema string `json:"schema,omitempty"` diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79d71a65f..79c1b32b5 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -16,6 +16,7 @@ type Resources struct { ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` Connection map[string]any `json:"databricks_connection,omitempty"` + Dashboard map[string]any `json:"databricks_dashboard,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -96,6 +97,7 @@ type Resources struct { VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } @@ -115,6 +117,7 @@ func NewResources() *Resources { ClusterPolicy: make(map[string]any), ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), Connection: make(map[string]any), + Dashboard: make(map[string]any), DbfsFile: make(map[string]any), DefaultNamespaceSetting: make(map[string]any), Directory: make(map[string]any), @@ -195,6 +198,7 @@ func NewResources() *Resources { VectorSearchEndpoint: make(map[string]any), VectorSearchIndex: make(map[string]any), Volume: make(map[string]any), + WorkspaceBinding: make(map[string]any), WorkspaceConf: make(map[string]any), WorkspaceFile: make(map[string]any), } From 5afcc25d27b6a7588f3dcf6178fe275eb51cd3f1 Mon Sep 17 00:00:00 2001 From: Cor Date: Wed, 31 Jul 2024 11:35:06 +0200 Subject: [PATCH 36/88] Add upgrade and upgrade eager flags to pip install call (#1636) ## Changes Add upgrade and upgrade eager flags to pip install call for Databricks labs projects. See [this documentation](https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-U) for more information about the flags. Resolves #1634 ## Tests - [x] Manually --- cmd/labs/project/installer.go | 4 +++- cmd/labs/project/installer_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 39ed9a966..041415964 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -272,8 +272,10 @@ func (i *installer) installPythonDependencies(ctx context.Context, spec string) // - python3 -m ensurepip --default-pip // - curl -o https://bootstrap.pypa.io/get-pip.py | python3 var buf bytes.Buffer + // Ensure latest version(s) is installed with the `--upgrade` and `--upgrade-strategy eager` flags + // https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-U _, err := process.Background(ctx, - []string{i.virtualEnvPython(ctx), "-m", "pip", "install", spec}, + []string{i.virtualEnvPython(ctx), "-m", "pip", "install", "--upgrade", "--upgrade-strategy", "eager", spec}, process.WithCombinedOutput(&buf), process.WithDir(libDir)) if err != nil { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index 0e049b4c0..8754a560b 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -199,7 +199,7 @@ func TestInstallerWorksForReleases(t *testing.T) { stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5") // on Unix, we call `python3`, but on Windows it is `python.exe` stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]") - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") // simulate the case of GitHub Actions @@ -406,7 +406,7 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Install stubs for the python calls we need to ensure were run in the // upgrade process. ctx, stub := process.WithStub(ctx) - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") py, _ := python.DetectExecutable(ctx) @@ -430,13 +430,13 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Check if the stub was called with the 'python -m pip install' command pi := false for _, call := range stub.Commands() { - if strings.HasSuffix(call, "-m pip install .") { + if strings.HasSuffix(call, "-m pip install --upgrade --upgrade-strategy eager .") { pi = true break } } if !pi { - t.Logf(`Expected stub command 'python[\S]+ -m pip install .' not found`) + t.Logf(`Expected stub command 'python[\S]+ -m pip install --upgrade --upgrade-strategy eager .' not found`) t.FailNow() } } From 89c0af5bdcee82bc01a27930c86d550402eedc96 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:46:28 +0530 Subject: [PATCH 37/88] Add resource for UC schemas to DABs (#1413) ## Changes This PR adds support for UC Schemas to DABs. This allows users to define schemas for tables and other assets their pipelines/workflows create as part of the DAB, thus managing the life-cycle in the DAB. The first version has a couple of intentional limitations: 1. The owner of the schema will be the deployment user. Changing the owner of the schema is not allowed (yet). `run_as` will not be restricted for DABs containing UC schemas. Let's limit the scope of run_as to the compute identity used instead of ownership of data assets like UC schemas. 2. API fields that are present in the update API but not the create API. For example: enabling predictive optimization is not supported in the create schema API and thus is not available in DABs at the moment. ## Tests Manually and integration test. Manually verified the following work: 1. Development mode adds a "dev_" prefix. 2. Modified status is correctly computed in the `bundle summary` command. 3. Grants work as expected, for assigning privileges. 4. Variable interpolation works for the schema ID. --- bundle/config/mutator/process_target_mode.go | 7 + .../mutator/process_target_mode_test.go | 6 + bundle/config/mutator/run_as_test.go | 2 + bundle/config/resources.go | 1 + bundle/config/resources/schema.go | 27 ++++ bundle/deploy/terraform/convert.go | 21 ++- bundle/deploy/terraform/convert_test.go | 57 ++++++++ bundle/deploy/terraform/interpolate.go | 2 + bundle/deploy/terraform/interpolate_test.go | 2 + .../deploy/terraform/tfdyn/convert_schema.go | 53 ++++++++ .../terraform/tfdyn/convert_schema_test.go | 75 +++++++++++ bundle/phases/deploy.go | 98 ++++++++++++-- cmd/bundle/deploy.go | 5 +- internal/acc/workspace.go | 28 ++++ .../uc_schema/databricks_template_schema.json | 8 ++ .../uc_schema/template/databricks.yml.tmpl | 19 +++ .../bundle/bundles/uc_schema/template/nb.sql | 2 + .../uc_schema/template/schema.yml.tmpl | 13 ++ internal/bundle/deploy_test.go | 125 ++++++++++++++++++ internal/bundle/helpers.go | 2 +- 20 files changed, 540 insertions(+), 13 deletions(-) create mode 100644 bundle/config/resources/schema.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_schema.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_schema_test.go create mode 100644 internal/bundle/bundles/uc_schema/databricks_template_schema.json create mode 100644 internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/uc_schema/template/nb.sql create mode 100644 internal/bundle/bundles/uc_schema/template/schema.yml.tmpl create mode 100644 internal/bundle/deploy_test.go diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index b50716fd6..9db97907d 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -112,6 +112,13 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) diag.Diagno } } + for i := range r.Schemas { + prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" + r.Schemas[i].Name = prefix + r.Schemas[i].Name + // HTTP API for schemas doesn't yet support tags. It's only supported in + // the Databricks UI and via the SQL API. + } + return nil } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 03da64e77..f0c8aee9e 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -114,6 +114,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Schemas: map[string]*resources.Schema{ + "schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}}, + }, }, }, // Use AWS implementation for testing. @@ -167,6 +170,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) assert.Nil(t, b.Config.Resources.QualityMonitors["qualityMonitor2"].Schedule) assert.Equal(t, catalog.MonitorCronSchedulePauseStatusUnpaused, b.Config.Resources.QualityMonitors["qualityMonitor3"].Schedule.PauseStatus) + + // Schema 1 + assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index 67bf7bcc2..e6cef9ba4 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -39,6 +39,7 @@ func allResourceTypes(t *testing.T) []string { "pipelines", "quality_monitors", "registered_models", + "schemas", }, resourceTypes, ) @@ -136,6 +137,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { "models", "registered_models", "experiments", + "schemas", } base := config.Root{ diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 062e38ed5..6c7a927f2 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -18,6 +18,7 @@ type Resources struct { ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` + Schemas map[string]*resources.Schema `json:"schemas,omitempty"` } type resource struct { diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go new file mode 100644 index 000000000..7ab00495a --- /dev/null +++ b/bundle/config/resources/schema.go @@ -0,0 +1,27 @@ +package resources + +import ( + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Schema struct { + // List of grants to apply on this schema. + Grants []Grant `json:"grants,omitempty"` + + // Full name of the schema (catalog_name.schema_name). This value is read from + // the terraform state after deployment succeeds. + ID string `json:"id,omitempty" bundle:"readonly"` + + *catalog.CreateSchema + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` +} + +func (s *Schema) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Schema) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index a6ec04d9a..f13c241ce 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -66,8 +66,10 @@ func convGrants(acl []resources.Grant) *schema.ResourceGrants { // BundleToTerraform converts resources in a bundle configuration // to the equivalent Terraform JSON representation. // -// NOTE: THIS IS CURRENTLY A HACK. WE NEED A BETTER WAY TO -// CONVERT TO/FROM TERRAFORM COMPATIBLE FORMAT. +// Note: This function is an older implementation of the conversion logic. It is +// no longer used in any code paths. It is kept around to be used in tests. +// New resources do not need to modify this function and can instead can define +// the conversion login in the tfdyn package. func BundleToTerraform(config *config.Root) *schema.Root { tfroot := schema.NewRoot() tfroot.Provider = schema.NewProviders() @@ -382,6 +384,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.QualityMonitors[resource.Name] = cur + case "databricks_schema": + if config.Resources.Schemas == nil { + config.Resources.Schemas = make(map[string]*resources.Schema) + } + cur := config.Resources.Schemas[resource.Name] + if cur == nil { + cur = &resources.Schema{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Schemas[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -426,6 +438,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Schemas { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 7ea448538..e4ef6114a 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -655,6 +655,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -681,6 +689,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -736,6 +747,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + }, }, } var tfState = resourcesState{ @@ -765,6 +783,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -855,6 +876,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + "test_schema_new": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -971,6 +1004,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "test_monitor_old"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1024,6 +1073,14 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus) assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Schemas["test_schema_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 608f1c795..faa098e1c 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -56,6 +56,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...) case dyn.Key("quality_monitors"): path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) + case dyn.Key("schemas"): + path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 9af4a1443..5ceb243bc 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -30,6 +30,7 @@ func TestInterpolate(t *testing.T) { "other_experiment": "${resources.experiments.other_experiment.id}", "other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}", "other_registered_model": "${resources.registered_models.other_registered_model.id}", + "other_schema": "${resources.schemas.other_schema.id}", }, Tasks: []jobs.Task{ { @@ -65,6 +66,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_mlflow_experiment.other_experiment.id}", j.Tags["other_experiment"]) assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"]) assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) + assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/tfdyn/convert_schema.go b/bundle/deploy/terraform/tfdyn/convert_schema.go new file mode 100644 index 000000000..b5e6a88c0 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema.go @@ -0,0 +1,53 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertSchemaResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + v, diags := convert.Normalize(schema.ResourceSchema{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "schema normalization diagnostic: %s", diag.Summary) + } + + // We always set force destroy as it allows DABs to manage the lifecycle + // of the schema. It's the responsibility of the CLI to ensure the user + // is adequately warned when they try to delete a UC schema. + vout, err := dyn.SetByPath(v, dyn.MustPathFromString("force_destroy"), dyn.V(true)) + if err != nil { + return dyn.InvalidValue, err + } + + return vout, nil +} + +type schemaConverter struct{} + +func (schemaConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertSchemaResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Schema[key] = vout.AsAny() + + // Configure grants for this resource. + if grants := convertGrantsResource(ctx, vin); grants != nil { + grants.Schema = fmt.Sprintf("${databricks_schema.%s.id}", key) + out.Grants["schema_"+key] = grants + } + + return nil +} + +func init() { + registerConverter("schemas", schemaConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_schema_test.go b/bundle/deploy/terraform/tfdyn/convert_schema_test.go new file mode 100644 index 000000000..2efbf3e43 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema_test.go @@ -0,0 +1,75 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertSchema(t *testing.T) { + var src = resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "name", + CatalogName: "catalog", + Comment: "comment", + Properties: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + StorageRoot: "root", + }, + Grants: []resources.Grant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = schemaConverter{}.Convert(ctx, "my_schema", vin, out) + require.NoError(t, err) + + // Assert equality on the schema + assert.Equal(t, map[string]any{ + "name": "name", + "catalog_name": "catalog", + "comment": "comment", + "properties": map[string]any{ + "k1": "v1", + "k2": "v2", + }, + "force_destroy": true, + "storage_root": "root", + }, out.Schema["my_schema"]) + + // Assert equality on the grants + assert.Equal(t, &schema.ResourceGrants{ + Schema: "${databricks_schema.my_schema.id}", + Grant: []schema.ResourceGrantsGrant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + }, out.Grants["schema_my_schema"]) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 46c389189..c68153f2d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -1,6 +1,9 @@ package phases import ( + "context" + "fmt" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" @@ -14,10 +17,91 @@ import ( "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/libs/cmdio" + terraformlib "github.com/databricks/cli/libs/terraform" ) +func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + actions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + // We only care about destructive actions on UC schema resources. + if rc.Type != "databricks_schema" { + continue + } + + var actionType terraformlib.ActionType + + switch { + case rc.Change.Actions.Delete(): + actionType = terraformlib.ActionTypeDelete + case rc.Change.Actions.Replace(): + actionType = terraformlib.ActionTypeRecreate + default: + // We don't need a prompt for non-destructive actions like creating + // or updating a schema. + continue + } + + actions = append(actions, terraformlib.Action{ + Action: actionType, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + + // No restricted actions planned. No need for approval. + if len(actions) == 0 { + return true, nil + } + + cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") + for _, action := range actions { + cmdio.Log(ctx, action) + } + + if b.AutoApprove { + return true, nil + } + + if !cmdio.IsPromptSupported(ctx) { + return false, fmt.Errorf("the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") + } + + cmdio.LogString(ctx, "") + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The deploy phase deploys artifacts and resources. func Deploy() bundle.Mutator { + // Core mutators that CRUD resources and modify deployment state. These + // mutators need informed consent if they are potentially destructive. + deployCore := bundle.Defer( + terraform.Apply(), + bundle.Seq( + terraform.StatePush(), + terraform.Load(), + metadata.Compute(), + metadata.Upload(), + bundle.LogString("Deployment complete!"), + ), + ) + deployMutator := bundle.Seq( scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), @@ -37,20 +121,16 @@ func Deploy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.CheckRunningResource(), - bundle.Defer( - terraform.Apply(), - bundle.Seq( - terraform.StatePush(), - terraform.Load(), - metadata.Compute(), - metadata.Upload(), - ), + terraform.Plan(terraform.PlanGoal("deploy")), + bundle.If( + approvalForUcSchemaDelete, + deployCore, + bundle.LogString("Deployment cancelled!"), ), ), lock.Release(lock.GoalDeploy), ), scripts.Execute(config.ScriptPostDeploy), - bundle.LogString("Deployment complete!"), ) return newPhase( diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 1232c8de5..1166875ab 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -24,10 +24,12 @@ func newDeployCommand() *cobra.Command { var forceLock bool var failOnActiveRuns bool var computeID string + var autoApprove bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") + cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -37,10 +39,11 @@ func newDeployCommand() *cobra.Command { bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { b.Config.Bundle.Force = force b.Config.Bundle.Deployment.Lock.Force = forceLock + b.AutoApprove = autoApprove + if cmd.Flag("compute-id").Changed { b.Config.Bundle.ComputeID = computeID } - if cmd.Flag("fail-on-active-runs").Changed { b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns } diff --git a/internal/acc/workspace.go b/internal/acc/workspace.go index 8944e199f..39374f229 100644 --- a/internal/acc/workspace.go +++ b/internal/acc/workspace.go @@ -2,6 +2,7 @@ package acc import ( "context" + "os" "testing" "github.com/databricks/databricks-sdk-go" @@ -38,6 +39,33 @@ func WorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { return wt.ctx, wt } +// Run the workspace test only on UC workspaces. +func UcWorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { + loadDebugEnvIfRunFromIDE(t, "workspace") + + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skipf("Skipping on non-UC workspaces") + } + if os.Getenv("DATABRICKS_ACCOUNT_ID") != "" { + t.Skipf("Skipping on accounts") + } + + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + wt := &WorkspaceT{ + T: t, + + W: w, + + ctx: context.Background(), + } + + return wt.ctx, wt +} + func (t *WorkspaceT) TestClusterID() string { clusterID := GetEnvOrSkipTest(t.T, "TEST_BRICKS_CLUSTER_ID") err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID) diff --git a/internal/bundle/bundles/uc_schema/databricks_template_schema.json b/internal/bundle/bundles/uc_schema/databricks_template_schema.json new file mode 100644 index 000000000..762f4470c --- /dev/null +++ b/internal/bundle/bundles/uc_schema/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema and pipeline names" + } + } +} diff --git a/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl new file mode 100644 index 000000000..961af25e8 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: "bundle-playground" + +resources: + pipelines: + foo: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + +include: + - "*.yml" + +targets: + development: + default: true diff --git a/internal/bundle/bundles/uc_schema/template/nb.sql b/internal/bundle/bundles/uc_schema/template/nb.sql new file mode 100644 index 000000000..199ff5078 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl new file mode 100644 index 000000000..50067036e --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl @@ -0,0 +1,13 @@ +resources: + schemas: + bar: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created from DABs + +targets: + development: + resources: + pipelines: + foo: + target: ${resources.schemas.bar.id} diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go new file mode 100644 index 000000000..3da885705 --- /dev/null +++ b/internal/bundle/deploy_test.go @@ -0,0 +1,125 @@ +package bundle + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/files" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupUcSchemaBundle(t *testing.T, ctx context.Context, w *databricks.WorkspaceClient, uniqueId string) string { + bundleRoot, err := initTestTemplate(t, ctx, "uc_schema", map[string]any{ + "unique_id": uniqueId, + }) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + // Assert the schema is created + catalogName := "main" + schemaName := "test-schema-" + uniqueId + schema, err := w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + require.NoError(t, err) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), schema.FullName) + require.Equal(t, "This schema was created from DABs", schema.Comment) + + // Assert the pipeline is created + pipelineName := "test-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + require.Equal(t, pipelineName, pipeline.Name) + id := pipeline.PipelineId + + // Assert the pipeline uses the schema + i, err := w.Pipelines.GetByPipelineId(ctx, id) + require.NoError(t, err) + require.Equal(t, catalogName, i.Spec.Catalog) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), i.Spec.Target) + + // Create a volume in the schema, and add a file to it. This ensures that the + // schema has some data in it and deletion will fail unless the generated + // terraform configuration has force_destroy set to true. + volumeName := "test-volume-" + uniqueId + volume, err := w.Volumes.Create(ctx, catalog.CreateVolumeRequestContent{ + CatalogName: catalogName, + SchemaName: schemaName, + Name: volumeName, + VolumeType: catalog.VolumeTypeManaged, + }) + require.NoError(t, err) + require.Equal(t, volume.Name, volumeName) + + fileName := "test-file-" + uniqueId + err = w.Files.Upload(ctx, files.UploadRequest{ + Contents: io.NopCloser(strings.NewReader("Hello, world!")), + FilePath: fmt.Sprintf("/Volumes/%s/%s/%s/%s", catalogName, schemaName, volumeName, fileName), + }) + require.NoError(t, err) + + return bundleRoot +} + +func TestAccBundleDeployUcSchema(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + schemaName := "test-schema-" + uniqueId + catalogName := "main" + + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + // Assert the schema is deleted + _, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + apiErr := &apierr.APIError{} + assert.True(t, errors.As(err, &apiErr)) + assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode) +} + +func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + stdout, _, err := c.Run() + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index c33c15331..1910a0148 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -64,7 +64,7 @@ func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, err func deployBundle(t *testing.T, ctx context.Context, path string) error { t.Setenv("BUNDLE_ROOT", path) - c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve") _, _, err := c.Run() return err } From 1fb8e324d5646ce3320ea4c64d79de332f17c53b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 31 Jul 2024 15:42:23 +0200 Subject: [PATCH 38/88] Added test for negation pattern in sync include exclude section (#1637) ## Changes Added test for negation pattern in sync include exclude section --- .../config/validate/validate_sync_patterns.go | 11 +- bundle/tests/sync/negate/databricks.yml | 22 ++++ bundle/tests/sync/negate/test.txt | 0 bundle/tests/sync/negate/test.yml | 0 .../sync_include_exclude_no_matches_test.go | 19 ++++ libs/sync/sync_test.go | 107 ++++++++---------- 6 files changed, 100 insertions(+), 59 deletions(-) create mode 100644 bundle/tests/sync/negate/databricks.yml create mode 100644 bundle/tests/sync/negate/test.txt create mode 100644 bundle/tests/sync/negate/test.yml diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 573077b66..fd011bf78 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -3,6 +3,7 @@ package validate import ( "context" "fmt" + "strings" "sync" "github.com/databricks/cli/bundle" @@ -49,7 +50,13 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di for i, pattern := range patterns { index := i - p := pattern + fullPattern := pattern + // If the pattern is negated, strip the negation prefix + // and check if the pattern matches any files. + // Negation in gitignore syntax means "don't look at this path' + // So if p matches nothing it's useless negation, but if there are matches, + // it means: do not include these files into result set + p := strings.TrimPrefix(fullPattern, "!") errs.Go(func() error { fs, err := fileset.NewGlobSet(rb.BundleRoot(), []string{p}) if err != nil { @@ -66,7 +73,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di mu.Lock() diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, - Summary: fmt.Sprintf("Pattern %s does not match any files", p), + Summary: fmt.Sprintf("Pattern %s does not match any files", fullPattern), Locations: []dyn.Location{loc.Location()}, Paths: []dyn.Path{loc.Path()}, }) diff --git a/bundle/tests/sync/negate/databricks.yml b/bundle/tests/sync/negate/databricks.yml new file mode 100644 index 000000000..3d591d19b --- /dev/null +++ b/bundle/tests/sync/negate/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: sync_negate + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + exclude: + - ./* + - '!*.txt' + include: + - '*.txt' + +targets: + default: + dev: + sync: + exclude: + - ./* + - '!*.txt2' + include: + - '*.txt' diff --git a/bundle/tests/sync/negate/test.txt b/bundle/tests/sync/negate/test.txt new file mode 100644 index 000000000..e69de29bb diff --git a/bundle/tests/sync/negate/test.yml b/bundle/tests/sync/negate/test.yml new file mode 100644 index 000000000..e69de29bb diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 23f99b3a7..0192b61e6 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -42,3 +42,22 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[2].Severity, diag.Warning) require.Contains(t, summaries, diags[2].Summary) } + +func TestSyncIncludeWithNegate(t *testing.T) { + b := loadTarget(t, "./sync/negate", "default") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 0) + require.NoError(t, diags.Error()) +} + +func TestSyncIncludeWithNegateNoMatches(t *testing.T) { + b := loadTarget(t, "./sync/negate", "dev") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 1) + require.NoError(t, diags.Error()) + + require.Equal(t, diags[0].Severity, diag.Warning) + require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files") +} diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index 292586e8d..2d800f466 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -2,70 +2,32 @@ package sync import ( "context" - "os" - "path/filepath" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) -func createFile(dir string, name string) error { - f, err := os.Create(filepath.Join(dir, name)) - if err != nil { - return err - } - - return f.Close() -} - func setupFiles(t *testing.T) string { dir := t.TempDir() - err := createFile(dir, "a.go") - require.NoError(t, err) - - err = createFile(dir, "b.go") - require.NoError(t, err) - - err = createFile(dir, "ab.go") - require.NoError(t, err) - - err = createFile(dir, "abc.go") - require.NoError(t, err) - - err = createFile(dir, "c.go") - require.NoError(t, err) - - err = createFile(dir, "d.go") - require.NoError(t, err) - - dbDir := filepath.Join(dir, ".databricks") - err = os.Mkdir(dbDir, 0755) - require.NoError(t, err) - - err = createFile(dbDir, "e.go") - require.NoError(t, err) - - testDir := filepath.Join(dir, "test") - err = os.Mkdir(testDir, 0755) - require.NoError(t, err) - - sub1 := filepath.Join(testDir, "sub1") - err = os.Mkdir(sub1, 0755) - require.NoError(t, err) - - err = createFile(sub1, "f.go") - require.NoError(t, err) - - sub2 := filepath.Join(sub1, "sub2") - err = os.Mkdir(sub2, 0755) - require.NoError(t, err) - - err = createFile(sub2, "g.go") - require.NoError(t, err) + for _, f := range []([]string){ + []string{dir, "a.go"}, + []string{dir, "b.go"}, + []string{dir, "ab.go"}, + []string{dir, "abc.go"}, + []string{dir, "c.go"}, + []string{dir, "d.go"}, + []string{dir, ".databricks", "e.go"}, + []string{dir, "test", "sub1", "f.go"}, + []string{dir, "test", "sub1", "sub2", "g.go"}, + []string{dir, "test", "sub1", "sub2", "h.txt"}, + } { + testutil.Touch(t, f...) + } return dir } @@ -97,7 +59,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 9) + require.Equal(t, len(fileList), 10) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -115,9 +77,9 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 1) + require.Equal(t, len(fileList), 2) - inc, err = fileset.NewGlobSet(root, []string{".databricks/*"}) + inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) excl, err = fileset.NewGlobSet(root, []string{}) @@ -133,7 +95,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 10) + require.Equal(t, len(fileList), 11) } func TestRecursiveExclude(t *testing.T) { @@ -165,3 +127,34 @@ func TestRecursiveExclude(t *testing.T) { require.NoError(t, err) require.Equal(t, len(fileList), 7) } + +func TestNegateExclude(t *testing.T) { + ctx := context.Background() + + dir := setupFiles(t) + root := vfs.MustNew(dir) + fileSet, err := git.NewFileSet(root) + require.NoError(t, err) + + err = fileSet.EnsureValidGitIgnoreExists() + require.NoError(t, err) + + inc, err := fileset.NewGlobSet(root, []string{}) + require.NoError(t, err) + + excl, err := fileset.NewGlobSet(root, []string{"./*", "!*.txt"}) + require.NoError(t, err) + + s := &Sync{ + SyncOptions: &SyncOptions{}, + + fileSet: fileSet, + includeFileSet: inc, + excludeFileSet: excl, + } + + fileList, err := s.GetFileList(ctx) + require.NoError(t, err) + require.Equal(t, len(fileList), 1) + require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt") +} From c454c2fd10534abe666c879b23de3859b738af79 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 31 Jul 2024 19:37:25 +0530 Subject: [PATCH 39/88] Use precomputed terraform plan for `bundle deploy` (#1640) # Changes With https://github.com/databricks/cli/pull/1413 we started to compute and partially print the plan if it contained deletion of UC schemas. This PR uses the precomputed plan to avoid double planning when actually doing the terraform plan. This fixes a performance regression introduced in https://github.com/databricks/cli/pull/1413. # Tests Tested manually. 1. Verified bundle deployment still works and deploys resources. 2. Verified that the precomputed plan is indeed being used by attaching a debugger and removing the plan file right before the terraform apply process is spawned and asserting that terraform apply fails because the plan is not found. --- bundle/deploy/terraform/apply.go | 21 ++++++------ bundle/deploy/terraform/destroy.go | 46 --------------------------- bundle/deploy/terraform/state_push.go | 8 +++++ bundle/phases/deploy.go | 5 ++- bundle/phases/destroy.go | 2 +- 5 files changed, 25 insertions(+), 57 deletions(-) delete mode 100644 bundle/deploy/terraform/destroy.go diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index e4acda852..e52d0ca8f 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" @@ -17,28 +16,32 @@ func (w *apply) Name() string { } func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // return early if plan is empty + if b.Plan.IsEmpty { + log.Debugf(ctx, "No changes in plan. Skipping terraform apply.") + return nil + } + tf := b.Terraform if tf == nil { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Deploying resources...") - - err := tf.Init(ctx, tfexec.Upgrade(true)) - if err != nil { - return diag.Errorf("terraform init: %v", err) + if b.Plan.Path == "" { + return diag.Errorf("no plan found") } - err = tf.Apply(ctx) + // Apply terraform according to the computed plan + err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { return diag.Errorf("terraform apply: %v", err) } - log.Infof(ctx, "Resource deployment completed") + log.Infof(ctx, "terraform apply completed") return nil } -// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply` +// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply ./plan` // from the bundle's ephemeral working directory for Terraform. func Apply() bundle.Mutator { return &apply{} diff --git a/bundle/deploy/terraform/destroy.go b/bundle/deploy/terraform/destroy.go deleted file mode 100644 index 9c63a0b37..000000000 --- a/bundle/deploy/terraform/destroy.go +++ /dev/null @@ -1,46 +0,0 @@ -package terraform - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" - "github.com/hashicorp/terraform-exec/tfexec" -) - -type destroy struct{} - -func (w *destroy) Name() string { - return "terraform.Destroy" -} - -func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // return early if plan is empty - if b.Plan.IsEmpty { - log.Debugf(ctx, "No resources to destroy in plan. Skipping destroy.") - return nil - } - - tf := b.Terraform - if tf == nil { - return diag.Errorf("terraform not initialized") - } - - if b.Plan.Path == "" { - return diag.Errorf("no plan found") - } - - // Apply terraform according to the computed destroy plan - err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) - if err != nil { - return diag.Errorf("terraform destroy: %v", err) - } - return nil -} - -// Destroy returns a [bundle.Mutator] that runs the conceptual equivalent of -// `terraform destroy ./plan` from the bundle's ephemeral working directory for Terraform. -func Destroy() bundle.Mutator { - return &destroy{} -} diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index b50983bd4..6cdde1371 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -2,6 +2,8 @@ package terraform import ( "context" + "errors" + "io/fs" "os" "path/filepath" @@ -34,6 +36,12 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic // Expect the state file to live under dir. local, err := os.Open(filepath.Join(dir, TerraformStateFileName)) + if errors.Is(err, fs.ErrNotExist) { + // The state file can be absent if terraform apply is skipped because + // there are no changes to apply in the plan. + log.Debugf(ctx, "Local terraform state file does not exist.") + return nil + } if err != nil { return diag.FromErr(err) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index c68153f2d..6929f74ba 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -92,7 +92,10 @@ func Deploy() bundle.Mutator { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. deployCore := bundle.Defer( - terraform.Apply(), + bundle.Seq( + bundle.LogString("Deploying resources..."), + terraform.Apply(), + ), bundle.Seq( terraform.StatePush(), terraform.Load(), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index bd99af789..01b276670 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -82,7 +82,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) { func Destroy() bundle.Mutator { // Core destructive mutators for destroy. These require informed user consent. destroyCore := bundle.Seq( - terraform.Destroy(), + terraform.Apply(), terraform.StatePush(), files.Delete(), bundle.LogString("Destroy complete!"), From 630a56e41e4489a4d5cd56444f716cda89fa224e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 31 Jul 2024 18:47:00 +0200 Subject: [PATCH 40/88] Release v0.225.0 (#1642) Bundles: * Add resource for UC schemas to DABs ([#1413](https://github.com/databricks/cli/pull/1413)). Internal: * Use dynamic walking to validate unique resource keys ([#1614](https://github.com/databricks/cli/pull/1614)). * Regenerate TF schema ([#1635](https://github.com/databricks/cli/pull/1635)). * Add upgrade and upgrade eager flags to pip install call ([#1636](https://github.com/databricks/cli/pull/1636)). * Added test for negation pattern in sync include exclude section ([#1637](https://github.com/databricks/cli/pull/1637)). * Use precomputed terraform plan for `bundle deploy` ([#1640](https://github.com/databricks/cli/pull/1640)). --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b272377ec..d1e0b9a5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Version changelog +## 0.225.0 + +Bundles: + * Add resource for UC schemas to DABs ([#1413](https://github.com/databricks/cli/pull/1413)). + +Internal: + * Use dynamic walking to validate unique resource keys ([#1614](https://github.com/databricks/cli/pull/1614)). + * Regenerate TF schema ([#1635](https://github.com/databricks/cli/pull/1635)). + * Add upgrade and upgrade eager flags to pip install call ([#1636](https://github.com/databricks/cli/pull/1636)). + * Added test for negation pattern in sync include exclude section ([#1637](https://github.com/databricks/cli/pull/1637)). + * Use precomputed terraform plan for `bundle deploy` ([#1640](https://github.com/databricks/cli/pull/1640)). + ## 0.224.1 Bundles: From a13d77f8eb29a6c7587509721217a137039f20d6 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:25:22 +0530 Subject: [PATCH 41/88] Fix python wheel task integration tests (#1648) ## Changes A new Service Control Policy has removed the `ec2.RunInstances` permission from our service principal for our AWS integration tests. This PR switches over to using the instance pool which does not require creating new clusters. ## Tests The integration tests pass now. --- .../bundles/python_wheel_task/databricks_template_schema.json | 4 ++++ .../bundles/python_wheel_task/template/databricks.yml.tmpl | 1 + internal/bundle/python_wheel_test.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json index 0695eb2ba..c4a74df07 100644 --- a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json +++ b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json @@ -20,6 +20,10 @@ "python_wheel_wrapper": { "type": "boolean", "description": "Whether or not to enable python wheel wrapper" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" } } } diff --git a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl index 8729dcba5..30b0a5eae 100644 --- a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl @@ -20,6 +20,7 @@ resources: spark_version: "{{.spark_version}}" node_type_id: "{{.node_type_id}}" data_security_mode: USER_ISOLATION + instance_pool_id: "{{.instance_pool_id}}" python_wheel_task: package_name: my_test_code entry_point: run diff --git a/internal/bundle/python_wheel_test.go b/internal/bundle/python_wheel_test.go index bf2462920..ed98efecd 100644 --- a/internal/bundle/python_wheel_test.go +++ b/internal/bundle/python_wheel_test.go @@ -14,11 +14,13 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo ctx, _ := acc.WorkspaceTest(t) nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{ "node_type_id": nodeTypeId, "unique_id": uuid.New().String(), "spark_version": sparkVersion, "python_wheel_wrapper": pythonWheelWrapper, + "instance_pool_id": instancePoolId, }) require.NoError(t, err) From ff4f0bb0fe3468e709948e795bb18c1daddea62b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:10:51 +0200 Subject: [PATCH 42/88] Bump github.com/hashicorp/hc-install from 0.7.0 to 0.8.0 (#1652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hashicorp/hc-install](https://github.com/hashicorp/hc-install) from 0.7.0 to 0.8.0.
Release notes

Sourced from github.com/hashicorp/hc-install's releases.

v0.8.0

ENHANCEMENTS:

BUG FIXES:

INTERNAL:

Commits
  • 6a754fc Update VERSION
  • b216d7f [fix] include custom url's "path" when creating Archive URL (#234)
  • 5efb089 build(deps): Bump workflows to latest trusted versions (#233)
  • 0c03a35 build(deps): Bump workflows to latest trusted versions (#231)
  • 321faf4 build(deps): bump golang.org/x/mod from 0.18.0 to 0.19.0 (#229)
  • 3f6f9f2 go: bump version to 1.22.4 (#227)
  • 2597d9e build(deps): Bump workflows to latest trusted versions (#226)
  • c4aaa60 build(deps): bump hashicorp/actions-packaging-linux from 1.7 to 1.8 in the gi...
  • 03e0bd6 build(deps): bump hashicorp/action-setup-bob from 2.0.3 to 2.1.0 in the githu...
  • f847221 Merge pull request #223 from hashicorp/dependabot/go_modules/golang.org/x/mod...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/hc-install&package-manager=go_modules&previous-version=0.7.0&new-version=0.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 ++- go.sum | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5e29d295e..3ec5a032e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.8.0 // MPL 2.0 github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause @@ -49,6 +49,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8f774a47a..c9af760f9 100644 --- a/go.sum +++ b/go.sum @@ -99,10 +99,14 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= -github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/hc-install v0.8.0 h1:LdpZeXkZYMQhoKPCecJHlKvUkQFixN/nvyR1CdfOLjI= +github.com/hashicorp/hc-install v0.8.0/go.mod h1:+MwJYjDfCruSD/udvBmRB22Nlkwwkwf5sAB6uTIhSaU= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= From 1c8023967224fe29e0a9530b14ef538fa27fdc4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:11:07 +0200 Subject: [PATCH 43/88] Bump golang.org/x/sync from 0.7.0 to 0.8.0 (#1655) Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.7.0 to 0.8.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/sync&package-manager=go_modules&previous-version=0.7.0&new-version=0.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3ec5a032e..0f4a9b515 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.19.0 golang.org/x/oauth2 v0.21.0 - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.8.0 golang.org/x/term v0.22.0 golang.org/x/text v0.16.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 diff --git a/go.sum b/go.sum index c9af760f9..a47fba29d 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,8 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 245d7e3aeed586df1b5afc124dcde7dd0467033f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 14:11:20 +0200 Subject: [PATCH 44/88] Bump golang.org/x/mod from 0.19.0 to 0.20.0 (#1654) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.19.0 to 0.20.0.
Commits
  • bc151c4 README: fix link to x/tools
  • d1f873e modfile: fix Cleanup clobbering Line reference
  • b56a28f modfile: Add support for tool lines
  • 79169e9 LICENSE: update per Google Legal
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.19.0&new-version=0.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0f4a9b515..897041ee8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.19.0 + golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.22.0 diff --git a/go.sum b/go.sum index a47fba29d..63fbc0d9e 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 3bc68e9dd2c6beaee624de4f333d53094804a524 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:54:22 +0530 Subject: [PATCH 45/88] Clarify file format required for the `config-file` flag in `bundle init` (#1651) --- cmd/bundle/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index c25391577..7f2c0efc5 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -148,7 +148,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf var templateDir string var tag string var branch string - cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") + cmd.Flags().StringVar(&configFile, "config-file", "", "JSON file containing key value pairs of input parameters required for template initialization.") cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory path within a Git repository containing the template.") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.") cmd.Flags().StringVar(&branch, "tag", "", "Git tag to use for template initialization") From ed4411f1a624d5143fefb88e22b67a347d1fd961 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:25:59 +0000 Subject: [PATCH 46/88] Bump golang.org/x/oauth2 from 0.21.0 to 0.22.0 (#1653) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.21.0 to 0.22.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 897041ee8..24064c9e1 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.20.0 - golang.org/x/oauth2 v0.21.0 + golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.22.0 golang.org/x/text v0.16.0 diff --git a/go.sum b/go.sum index 63fbc0d9e..bc8fd5f29 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 809c67b675fac5b8f8fc312cccca667929f94267 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 5 Aug 2024 16:44:23 +0200 Subject: [PATCH 47/88] Expand and upload local wheel libraries for all task types (#1649) ## Changes Fixes #1553 ## Tests Added regression test --- bundle/artifacts/whl/autodetect.go | 6 ++--- bundle/artifacts/whl/from_libraries.go | 7 +++++- bundle/libraries/libraries.go | 22 ++++++++----------- bundle/python/warning.go | 2 +- .../.gitignore | 3 +++ .../bundle.yml | 14 ++++++++++++ .../my_test_code/__init__.py | 2 ++ .../my_test_code/__main__.py | 16 ++++++++++++++ .../notebook.py | 3 +++ .../setup.py | 15 +++++++++++++ bundle/tests/python_wheel_test.go | 17 ++++++++++++++ 11 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py create mode 100644 bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index ee77fff01..1601767f6 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -27,9 +27,9 @@ func (m *detectPkg) Name() string { } func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - wheelTasks := libraries.FindAllWheelTasksWithLocalLibraries(b) - if len(wheelTasks) == 0 { - log.Infof(ctx, "No local wheel tasks in databricks.yml config, skipping auto detect") + tasks := libraries.FindTasksWithLocalLibraries(b) + if len(tasks) == 0 { + log.Infof(ctx, "No local tasks in databricks.yml config, skipping auto detect") return nil } log.Infof(ctx, "Detecting Python wheel project...") diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index ad321557c..79161a82d 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -27,8 +27,13 @@ func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return nil } - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) + tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { + // Skip tasks that are not PythonWheelTasks for now, we can later support Jars too + if task.PythonWheelTask == nil { + continue + } + for _, lib := range task.Libraries { matchAndAdd(ctx, lib.Whl, b) } diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 84ead052b..72e5bcc66 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -44,29 +44,25 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { return false } -func FindAllWheelTasksWithLocalLibraries(b *bundle.Bundle) []*jobs.Task { +func FindTasksWithLocalLibraries(b *bundle.Bundle) []jobs.Task { tasks := findAllTasks(b) envs := FindAllEnvironments(b) - wheelTasks := make([]*jobs.Task, 0) + allTasks := make([]jobs.Task, 0) for k, jobTasks := range tasks { for i := range jobTasks { - task := &jobTasks[i] - if task.PythonWheelTask == nil { - continue + task := jobTasks[i] + if isTaskWithLocalLibraries(task) { + allTasks = append(allTasks, task) } + } - if isTaskWithLocalLibraries(*task) { - wheelTasks = append(wheelTasks, task) - } - - if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { - wheelTasks = append(wheelTasks, task) - } + if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { + allTasks = append(allTasks, jobTasks...) } } - return wheelTasks + return allTasks } func isTaskWithLocalLibraries(task jobs.Task) bool { diff --git a/bundle/python/warning.go b/bundle/python/warning.go index 3da88b0d7..d53796d73 100644 --- a/bundle/python/warning.go +++ b/bundle/python/warning.go @@ -35,7 +35,7 @@ func isPythonWheelWrapperOn(b *bundle.Bundle) bool { } func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) + tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { if task.NewCluster != nil { if lowerThanExpectedVersion(ctx, task.NewCluster.SparkVersion) { diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml new file mode 100644 index 000000000..93e4e6918 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml @@ -0,0 +1,14 @@ +bundle: + name: python-wheel-notebook + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + notebook_task: + notebook_path: "/notebook.py" + libraries: + - whl: ./dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py new file mode 100644 index 000000000..909f1f322 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py new file mode 100644 index 000000000..73d045afb --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py new file mode 100644 index 000000000..24dc150ff --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py @@ -0,0 +1,3 @@ +# Databricks notebook source + +print("Hello, World!") diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py new file mode 100644 index 000000000..7a1317b2f --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group_1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 52b3d6e07..53c6764ea 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -45,6 +45,23 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { require.NoError(t, diags.Error()) } +func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") + require.NoError(t, err) + require.Equal(t, 1, len(matches)) + + match := libraries.ValidateLocalLibrariesExist() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + func TestPythonWheelWithDBFSLib(t *testing.T) { ctx := context.Background() b, err := bundle.Load(ctx, "./python_wheel/python_wheel_dbfs_lib") From d26f3f4863bfe7b4532a31a67f33863226529045 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 6 Aug 2024 11:54:58 +0200 Subject: [PATCH 48/88] Fixed incorrectly cleaning up python wheel dist folder (#1656) ## Changes In https://github.com/databricks/cli/pull/1618 we introduced prepare step in which Python wheel folder was cleaned. Now it was cleaned everytime instead of only when there is a build command how it is used to work. This PR fixes it by only cleaning up dist folder when there is a build command for wheels. Fixes #1638 ## Tests Added regression test --- bundle/artifacts/whl/prepare.go | 5 +++++ .../python_wheel_no_build/.gitignore | 3 +++ .../python_wheel_no_build/bundle.yml | 16 ++++++++++++++++ .../lib/my_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1832 bytes .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1832 bytes bundle/tests/python_wheel_test.go | 13 +++++++++++++ 6 files changed, 37 insertions(+) create mode 100644 bundle/tests/python_wheel/python_wheel_no_build/.gitignore create mode 100644 bundle/tests/python_wheel/python_wheel_no_build/bundle.yml create mode 100644 bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl create mode 100644 bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/artifacts/whl/prepare.go b/bundle/artifacts/whl/prepare.go index 7284b11ec..0fbb2080a 100644 --- a/bundle/artifacts/whl/prepare.go +++ b/bundle/artifacts/whl/prepare.go @@ -32,6 +32,11 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diag.Errorf("artifact doesn't exist: %s", m.name) } + // If there is no build command for the artifact, we don't need to cleanup the dist folder before + if artifact.BuildCommand == "" { + return nil + } + dir := artifact.Path distPath := filepath.Join(dir, "dist") diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml new file mode 100644 index 000000000..91b8b1556 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml @@ -0,0 +1,16 @@ +bundle: + name: python-wheel + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./dist/*.whl + - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..4bb80477caf51393354453b18c0c88711098825e GIT binary patch literal 1832 zcmWIWW@Zs#U|`^2xLh&ryk0?NcfY>3uGYCT z=ezsEH-rWlT`<029P+5E-(N@fdGY9=0ryhVJ$tvvUNQc+{d3{D zn|B2?ro>EK#LF7E*yLqccPAsi^Am}D%e30AoeArVlYJ((&Chx9t8w97&hq5+iY(pM z?WKI0`Ng-xP8DwL5?S^qs;1iP{ao7%o!XY-3yZ&e5Ut|$zxe;x^oV^&AD++Fd7iem zR(X?DP}|ALQzYCuG|lZak1|#q*u;_lsY}J+e|O){(NT&M^DqET*~S;Ni)SUsN4C z9Dn1{?j=1*vsGR>2Ap1Urdi@ME2@w3r*1Nd0r~`ll?eGL+{4w?hx@#T@7asIkTkJ5 z$l!|cgChU4-nw2oC-pZ4d3c@F(d*P_Um1A8;DWK?CF2WExK5qdzxY+>8n1?~*V&Uk zn}RenwS1p)dHQS*(pX_~?d<9E-dBCktbX!{2`Tk}Lc4^`-&YapE?q)yOU)}Os*Eql z&&(?+)+?zf>Gsnum6bPz^n=Yu)|leb1cf?%&Q77=vb)*4}*!*?@Mi z0I?P!yGrs4;&W2VQgc8SY3q4CLw9*{w)FQ{Mg|65CI$v&LZ$_|I{OE?w4OU%)MUWm zaB=T_A&mrw*ph>*mAaB@F8Q(@M!#{WU5W^A%QRiSMW z{ozJ=^Fq0UM|XJ53fr~pZf|;Euh=Z{Z-Tx4a;F>EwmRNScHD8mvs7)8lJ+L8`I_QF zshMAec0^w`-2TiwZ^_?(T*a4Um+uz%9hJHAo6n-@<%iYQ%+*hCy{@^uvZID|#pH|s ze;h2ndvtru#jke9t4>^Ne8ji;gOB{FCkgtet-sDum^X=S@}n%?$yyAP)vvd7E<0Z8 zI`hG%7f1XLTdv`o%)7mde~0q)?pa+LTXvaz|5|P@sQzMo+2)nERD@WZf7CmNnCf|W zcinFk%sFDWcj{{;(KVwNY7ngq3`-iL;hNz^9I|F?B?v;ZHZZ*qRi2<5iJpZK zM*byaBsfE(n}?q55a!JUCTF}(M9+liMx*C4gwd?b1dK+>bLi%x=Qo(STN;B2nG4B% V0p6^j+|R%cgo;34egu^R3;+-1eXjrj literal 0 HcmV?d00001 diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..4bb80477caf51393354453b18c0c88711098825e GIT binary patch literal 1832 zcmWIWW@Zs#U|`^2xLh&ryk0?NcfY>3uGYCT z=ezsEH-rWlT`<029P+5E-(N@fdGY9=0ryhVJ$tvvUNQc+{d3{D zn|B2?ro>EK#LF7E*yLqccPAsi^Am}D%e30AoeArVlYJ((&Chx9t8w97&hq5+iY(pM z?WKI0`Ng-xP8DwL5?S^qs;1iP{ao7%o!XY-3yZ&e5Ut|$zxe;x^oV^&AD++Fd7iem zR(X?DP}|ALQzYCuG|lZak1|#q*u;_lsY}J+e|O){(NT&M^DqET*~S;Ni)SUsN4C z9Dn1{?j=1*vsGR>2Ap1Urdi@ME2@w3r*1Nd0r~`ll?eGL+{4w?hx@#T@7asIkTkJ5 z$l!|cgChU4-nw2oC-pZ4d3c@F(d*P_Um1A8;DWK?CF2WExK5qdzxY+>8n1?~*V&Uk zn}RenwS1p)dHQS*(pX_~?d<9E-dBCktbX!{2`Tk}Lc4^`-&YapE?q)yOU)}Os*Eql z&&(?+)+?zf>Gsnum6bPz^n=Yu)|leb1cf?%&Q77=vb)*4}*!*?@Mi z0I?P!yGrs4;&W2VQgc8SY3q4CLw9*{w)FQ{Mg|65CI$v&LZ$_|I{OE?w4OU%)MUWm zaB=T_A&mrw*ph>*mAaB@F8Q(@M!#{WU5W^A%QRiSMW z{ozJ=^Fq0UM|XJ53fr~pZf|;Euh=Z{Z-Tx4a;F>EwmRNScHD8mvs7)8lJ+L8`I_QF zshMAec0^w`-2TiwZ^_?(T*a4Um+uz%9hJHAo6n-@<%iYQ%+*hCy{@^uvZID|#pH|s ze;h2ndvtru#jke9t4>^Ne8ji;gOB{FCkgtet-sDum^X=S@}n%?$yyAP)vvd7E<0Z8 zI`hG%7f1XLTdv`o%)7mde~0q)?pa+LTXvaz|5|P@sQzMo+2)nERD@WZf7CmNnCf|W zcinFk%sFDWcj{{;(KVwNY7ngq3`-iL;hNz^9I|F?B?v;ZHZZ*qRi2<5iJpZK zM*byaBsfE(n}?q55a!JUCTF}(M9+liMx*C4gwd?b1dK+>bLi%x=Qo(STN;B2nG4B% V0p6^j+|R%cgo;34egu^R3;+-1eXjrj literal 0 HcmV?d00001 diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 53c6764ea..05e4fdfaf 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -130,3 +130,16 @@ func TestPythonWheelBuildMultiple(t *testing.T) { diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } + +func TestPythonWheelNoBuild(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + match := libraries.ValidateLocalLibrariesExist() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} From f3ffded3bf8b3800e4042d04fabcf3ab0092ac5c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 6 Aug 2024 18:12:18 +0200 Subject: [PATCH 49/88] Merge job parameters based on their name (#1659) ## Changes This change enables overriding the default value of job parameters in target overrides. This is the same approach we already take for job clusters and job tasks. Closes #1620. ## Tests Mutator unit tests and lightweight end-to-end tests. --- bundle/config/mutator/merge_job_parameters.go | 45 +++++++++++ .../mutator/merge_job_parameters_test.go | 80 +++++++++++++++++++ bundle/phases/initialize.go | 1 + bundle/tests/loader.go | 1 + .../override_job_parameters/databricks.yml | 32 ++++++++ bundle/tests/override_job_parameters_test.go | 31 +++++++ 6 files changed, 190 insertions(+) create mode 100644 bundle/config/mutator/merge_job_parameters.go create mode 100644 bundle/config/mutator/merge_job_parameters_test.go create mode 100644 bundle/tests/override_job_parameters/databricks.yml create mode 100644 bundle/tests/override_job_parameters_test.go diff --git a/bundle/config/mutator/merge_job_parameters.go b/bundle/config/mutator/merge_job_parameters.go new file mode 100644 index 000000000..51a919d98 --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters.go @@ -0,0 +1,45 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" +) + +type mergeJobParameters struct{} + +func MergeJobParameters() bundle.Mutator { + return &mergeJobParameters{} +} + +func (m *mergeJobParameters) Name() string { + return "MergeJobParameters" +} + +func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { + switch v.Kind() { + case dyn.KindInvalid, dyn.KindNil: + return "" + case dyn.KindString: + return v.MustString() + default: + panic("task key must be a string") + } +} + +func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + if v.Kind() == dyn.KindNil { + return v, nil + } + + return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { + return dyn.Map(job, "parameters", merge.ElementsByKey("name", m.parameterNameString)) + })) + }) + + return diag.FromErr(err) +} diff --git a/bundle/config/mutator/merge_job_parameters_test.go b/bundle/config/mutator/merge_job_parameters_test.go new file mode 100644 index 000000000..f03dea734 --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters_test.go @@ -0,0 +1,80 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestMergeJobParameters(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Name: "foo", + Default: "v1", + }, + { + Name: "bar", + Default: "v1", + }, + { + Name: "foo", + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + + j := b.Config.Resources.Jobs["foo"] + + assert.Len(t, j.Parameters, 2) + assert.Equal(t, "foo", j.Parameters[0].Name) + assert.Equal(t, "v2", j.Parameters[0].Default) + assert.Equal(t, "bar", j.Parameters[1].Name) + assert.Equal(t, "v1", j.Parameters[1].Default) +} + +func TestMergeJobParametersWithNilKey(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Default: "v1", + }, + { + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + assert.Len(t, b.Config.Resources.Jobs["foo"].Parameters, 1) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a32de2c56..7b4dc6d41 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -21,6 +21,7 @@ func Initialize() bundle.Mutator { []bundle.Mutator{ mutator.RewriteSyncPaths(), mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 8eddcf9a1..069f09358 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -37,6 +37,7 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { phases.LoadNamedTarget(env), mutator.RewriteSyncPaths(), mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), )) diff --git a/bundle/tests/override_job_parameters/databricks.yml b/bundle/tests/override_job_parameters/databricks.yml new file mode 100644 index 000000000..9c333c323 --- /dev/null +++ b/bundle/tests/override_job_parameters/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: override_job_parameters + +workspace: + host: https://acme.cloud.databricks.com/ + +resources: + jobs: + foo: + name: job + parameters: + - name: foo + default: v1 + - name: bar + default: v1 + +targets: + development: + resources: + jobs: + foo: + parameters: + - name: foo + default: v2 + + staging: + resources: + jobs: + foo: + parameters: + - name: bar + default: v2 diff --git a/bundle/tests/override_job_parameters_test.go b/bundle/tests/override_job_parameters_test.go new file mode 100644 index 000000000..21e0e35a6 --- /dev/null +++ b/bundle/tests/override_job_parameters_test.go @@ -0,0 +1,31 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOverrideJobParametersDev(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "development") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v2", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v1", p[1].Default) +} + +func TestOverrideJobParametersStaging(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "staging") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v1", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v2", p[1].Default) +} From 9d1fbbb39c9628b765b76809cec1867889f393bf Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 6 Aug 2024 20:58:34 +0200 Subject: [PATCH 50/88] Enable Spark JAR task test (#1658) ## Changes Enable Spark JAR task test ## Tests ``` Updating deployment state... Deleting files... Destroy complete! --- PASS: TestAccSparkJarTaskDeployAndRunOnVolumes (194.13s) PASS coverage: 51.9% of statements in ./... ok github.com/databricks/cli/internal/bundle 194.586s coverage: 51.9% of statements in ./... ``` --- .../databricks_template_schema.json | 4 ++++ .../template/databricks.yml.tmpl | 1 + internal/bundle/spark_jar_test.go | 21 ++++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json index 078dff976..1381da1dd 100644 --- a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json +++ b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json @@ -24,6 +24,10 @@ "artifact_path": { "type": "string", "description": "Path to the remote base path for artifacts" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" } } } diff --git a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl index 24a6d7d8a..8c9331fe6 100644 --- a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl @@ -22,6 +22,7 @@ resources: num_workers: 1 spark_version: "{{.spark_version}}" node_type_id: "{{.node_type_id}}" + instance_pool_id: "{{.instance_pool_id}}" spark_jar_task: main_class_name: PrintArgs libraries: diff --git a/internal/bundle/spark_jar_test.go b/internal/bundle/spark_jar_test.go index c981e7750..98bfa4a9d 100644 --- a/internal/bundle/spark_jar_test.go +++ b/internal/bundle/spark_jar_test.go @@ -6,15 +6,14 @@ import ( "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/env" "github.com/google/uuid" "github.com/stretchr/testify/require" ) func runSparkJarTest(t *testing.T, sparkVersion string) { - t.Skip("Temporarily skipping the test until auth / permission issues for UC volumes are resolved.") - - env := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") - t.Log(env) + cloudEnv := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + t.Log(cloudEnv) if os.Getenv("TEST_METASTORE_ID") == "" { t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") @@ -24,14 +23,16 @@ func runSparkJarTest(t *testing.T, sparkVersion string) { w := wt.W volumePath := internal.TemporaryUcVolume(t, w) - nodeTypeId := internal.GetNodeTypeId(env) + nodeTypeId := internal.GetNodeTypeId(cloudEnv) tmpDir := t.TempDir() + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") bundleRoot, err := initTestTemplateWithBundleRoot(t, ctx, "spark_jar_task", map[string]any{ - "node_type_id": nodeTypeId, - "unique_id": uuid.New().String(), - "spark_version": sparkVersion, - "root": tmpDir, - "artifact_path": volumePath, + "node_type_id": nodeTypeId, + "unique_id": uuid.New().String(), + "spark_version": sparkVersion, + "root": tmpDir, + "artifact_path": volumePath, + "instance_pool_id": instancePoolId, }, tmpDir) require.NoError(t, err) From d3d828d175afcee48449bd530a1f19f798fe3831 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 7 Aug 2024 16:47:03 +0200 Subject: [PATCH 51/88] Fix glob expansion after running a generic build command (#1662) ## Changes This didn't work as expected because the generic build mutator called into the type-specific build mutator in the middle of the function. This invalidated the `config.Artifact` pointer that was being mutated later on, effectively hiding these mutations from its caller. To fix this, I turned glob expansion into its own mutator. It now works as expected, _and_ produces better errors if the glob patterns are invalid or do not match files. ## Tests Unit tests. Manual verification: ``` % databricks bundle deploy Building sbt_example... Error: target/scala-2.12/sbt-e[xam22ple*.jar: syntax error in pattern at artifacts.sbt_example.files[1].source in databricks.yml:15:17 ``` --- bundle/artifacts/build.go | 43 +------ bundle/artifacts/expand_globs.go | 110 ++++++++++++++++++ bundle/artifacts/expand_globs_test.go | 156 ++++++++++++++++++++++++++ bundle/artifacts/upload_test.go | 2 +- 4 files changed, 273 insertions(+), 38 deletions(-) create mode 100644 bundle/artifacts/expand_globs.go create mode 100644 bundle/artifacts/expand_globs_test.go diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 47c2f24d8..0446135b6 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -3,10 +3,8 @@ package artifacts import ( "context" "fmt" - "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" ) @@ -35,6 +33,8 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("artifact doesn't exist: %s", m.name) } + var mutators []bundle.Mutator + // Skip building if build command is not specified or infered if artifact.BuildCommand == "" { // If no build command was specified or infered and there is no @@ -45,44 +45,13 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We can skip calling build mutator if there is no build command // But we still need to expand glob references in files source path. - diags := expandGlobReference(artifact) - return diags - } - - diags := bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) - if diags.HasError() { - return diags + } else { + mutators = append(mutators, getBuildMutator(artifact.Type, m.name)) } // We need to expand glob reference after build mutator is applied because // if we do it before, any files that are generated by build command will // not be included into artifact.Files and thus will not be uploaded. - d := expandGlobReference(artifact) - return diags.Extend(d) -} - -func expandGlobReference(artifact *config.Artifact) diag.Diagnostics { - var diags diag.Diagnostics - - // Expand any glob reference in files source path - files := make([]config.ArtifactFile, 0, len(artifact.Files)) - for _, f := range artifact.Files { - matches, err := filepath.Glob(f.Source) - if err != nil { - return diags.Extend(diag.Errorf("unable to find files for %s: %v", f.Source, err)) - } - - if len(matches) == 0 { - return diags.Extend(diag.Errorf("no files found for %s", f.Source)) - } - - for _, match := range matches { - files = append(files, config.ArtifactFile{ - Source: match, - }) - } - } - - artifact.Files = files - return diags + mutators = append(mutators, &expandGlobs{name: m.name}) + return bundle.Apply(ctx, b, bundle.Seq(mutators...)) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go new file mode 100644 index 000000000..617444054 --- /dev/null +++ b/bundle/artifacts/expand_globs.go @@ -0,0 +1,110 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expandGlobs struct { + name string +} + +func (m *expandGlobs) Name() string { + return fmt.Sprintf("artifacts.ExpandGlobs(%s)", m.name) +} + +func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { + // The pattern contained in v is an absolute path. + // Make it relative to the value's location to make it more readable. + source := v.MustString() + if l := v.Location(); l.File != "" { + rel, err := filepath.Rel(filepath.Dir(l.File), source) + if err == nil { + source = rel + } + } + + return diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s: %s", source, message), + Locations: []dyn.Location{v.Location()}, + + Paths: []dyn.Path{ + // Hack to clone the path. This path copy is mutable. + // To be addressed in a later PR. + p.Append(), + }, + } +} + +func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Base path for this mutator. + // This path is set with the list of expanded globs when done. + base := dyn.NewPath( + dyn.Key("artifacts"), + dyn.Key(m.name), + dyn.Key("files"), + ) + + // Pattern to match the source key in the files sequence. + pattern := dyn.NewPatternFromPath(base).Append( + dyn.AnyIndex(), + dyn.Key("source"), + ) + + var diags diag.Diagnostics + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var output []dyn.Value + _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() != dyn.KindString { + return v, nil + } + + source := v.MustString() + + // Expand any glob reference in files source path + matches, err := filepath.Glob(source) + if err != nil { + diags = diags.Append(createGlobError(v, p, err.Error())) + + // Continue processing and leave this value unchanged. + return v, nil + } + + if len(matches) == 0 { + diags = diags.Append(createGlobError(v, p, "no matching files")) + + // Continue processing and leave this value unchanged. + return v, nil + } + + for _, match := range matches { + output = append(output, dyn.V( + map[string]dyn.Value{ + "source": dyn.NewValue(match, v.Locations()), + }, + )) + } + + return v, nil + }) + + if err != nil || diags.HasError() { + return v, err + } + + // Set the expanded globs back into the configuration. + return dyn.SetByPath(v, base, dyn.V(output)) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go new file mode 100644 index 000000000..c9c478448 --- /dev/null +++ b/bundle/artifacts/expand_globs_test.go @@ -0,0 +1,156 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpandGlobs_Nominal(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "aa1.txt") + testutil.Touch(t, tmpDir, "aa2.txt") + testutil.Touch(t, tmpDir, "bb.txt") + testutil.Touch(t, tmpDir, "bc.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "./aa*.txt"}, + {Source: "./b[bc].txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + require.NoError(t, diags.Error()) + + // Assert that the expanded paths are correct. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + assert.Len(t, a.Files, 4) + assert.Equal(t, filepath.Join(tmpDir, "aa1.txt"), a.Files[0].Source) + assert.Equal(t, filepath.Join(tmpDir, "aa2.txt"), a.Files[1].Source) + assert.Equal(t, filepath.Join(tmpDir, "bb.txt"), a.Files[2].Source) + assert.Equal(t, filepath.Join(tmpDir, "bc.txt"), a.Files[3].Source) +} + +func TestExpandGlobs_InvalidPattern(t *testing.T) { + tmpDir := t.TempDir() + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a[.txt"}, + {Source: "./a[.txt"}, + {Source: "../a[.txt"}, + {Source: "subdir/a[.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 4) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String()) +} + +func TestExpandGlobs_NoMatches(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "a1.txt") + testutil.Touch(t, tmpDir, "a2.txt") + testutil.Touch(t, tmpDir, "b1.txt") + testutil.Touch(t, tmpDir, "b2.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a*.txt"}, + {Source: "b*.txt"}, + {Source: "c*.txt"}, + {Source: "d*.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 2) + assert.Equal(t, "c*.txt: no matching files", diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[0].Paths[0].String()) + assert.Equal(t, "d*.txt: no matching files", diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[1].Paths[0].String()) + + // Assert that the original paths are unchanged. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + + assert.Len(t, a.Files, 4) + assert.Equal(t, "a*.txt", filepath.Base(a.Files[0].Source)) + assert.Equal(t, "b*.txt", filepath.Base(a.Files[1].Source)) + assert.Equal(t, "c*.txt", filepath.Base(a.Files[2].Source)) + assert.Equal(t, "d*.txt", filepath.Base(a.Files[3].Source)) +} diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go index a71610b03..202086bd3 100644 --- a/bundle/artifacts/upload_test.go +++ b/bundle/artifacts/upload_test.go @@ -110,5 +110,5 @@ func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { } diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.ErrorContains(t, diags.Error(), "no files found for") + require.ErrorContains(t, diags.Error(), "no matching files") } From 65f4aad87caa86f1a8fbc76fc100f652bb0cc819 Mon Sep 17 00:00:00 2001 From: andersrexdb Date: Fri, 9 Aug 2024 12:40:25 +0300 Subject: [PATCH 52/88] Add command line autocomplete to the fs commands (#1622) ## Changes This PR adds autocomplete for cat, cp, ls, mkdir and rm. The new completer can do completion for any `Filer`. The command completion for the `sync` command can be moved to use this general completer as a follow-up. ## Tests - Tested manually against a workspace - Unit tests --- cmd/fs/cat.go | 3 + cmd/fs/cp.go | 5 + cmd/fs/helpers.go | 56 ++++++++++- cmd/fs/helpers_test.go | 89 ++++++++++++++++ cmd/fs/ls.go | 4 + cmd/fs/mkdir.go | 4 + cmd/fs/rm.go | 3 + internal/completer_test.go | 27 +++++ libs/filer/completer/completer.go | 95 ++++++++++++++++++ libs/filer/completer/completer_test.go | 104 +++++++++++++++++++ libs/filer/fake_filer.go | 134 +++++++++++++++++++++++++ libs/filer/fs_test.go | 132 ++---------------------- libs/fileset/glob_test.go | 5 +- 13 files changed, 532 insertions(+), 129 deletions(-) create mode 100644 internal/completer_test.go create mode 100644 libs/filer/completer/completer.go create mode 100644 libs/filer/completer/completer_test.go create mode 100644 libs/filer/fake_filer.go diff --git a/cmd/fs/cat.go b/cmd/fs/cat.go index 7a6f42cba..28df80d70 100644 --- a/cmd/fs/cat.go +++ b/cmd/fs/cat.go @@ -30,5 +30,8 @@ func newCatCommand() *cobra.Command { return cmdio.Render(ctx, r) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 52feb8905..6fb3e5e6f 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -200,5 +200,10 @@ func newCpCommand() *cobra.Command { return c.cpFileToFile(sourcePath, targetPath) } + v := newValidArgs() + // The copy command has two paths that can be completed (SOURCE_PATH & TARGET_PATH) + v.pathArgCount = 2 + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 43d65b5dd..bda3239cf 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -8,6 +8,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/filer/completer" + "github.com/spf13/cobra" ) func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { @@ -46,6 +48,58 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return f, path, err } +const dbfsPrefix string = "dbfs:" + func isDbfsPath(path string) bool { - return strings.HasPrefix(path, "dbfs:/") + return strings.HasPrefix(path, dbfsPrefix) +} + +type validArgs struct { + mustWorkspaceClientFunc func(cmd *cobra.Command, args []string) error + filerForPathFunc func(ctx context.Context, fullPath string) (filer.Filer, string, error) + pathArgCount int + onlyDirs bool +} + +func newValidArgs() *validArgs { + return &validArgs{ + mustWorkspaceClientFunc: root.MustWorkspaceClient, + filerForPathFunc: filerForPath, + pathArgCount: 1, + onlyDirs: false, + } +} + +func (v *validArgs) Validate(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + cmd.SetContext(root.SkipPrompt(cmd.Context())) + + if len(args) >= v.pathArgCount { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + err := v.mustWorkspaceClientFunc(cmd, args) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + filer, toCompletePath, err := v.filerForPathFunc(cmd.Context(), toComplete) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + completer := completer.New(cmd.Context(), filer, v.onlyDirs) + + // Dbfs should have a prefix and always use the "/" separator + isDbfsPath := isDbfsPath(toComplete) + if isDbfsPath { + completer.SetPrefix(dbfsPrefix) + completer.SetIsLocalPath(false) + } + + completions, directive, err := completer.CompletePath(toCompletePath) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + return completions, directive } diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index d86bd46e1..10b4aa160 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -3,9 +3,13 @@ package fs import ( "context" "runtime" + "strings" "testing" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -60,3 +64,88 @@ func TestFilerForWindowsLocalPaths(t *testing.T) { testWindowsFilerForPath(t, ctx, `d:\abc`) testWindowsFilerForPath(t, ctx, `f:\abc\ef`) } + +func mockMustWorkspaceClientFunc(cmd *cobra.Command, args []string) error { + return nil +} + +func setupCommand(t *testing.T) (*cobra.Command, *mocks.MockWorkspaceClient) { + m := mocks.NewMockWorkspaceClient(t) + ctx := context.Background() + ctx = root.SetWorkspaceClient(ctx, m.WorkspaceClient) + + cmd := &cobra.Command{} + cmd.SetContext(ctx) + + return cmd, m +} + +func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceClient) { + cmd, m := setupCommand(t) + + fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) { + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + return fakeFiler, strings.TrimPrefix(fullPath, "dbfs:/"), nil + } + + v := newValidArgs() + v.filerForPathFunc = fakeFilerForPath + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + return v, cmd, m +} + +func TestGetValidArgsFunctionDbfsCompletion(t *testing.T) { + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/", "dbfs:/dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletion(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletionWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dir\\fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionCompletionOnlyDirs(t *testing.T) { + v, cmd, _ := setupTest(t) + v.onlyDirs = true + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionNotCompletedArgument(t *testing.T) { + cmd, _ := setupCommand(t) + + v := newValidArgs() + v.pathArgCount = 0 + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + completions, directive := v.Validate(cmd, []string{}, "dbfs:/") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) +} diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index cec9b98ba..d7eac513a 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -89,5 +89,9 @@ func newLsCommand() *cobra.Command { `)) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/mkdir.go b/cmd/fs/mkdir.go index 074a7543d..5e9ac7842 100644 --- a/cmd/fs/mkdir.go +++ b/cmd/fs/mkdir.go @@ -28,5 +28,9 @@ func newMkdirCommand() *cobra.Command { return f.Mkdir(ctx, path) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/rm.go b/cmd/fs/rm.go index 5f2904e71..a133a8309 100644 --- a/cmd/fs/rm.go +++ b/cmd/fs/rm.go @@ -32,5 +32,8 @@ func newRmCommand() *cobra.Command { return f.Delete(ctx, path) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/internal/completer_test.go b/internal/completer_test.go new file mode 100644 index 000000000..0f7d2093d --- /dev/null +++ b/internal/completer_test.go @@ -0,0 +1,27 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "testing" + + _ "github.com/databricks/cli/cmd/fs" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupCompletionFile(t *testing.T, f filer.Filer) { + err := f.Write(context.Background(), "dir1/file1.txt", strings.NewReader("abc"), filer.CreateParentDirectories) + require.NoError(t, err) +} + +func TestAccFsCompletion(t *testing.T) { + f, tmpDir := setupDbfsFiler(t) + setupCompletionFile(t, f) + + stdout, _ := RequireSuccessfulRun(t, "__complete", "fs", "ls", tmpDir) + expectedOutput := fmt.Sprintf("%s/dir1/\n:2\n", tmpDir) + assert.Equal(t, expectedOutput, stdout.String()) +} diff --git a/libs/filer/completer/completer.go b/libs/filer/completer/completer.go new file mode 100644 index 000000000..569286ca3 --- /dev/null +++ b/libs/filer/completer/completer.go @@ -0,0 +1,95 @@ +package completer + +import ( + "context" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/filer" + "github.com/spf13/cobra" +) + +type completer struct { + ctx context.Context + + // The filer to use for completing remote or local paths. + filer filer.Filer + + // CompletePath will only return directories when onlyDirs is true. + onlyDirs bool + + // Prefix to prepend to completions. + prefix string + + // Whether the path is local or remote. If the path is local we use the `filepath` + // package for path manipulation. Otherwise we use the `path` package. + isLocalPath bool +} + +// General completer that takes a filer to complete remote paths when TAB-ing through a path. +func New(ctx context.Context, filer filer.Filer, onlyDirs bool) *completer { + return &completer{ctx: ctx, filer: filer, onlyDirs: onlyDirs, prefix: "", isLocalPath: true} +} + +func (c *completer) SetPrefix(p string) { + c.prefix = p +} + +func (c *completer) SetIsLocalPath(i bool) { + c.isLocalPath = i +} + +func (c *completer) CompletePath(p string) ([]string, cobra.ShellCompDirective, error) { + trailingSeparator := "/" + joinFunc := path.Join + + // Use filepath functions if we are in a local path. + if c.isLocalPath { + joinFunc = filepath.Join + trailingSeparator = string(filepath.Separator) + } + + // If the user is TAB-ing their way through a path and the + // path ends in a trailing slash, we should list nested directories. + // If the path is incomplete, however, then we should list adjacent + // directories. + dirPath := p + if !strings.HasSuffix(p, trailingSeparator) { + dirPath = path.Dir(p) + } + + entries, err := c.filer.ReadDir(c.ctx, dirPath) + if err != nil { + return nil, cobra.ShellCompDirectiveError, err + } + + completions := []string{} + for _, entry := range entries { + if c.onlyDirs && !entry.IsDir() { + continue + } + + // Join directory path and entry name + completion := joinFunc(dirPath, entry.Name()) + + // Prepend prefix if it has been set + if c.prefix != "" { + completion = joinFunc(c.prefix, completion) + } + + // Add trailing separator for directories. + if entry.IsDir() { + completion += trailingSeparator + } + + completions = append(completions, completion) + } + + // If the path is local, we add the dbfs:/ prefix suggestion as an option + if c.isLocalPath { + completions = append(completions, "dbfs:/") + } + + return completions, cobra.ShellCompDirectiveNoSpace, err +} diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go new file mode 100644 index 000000000..c533f0b6c --- /dev/null +++ b/libs/filer/completer/completer_test.go @@ -0,0 +1,104 @@ +package completer + +import ( + "context" + "runtime" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func setupCompleter(t *testing.T, onlyDirs bool) *completer { + ctx := context.Background() + // Needed to make type context.valueCtx for mockFilerForPath + ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient) + + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + + completer := New(ctx, fakeFiler, onlyDirs) + completer.SetIsLocalPath(false) + return completer +} + +func TestFilerCompleterSetsPrefix(t *testing.T) { + completer := setupCompleter(t, true) + completer.SetPrefix("dbfs:") + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/wrong_path") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { + completer := setupCompleter(t, false) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterAddsDbfsPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterWindowsSeparator(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterNoCompletions(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("wrong_dir/wrong_dir") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveError, directive) + assert.Error(t, err) +} diff --git a/libs/filer/fake_filer.go b/libs/filer/fake_filer.go new file mode 100644 index 000000000..0e650ff60 --- /dev/null +++ b/libs/filer/fake_filer.go @@ -0,0 +1,134 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "sort" + "strings" + "time" +) + +type FakeDirEntry struct { + FakeFileInfo +} + +func (entry FakeDirEntry) Type() fs.FileMode { + typ := fs.ModePerm + if entry.FakeDir { + typ |= fs.ModeDir + } + return typ +} + +func (entry FakeDirEntry) Info() (fs.FileInfo, error) { + return entry.FakeFileInfo, nil +} + +type FakeFileInfo struct { + FakeName string + FakeSize int64 + FakeDir bool + FakeMode fs.FileMode +} + +func (info FakeFileInfo) Name() string { + return info.FakeName +} + +func (info FakeFileInfo) Size() int64 { + return info.FakeSize +} + +func (info FakeFileInfo) Mode() fs.FileMode { + return info.FakeMode +} + +func (info FakeFileInfo) ModTime() time.Time { + return time.Now() +} + +func (info FakeFileInfo) IsDir() bool { + return info.FakeDir +} + +func (info FakeFileInfo) Sys() any { + return nil +} + +type FakeFiler struct { + entries map[string]FakeFileInfo +} + +func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { + _, ok := f.entries[p] + if !ok { + return nil, fs.ErrNotExist + } + + return io.NopCloser(strings.NewReader("foo")), nil +} + +func (f *FakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { + p = strings.TrimSuffix(p, "/") + entry, ok := f.entries[p] + if !ok { + return nil, NoSuchDirectoryError{p} + } + + if !entry.FakeDir { + return nil, fs.ErrInvalid + } + + // Find all entries contained in the specified directory `p`. + var out []fs.DirEntry + for k, v := range f.entries { + if k == p || path.Dir(k) != p { + continue + } + + out = append(out, FakeDirEntry{v}) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) + return out, nil +} + +func (f *FakeFiler) Mkdir(ctx context.Context, path string) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { + entry, ok := f.entries[path] + if !ok { + return nil, fs.ErrNotExist + } + + return entry, nil +} + +func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler { + fakeFiler := &FakeFiler{ + entries: entries, + } + + for k, v := range fakeFiler.entries { + if v.FakeName != "" { + continue + } + v.FakeName = path.Base(k) + fakeFiler.entries[k] = v + } + + return fakeFiler +} diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 03ed312b4..a74c10f0b 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -2,124 +2,14 @@ package filer import ( "context" - "fmt" "io" "io/fs" - "path" - "sort" - "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type fakeDirEntry struct { - fakeFileInfo -} - -func (entry fakeDirEntry) Type() fs.FileMode { - typ := fs.ModePerm - if entry.dir { - typ |= fs.ModeDir - } - return typ -} - -func (entry fakeDirEntry) Info() (fs.FileInfo, error) { - return entry.fakeFileInfo, nil -} - -type fakeFileInfo struct { - name string - size int64 - dir bool - mode fs.FileMode -} - -func (info fakeFileInfo) Name() string { - return info.name -} - -func (info fakeFileInfo) Size() int64 { - return info.size -} - -func (info fakeFileInfo) Mode() fs.FileMode { - return info.mode -} - -func (info fakeFileInfo) ModTime() time.Time { - return time.Now() -} - -func (info fakeFileInfo) IsDir() bool { - return info.dir -} - -func (info fakeFileInfo) Sys() any { - return nil -} - -type fakeFiler struct { - entries map[string]fakeFileInfo -} - -func (f *fakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { - _, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - return io.NopCloser(strings.NewReader("foo")), nil -} - -func (f *fakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { - entry, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - if !entry.dir { - return nil, fs.ErrInvalid - } - - // Find all entries contained in the specified directory `p`. - var out []fs.DirEntry - for k, v := range f.entries { - if k == p || path.Dir(k) != p { - continue - } - - out = append(out, fakeDirEntry{v}) - } - - sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) - return out, nil -} - -func (f *fakeFiler) Mkdir(ctx context.Context, path string) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { - entry, ok := f.entries[path] - if !ok { - return nil, fs.ErrNotExist - } - - return entry, nil -} - func TestFsImplementsFS(t *testing.T) { var _ fs.FS = &filerFS{} } @@ -145,22 +35,12 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) { } func fakeFS() fs.FS { - fakeFiler := &fakeFiler{ - entries: map[string]fakeFileInfo{ - ".": {name: "root", dir: true}, - "dirA": {dir: true}, - "dirB": {dir: true}, - "fileA": {size: 3}, - }, - } - - for k, v := range fakeFiler.entries { - if v.name != "" { - continue - } - v.name = path.Base(k) - fakeFiler.entries[k] = v - } + fakeFiler := NewFakeFiler(map[string]FakeFileInfo{ + ".": {FakeName: "root", FakeDir: true}, + "dirA": {FakeDir: true}, + "dirB": {FakeDir: true}, + "fileA": {FakeSize: 3}, + }) return NewFS(context.Background(), fakeFiler) } diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 70b9c444b..8418df73a 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -20,7 +20,7 @@ func collectRelativePaths(files []File) []string { } func TestGlobFileset(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("./") entries, err := root.ReadDir(".") require.NoError(t, err) @@ -32,6 +32,7 @@ func TestGlobFileset(t *testing.T) { files, err := g.All() require.NoError(t, err) + // +1 as there's one folder in ../filer require.Equal(t, len(files), len(entries)) for _, f := range files { exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool { @@ -51,7 +52,7 @@ func TestGlobFileset(t *testing.T) { } func TestGlobFilesetWithRelativeRoot(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("../set") entries, err := root.ReadDir(".") require.NoError(t, err) From a240be0b5a7eed8d0746ba0f167df0dd465f06c1 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 9 Aug 2024 17:13:31 +0200 Subject: [PATCH 53/88] Run Spark JAR task test on multiple DBR versions (#1665) ## Changes This explores error messages on older DBRs and UC vs non-UC. ## Tests Integration tests pass. --- .../template/databricks.yml.tmpl | 28 ++++++- internal/bundle/helpers.go | 11 ++- internal/bundle/spark_jar_test.go | 77 +++++++++++++++---- internal/testutil/jdk.go | 24 ++++++ 4 files changed, 120 insertions(+), 20 deletions(-) create mode 100644 internal/testutil/jdk.go diff --git a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl index 8c9331fe6..db451cd93 100644 --- a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl @@ -3,7 +3,6 @@ bundle: workspace: root_path: "~/.bundle/{{.unique_id}}" - artifact_path: {{.artifact_path}} artifacts: my_java_code: @@ -27,3 +26,30 @@ resources: main_class_name: PrintArgs libraries: - jar: ./{{.project_name}}/PrintArgs.jar + +targets: + volume: + # Override the artifact path to upload artifacts to a volume path + workspace: + artifact_path: {{.artifact_path}} + + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in single user mode (force it to be a UC cluster) + data_security_mode: SINGLE_USER + + workspace: + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in no isolation mode (force it to be a non-UC cluster) + data_security_mode: NONE diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 1910a0148..03d9cff70 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/template" "github.com/databricks/databricks-sdk-go" @@ -56,21 +57,21 @@ func writeConfigFile(t *testing.T, config map[string]any) (string, error) { } func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, error) { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "validate", "--output", "json") stdout, _, err := c.Run() return stdout.Bytes(), err } func deployBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve") _, _, err := c.Run() return err } func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) args := []string{"bundle", "deploy", "--force-lock"} args = append(args, flags...) c := internal.NewCobraTestRunnerWithContext(t, ctx, args...) @@ -79,6 +80,7 @@ func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags } func runResource(t *testing.T, ctx context.Context, path string, key string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "run", key) @@ -87,6 +89,7 @@ func runResource(t *testing.T, ctx context.Context, path string, key string) (st } func runResourceWithParams(t *testing.T, ctx context.Context, path string, key string, params ...string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) args := make([]string, 0) @@ -98,7 +101,7 @@ func runResourceWithParams(t *testing.T, ctx context.Context, path string, key s } func destroyBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "destroy", "--auto-approve") _, _, err := c.Run() return err diff --git a/internal/bundle/spark_jar_test.go b/internal/bundle/spark_jar_test.go index 98bfa4a9d..4b469617c 100644 --- a/internal/bundle/spark_jar_test.go +++ b/internal/bundle/spark_jar_test.go @@ -1,28 +1,19 @@ package bundle import ( - "os" + "context" "testing" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/env" "github.com/google/uuid" "github.com/stretchr/testify/require" ) -func runSparkJarTest(t *testing.T, sparkVersion string) { +func runSparkJarTestCommon(t *testing.T, ctx context.Context, sparkVersion string, artifactPath string) { cloudEnv := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") - t.Log(cloudEnv) - - if os.Getenv("TEST_METASTORE_ID") == "" { - t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") - } - - ctx, wt := acc.WorkspaceTest(t) - w := wt.W - volumePath := internal.TemporaryUcVolume(t, w) - nodeTypeId := internal.GetNodeTypeId(cloudEnv) tmpDir := t.TempDir() instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") @@ -31,7 +22,7 @@ func runSparkJarTest(t *testing.T, sparkVersion string) { "unique_id": uuid.New().String(), "spark_version": sparkVersion, "root": tmpDir, - "artifact_path": volumePath, + "artifact_path": artifactPath, "instance_pool_id": instancePoolId, }, tmpDir) require.NoError(t, err) @@ -48,6 +39,62 @@ func runSparkJarTest(t *testing.T, sparkVersion string) { require.Contains(t, out, "Hello from Jar!") } -func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { - runSparkJarTest(t, "14.3.x-scala2.12") +func runSparkJarTestFromVolume(t *testing.T, sparkVersion string) { + ctx, wt := acc.UcWorkspaceTest(t) + volumePath := internal.TemporaryUcVolume(t, wt.W) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "volume") + runSparkJarTestCommon(t, ctx, sparkVersion, volumePath) +} + +func runSparkJarTestFromWorkspace(t *testing.T, sparkVersion string) { + ctx, _ := acc.WorkspaceTest(t) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "workspace") + runSparkJarTestCommon(t, ctx, sparkVersion, "n/a") +} + +func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // JAR installation from Volumes is supported on UC Clusters with DBR >= 13.3. + // Denied library is Jar(/Volumes/main/test-schema-ldgaklhcahlg/my-volume/.internal/PrintArgs.jar) + // + + versions := []string{ + "13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12) + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromVolume(t, version) + }) + } +} + +func TestAccSparkJarTaskDeployAndRunOnWorkspace(t *testing.T) { + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // Library from /Workspace is not allowed on this cluster. + // Please switch to using DBR 14.1+ No Isolation Shared or DBR 13.1+ Shared cluster or 13.2+ Assigned cluster to use /Workspace libraries. + // + + versions := []string{ + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromWorkspace(t, version) + }) + } } diff --git a/internal/testutil/jdk.go b/internal/testutil/jdk.go new file mode 100644 index 000000000..05bd7d6d6 --- /dev/null +++ b/internal/testutil/jdk.go @@ -0,0 +1,24 @@ +package testutil + +import ( + "bytes" + "context" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func RequireJDK(t *testing.T, ctx context.Context, version string) { + var stderr bytes.Buffer + + cmd := exec.Command("javac", "-version") + cmd.Stderr = &stderr + err := cmd.Run() + require.NoError(t, err, "Unable to run javac -version") + + // Get the first line of the output + line := strings.Split(stderr.String(), "\n")[0] + require.Contains(t, line, version, "Expected JDK version %s, got %s", version, line) +} From 0a4c0fb588eaf9bda3dc8e4184286a99541bc869 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 12 Aug 2024 09:07:50 +0200 Subject: [PATCH 54/88] Add trailing slash to directory to produce completions for (#1666) ## Changes The integration test for `fs ls` tried to produce completions for the temporary directory without a trailing slash. The command will list the parent directory to see if there are more directories with the same prefix that are also valid completions. The test intends to test completions for files inside the temporary directory, so we need that trailing slash. This test was hanging because the parent directory of the temporary DBFS directory has more than 1k entries (on our integration testing workspaces) and the line buffer in the test runner has a capacity of 1k lines. To avoid the same hang in the future, this change modifies the test runner to panic if the line buffer is full. ## Tests Confirmed the integration test no longer hangs. --- internal/completer_test.go | 2 +- internal/helpers.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/completer_test.go b/internal/completer_test.go index 0f7d2093d..b2c936886 100644 --- a/internal/completer_test.go +++ b/internal/completer_test.go @@ -21,7 +21,7 @@ func TestAccFsCompletion(t *testing.T) { f, tmpDir := setupDbfsFiler(t) setupCompletionFile(t, f) - stdout, _ := RequireSuccessfulRun(t, "__complete", "fs", "ls", tmpDir) + stdout, _ := RequireSuccessfulRun(t, "__complete", "fs", "ls", tmpDir+"/") expectedOutput := fmt.Sprintf("%s/dir1/\n:2\n", tmpDir) assert.Equal(t, expectedOutput, stdout.String()) } diff --git a/internal/helpers.go b/internal/helpers.go index 972a2322b..5d9aead1f 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -94,10 +94,16 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s defer wg.Done() scanner := bufio.NewScanner(r) for scanner.Scan() { + // We expect to be able to always send these lines into the channel. + // If we can't, it means the channel is full and likely there is a problem + // in either the test or the code under test. select { case <-ctx.Done(): return case ch <- scanner.Text(): + continue + default: + panic("line buffer is full") } } }() From 1b984b4f62f01945b239515fa1ae10e2814bd9d8 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:49:54 +0530 Subject: [PATCH 55/88] Skip pushing Terraform state after destroy (#1667) ## Changes Following up https://github.com/databricks/cli/pull/1583#discussion_r1681126323. We can skip pushing because right after `root_path` is deleted, making this a no-op effectively. ## Tests --- bundle/phases/destroy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 01b276670..6eb8b6a01 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -83,7 +83,6 @@ func Destroy() bundle.Mutator { // Core destructive mutators for destroy. These require informed user consent. destroyCore := bundle.Seq( terraform.Apply(), - terraform.StatePush(), files.Delete(), bundle.LogString("Destroy complete!"), ) From 26bb4b9c7577de8e4a50cee2c40e0d617665c788 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:03:13 +0200 Subject: [PATCH 56/88] Bump golang.org/x/text from 0.16.0 to 0.17.0 (#1670) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.16.0 to 0.17.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.16.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 24064c9e1..aa05ffbc5 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.22.0 - golang.org/x/text v0.16.0 + golang.org/x/text v0.17.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index bc8fd5f29..3f291b4ce 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,8 @@ golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From a38e16c6547b74edd2a0c8260c35c9869cc36f57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:49:53 +0200 Subject: [PATCH 57/88] Bump golang.org/x/term from 0.22.0 to 0.23.0 (#1669) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.22.0 to 0.23.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.22.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index aa05ffbc5..3f5af0815 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.22.0 + golang.org/x/term v0.23.0 golang.org/x/text v0.17.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -62,7 +62,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 3f291b4ce..f33a9562a 100644 --- a/go.sum +++ b/go.sum @@ -212,10 +212,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= From ad8e61c73925b9f1b24cbc63ac1bc5b51348dbe0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 12 Aug 2024 16:20:04 +0200 Subject: [PATCH 58/88] Fix ability to import the CLI repository as module (#1671) ## Changes While investigating #1629, I found that Go doesn't allow characters outside the set documented at https://pkg.go.dev/golang.org/x/mod/module#CheckFilePath. To fix this, I changed the relevant test case to create the fixtures it needs instead of loading it from the `testdata` directory (in `renderer_test.go`). Some test cases in `config_test.go` depended on templated paths without needing to do so. In the process of fixing this, I refactored these tests slightly to reduce dependencies between them. This change also adds a test case to ensure that all files in the repository are allowed to be part of a module (per the earlier `CheckFilePath` function). Fixes #1629. ## Tests I manually confirmed I could import the repository as a Go module. --- internal/testutil/copy.go | 48 ++++++ libs/template/config_test.go | 153 +++++++++++------- libs/template/renderer_test.go | 23 ++- .../schema.json | 24 +++ .../schema.json | 20 +++ .../schema.json | 20 +++ .../config-assign-from-file/schema.json | 20 +++ .../schema.json | 24 +++ .../template/library/my_funcs.tmpl | 3 + .../template/template/.gitkeep} | 0 .../config-test-schema/test-schema.json | 6 +- .../library/.gitkeep} | 0 .../template/testdata/empty/template/.gitkeep | 0 .../template-in-path/template/.gitkeep | 0 .../templated-defaults/library/my_funcs.tmpl | 7 - main_test.go | 25 +++ 16 files changed, 297 insertions(+), 76 deletions(-) create mode 100644 internal/testutil/copy.go create mode 100644 libs/template/testdata/config-assign-from-default-value/schema.json create mode 100644 libs/template/testdata/config-assign-from-file-invalid-int/schema.json create mode 100644 libs/template/testdata/config-assign-from-file-unknown-property/schema.json create mode 100644 libs/template/testdata/config-assign-from-file/schema.json create mode 100644 libs/template/testdata/config-assign-from-templated-default-value/schema.json create mode 100644 libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl rename libs/template/testdata/{template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} => config-assign-from-templated-default-value/template/template/.gitkeep} (100%) rename libs/template/testdata/{templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} => empty/library/.gitkeep} (100%) create mode 100644 libs/template/testdata/empty/template/.gitkeep create mode 100644 libs/template/testdata/template-in-path/template/.gitkeep delete mode 100644 libs/template/testdata/templated-defaults/library/my_funcs.tmpl diff --git a/internal/testutil/copy.go b/internal/testutil/copy.go new file mode 100644 index 000000000..21faece00 --- /dev/null +++ b/internal/testutil/copy.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// CopyDirectory copies the contents of a directory to another directory. +// The destination directory is created if it does not exist. +func CopyDirectory(t *testing.T, src, dst string) { + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(src, path) + require.NoError(t, err) + + if d.IsDir() { + return os.MkdirAll(filepath.Join(dst, rel), 0755) + } + + // Copy the file to the temporary directory + in, err := os.Open(path) + if err != nil { + return err + } + + defer in.Close() + + out, err := os.Create(filepath.Join(dst, rel)) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, in) + return err + }) + + require.NoError(t, err) +} diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 1af2e5f5a..73b47f289 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -3,59 +3,70 @@ package template import ( "context" "fmt" + "path/filepath" "testing" "text/template" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testConfig(t *testing.T) *config { - c, err := newConfig(context.Background(), "./testdata/config-test-schema/test-schema.json") - require.NoError(t, err) - return c -} - func TestTemplateConfigAssignValuesFromFile(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file" - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "hello", c.values["string_val"]) -} - -func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { - c := testConfig(t) - - err := c.assignValuesFromFile("./testdata/config-assign-from-file-invalid-int/config.json") - assert.EqualError(t, err, "failed to load config from file ./testdata/config-assign-from-file-invalid-int/config.json: failed to parse property int_val: cannot convert \"abc\" to an integer") + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } } func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + c.values = map[string]any{ "string_val": "this-is-not-overwritten", } - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) + } +} - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) +func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { + testDir := "./testdata/config-assign-from-file-invalid-int" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + assert.EqualError(t, err, fmt.Sprintf("failed to load config from file %s: failed to parse property int_val: cannot convert \"abc\" to an integer", filepath.Join(testDir, "config.json"))) } func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file-unknown-property" - err := c.assignValuesFromFile("./testdata/config-assign-from-file-unknown-property/config.json") + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) assert.NoError(t, err) // assert only the known property is loaded @@ -63,37 +74,66 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te assert.Equal(t, "i am a known property", c.values["string_val"]) } -func TestTemplateConfigAssignDefaultValues(t *testing.T) { - c := testConfig(t) +func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-default-value" ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", t.TempDir()) + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir()) require.NoError(t, err) err = c.assignDefaultValues(r) - assert.NoError(t, err) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } +} - assert.Len(t, c.values, 2) - assert.Equal(t, "my_file", c.values["string_val"]) - assert.Equal(t, int64(123), c.values["int_val"]) +func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-templated-default-value" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir()) + require.NoError(t, err) + + // Note: only the string value is templated. + // The JSON schema package doesn't allow using a string default for integer types. + err = c.assignDefaultValues(r) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "world", c.values["string_val"]) + } } func TestTemplateConfigValidateValuesDefined(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.0, "bool_val": false, } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. no value provided for required property string_val") } func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.1, @@ -101,12 +141,15 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.NoError(t, err) } func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "unknown_prop": 1, "int_val": 1, @@ -115,12 +158,15 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. property unknown_prop is not defined in the schema") } func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": "this-should-be-an-int", "float_val": 1.1, @@ -128,7 +174,7 @@ func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. incorrect type for property int_val: expected type integer, but value is \"this-should-be-an-int\"") } @@ -224,19 +270,6 @@ func TestTemplateEnumValidation(t *testing.T) { assert.NoError(t, c.validate()) } -func TestAssignDefaultValuesWithTemplatedDefaults(t *testing.T) { - c := testConfig(t) - ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/templated-defaults/template", "./testdata/templated-defaults/library", t.TempDir()) - require.NoError(t, err) - - err = c.assignDefaultValues(r) - assert.NoError(t, err) - assert.Equal(t, "my_file", c.values["string_val"]) -} - func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) { _, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json") assert.EqualError(t, err, "template property property-without-description is missing a description") diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index a8678a525..92133c5fe 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -16,6 +16,7 @@ import ( bundleConfig "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go" @@ -655,15 +656,27 @@ func TestRendererFileTreeRendering(t *testing.T) { func TestRendererSubTemplateInPath(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", tmpDir) + // Copy the template directory to a temporary directory where we can safely include a templated file path. + // These paths include characters that are forbidden in Go modules, so we can't use the testdata directory. + // Also see https://github.com/databricks/cli/pull/1671. + templateDir := t.TempDir() + testutil.CopyDirectory(t, "./testdata/template-in-path", templateDir) + + // Use a backtick-quoted string; double quotes are a reserved character for Windows paths: + // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file. + testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}")) + + tmpDir := t.TempDir() + r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir) require.NoError(t, err) err = r.walk() require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), r.files[0].DstPath().absPath()) - assert.Equal(t, "my_directory/my_file", r.files[0].DstPath().relPath) + if assert.Len(t, r.files, 2) { + f := r.files[1] + assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), f.DstPath().absPath()) + assert.Equal(t, "my_directory/my_file", f.DstPath().relPath) + } } diff --git a/libs/template/testdata/config-assign-from-default-value/schema.json b/libs/template/testdata/config-assign-from-default-value/schema.json new file mode 100644 index 000000000..259bb9a7f --- /dev/null +++ b/libs/template/testdata/config-assign-from-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "hello" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-invalid-int/schema.json b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-unknown-property/schema.json b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file/schema.json b/libs/template/testdata/config-assign-from-file/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/schema.json b/libs/template/testdata/config-assign-from-templated-default-value/schema.json new file mode 100644 index 000000000..fe664430b --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "{{ template \"string_val\" }}" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl new file mode 100644 index 000000000..41c50d7e5 --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl @@ -0,0 +1,3 @@ +{{define "string_val" -}} +world +{{- end}} diff --git a/libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} b/libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep similarity index 100% rename from libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} rename to libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep diff --git a/libs/template/testdata/config-test-schema/test-schema.json b/libs/template/testdata/config-test-schema/test-schema.json index 10f8652f4..80c44d6d9 100644 --- a/libs/template/testdata/config-test-schema/test-schema.json +++ b/libs/template/testdata/config-test-schema/test-schema.json @@ -2,8 +2,7 @@ "properties": { "int_val": { "type": "integer", - "description": "This is an integer value", - "default": 123 + "description": "This is an integer value" }, "float_val": { "type": "number", @@ -15,8 +14,7 @@ }, "string_val": { "type": "string", - "description": "This is a string value", - "default": "{{template \"file_name\"}}" + "description": "This is a string value" } } } diff --git a/libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} b/libs/template/testdata/empty/library/.gitkeep similarity index 100% rename from libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} rename to libs/template/testdata/empty/library/.gitkeep diff --git a/libs/template/testdata/empty/template/.gitkeep b/libs/template/testdata/empty/template/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/template-in-path/template/.gitkeep b/libs/template/testdata/template-in-path/template/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl b/libs/template/testdata/templated-defaults/library/my_funcs.tmpl deleted file mode 100644 index 3415ad774..000000000 --- a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -{{define "dir_name" -}} -my_directory -{{- end}} - -{{define "file_name" -}} -my_file -{{- end}} diff --git a/main_test.go b/main_test.go index 34ecdca0f..dea82e9b9 100644 --- a/main_test.go +++ b/main_test.go @@ -2,11 +2,14 @@ package main import ( "context" + "io/fs" + "path/filepath" "testing" "github.com/databricks/cli/cmd" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" ) func TestCommandsDontUseUnderscoreInName(t *testing.T) { @@ -23,3 +26,25 @@ func TestCommandsDontUseUnderscoreInName(t *testing.T) { queue = append(queue[1:], cmd.Commands()...) } } + +func TestFilePath(t *testing.T) { + // To import this repository as a library, all files must match the + // file path constraints made by Go. This test ensures that all files + // in the repository have a valid file path. + // + // See https://github.com/databricks/cli/issues/1629 + // + err := filepath.WalkDir(".", func(path string, _ fs.DirEntry, err error) error { + switch path { + case ".": + return nil + case ".git": + return filepath.SkipDir + } + if assert.NoError(t, err) { + assert.NoError(t, module.CheckFilePath(filepath.ToSlash(path))) + } + return nil + }) + assert.NoError(t, err) +} From 7ae80de351ae57f5dcc826d4ee3df7651dbe1f00 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 13 Aug 2024 18:20:15 +0530 Subject: [PATCH 59/88] Stop tracking file path locations in bundle resources (#1673) ## Changes Since locations are already tracked in the dynamic value tree, we no longer need to track it at the resource/artifact level. This PR: 1. Removes use of `paths.Paths`. Uses dyn.Location instead. 2. Refactors the validation of resources not being empty valued to be generic across all resource types. ## Tests Existing unit tests. --- bundle/artifacts/prepare.go | 5 +- bundle/config/artifact.go | 9 --- bundle/config/mutator/trampoline_test.go | 4 - bundle/config/paths/paths.go | 22 ------ bundle/config/resources.go | 77 ++----------------- bundle/config/resources/job.go | 12 --- bundle/config/resources/mlflow_experiment.go | 12 --- bundle/config/resources/mlflow_model.go | 12 --- .../resources/model_serving_endpoint.go | 14 ---- bundle/config/resources/pipeline.go | 12 --- bundle/config/resources/quality_monitor.go | 14 ---- bundle/config/resources/registered_model.go | 14 ---- bundle/config/root.go | 20 ----- .../validate/all_resources_have_values.go | 47 +++++++++++ bundle/deploy/metadata/compute.go | 3 +- bundle/internal/bundletest/location.go | 2 - bundle/phases/initialize.go | 2 + bundle/python/transform_test.go | 4 - .../environments_job_and_pipeline_test.go | 12 ++- bundle/tests/include_test.go | 9 ++- bundle/tests/job_and_pipeline_test.go | 5 -- bundle/tests/model_serving_endpoint_test.go | 2 - bundle/tests/registered_model_test.go | 2 - bundle/tests/undefined_job_test.go | 12 ++- .../tests/undefined_pipeline/databricks.yml | 8 ++ libs/dyn/convert/to_typed.go | 6 ++ 26 files changed, 98 insertions(+), 243 deletions(-) delete mode 100644 bundle/config/paths/paths.go create mode 100644 bundle/config/validate/all_resources_have_values.go create mode 100644 bundle/tests/undefined_pipeline/databricks.yml diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index 493e8f7a8..fb61ed9e2 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -34,11 +34,13 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diag.Errorf("artifact doesn't exist: %s", m.name) } + l := b.Config.GetLocation("artifacts." + m.name) + dirPath := filepath.Dir(l.File) + // Check if source paths are absolute, if not, make them absolute for k := range artifact.Files { f := &artifact.Files[k] if !filepath.IsAbs(f.Source) { - dirPath := filepath.Dir(artifact.ConfigFilePath) f.Source = filepath.Join(dirPath, f.Source) } } @@ -49,7 +51,6 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } if !filepath.IsAbs(artifact.Path) { - dirPath := filepath.Dir(artifact.ConfigFilePath) artifact.Path = filepath.Join(dirPath, artifact.Path) } diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index 219def571..9a5690f57 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -4,18 +4,11 @@ import ( "context" "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/exec" ) type Artifacts map[string]*Artifact -func (artifacts Artifacts) ConfigureConfigFilePath() { - for _, artifact := range artifacts { - artifact.ConfigureConfigFilePath() - } -} - type ArtifactType string const ArtifactPythonWheel ArtifactType = `whl` @@ -40,8 +33,6 @@ type Artifact struct { BuildCommand string `json:"build,omitempty"` Executable exec.ExecutableType `json:"executable,omitempty"` - - paths.Paths } func (a *Artifact) Build(ctx context.Context) ([]byte, error) { diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index e39076647..de395c165 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -65,9 +64,6 @@ func TestGenerateTrampoline(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: tasks, }, diff --git a/bundle/config/paths/paths.go b/bundle/config/paths/paths.go deleted file mode 100644 index 95977ee37..000000000 --- a/bundle/config/paths/paths.go +++ /dev/null @@ -1,22 +0,0 @@ -package paths - -import ( - "github.com/databricks/cli/libs/dyn" -) - -type Paths struct { - // Absolute path on the local file system to the configuration file that holds - // the definition of this resource. - ConfigFilePath string `json:"-" bundle:"readonly"` - - // DynamicValue stores the [dyn.Value] of the containing struct. - // This assumes that this struct is always embedded. - DynamicValue dyn.Value `json:"-"` -} - -func (p *Paths) ConfigureConfigFilePath() { - if !p.DynamicValue.IsValid() { - panic("DynamicValue not set") - } - p.ConfigFilePath = p.DynamicValue.Location().File -} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 6c7a927f2..22d69ffb5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -21,81 +21,14 @@ type Resources struct { Schemas map[string]*resources.Schema `json:"schemas,omitempty"` } -type resource struct { - resource ConfigResource - resource_type string - key string -} - -func (r *Resources) allResources() []resource { - all := make([]resource, 0) - for k, e := range r.Jobs { - all = append(all, resource{resource_type: "job", resource: e, key: k}) - } - for k, e := range r.Pipelines { - all = append(all, resource{resource_type: "pipeline", resource: e, key: k}) - } - for k, e := range r.Models { - all = append(all, resource{resource_type: "model", resource: e, key: k}) - } - for k, e := range r.Experiments { - all = append(all, resource{resource_type: "experiment", resource: e, key: k}) - } - for k, e := range r.ModelServingEndpoints { - all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k}) - } - for k, e := range r.RegisteredModels { - all = append(all, resource{resource_type: "registered model", resource: e, key: k}) - } - for k, e := range r.QualityMonitors { - all = append(all, resource{resource_type: "quality monitor", resource: e, key: k}) - } - return all -} - -func (r *Resources) VerifyAllResourcesDefined() error { - all := r.allResources() - for _, e := range all { - err := e.resource.Validate() - if err != nil { - return fmt.Errorf("%s %s is not defined", e.resource_type, e.key) - } - } - - return nil -} - -// ConfigureConfigFilePath sets the specified path for all resources contained in this instance. -// This property is used to correctly resolve paths relative to the path -// of the configuration file they were defined in. -func (r *Resources) ConfigureConfigFilePath() { - for _, e := range r.Jobs { - e.ConfigureConfigFilePath() - } - for _, e := range r.Pipelines { - e.ConfigureConfigFilePath() - } - for _, e := range r.Models { - e.ConfigureConfigFilePath() - } - for _, e := range r.Experiments { - e.ConfigureConfigFilePath() - } - for _, e := range r.ModelServingEndpoints { - e.ConfigureConfigFilePath() - } - for _, e := range r.RegisteredModels { - e.ConfigureConfigFilePath() - } - for _, e := range r.QualityMonitors { - e.ConfigureConfigFilePath() - } -} - type ConfigResource interface { + // Function to assert if the resource exists in the workspace configured in + // the input workspace client. Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) + + // Terraform equivalent name of the resource. For example "databricks_job" + // for jobs and "databricks_pipeline" for pipelines. TerraformResourceName() string - Validate() error } func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) { diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index dde5d5663..d8f97a2db 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,10 +2,8 @@ package resources import ( "context" - "fmt" "strconv" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -17,8 +15,6 @@ type Job struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *jobs.JobSettings } @@ -48,11 +44,3 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri func (j *Job) TerraformResourceName() string { return "databricks_job" } - -func (j *Job) Validate() error { - if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil { - return fmt.Errorf("job is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index 7854ee7e8..0ab486436 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowExperiment struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Experiment } @@ -43,11 +39,3 @@ func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceCl func (s *MlflowExperiment) TerraformResourceName() string { return "databricks_mlflow_experiment" } - -func (s *MlflowExperiment) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("experiment is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 40da9f87d..300474e35 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowModel struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Model } @@ -43,11 +39,3 @@ func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, func (s *MlflowModel) TerraformResourceName() string { return "databricks_mlflow_model" } - -func (s *MlflowModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("model is not defined") - } - - return nil -} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 503cfbbb7..5efb7ea26 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -20,10 +18,6 @@ type ModelServingEndpoint struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This is a resource agnostic implementation of permissions for ACLs. // Implementation could be different based on the resource type. Permissions []Permission `json:"permissions,omitempty"` @@ -53,11 +47,3 @@ func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.Workspa func (s *ModelServingEndpoint) TerraformResourceName() string { return "databricks_model_serving" } - -func (s *ModelServingEndpoint) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("serving endpoint is not defined") - } - - return nil -} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 7e914b909..55270be65 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type Pipeline struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *pipelines.PipelineSpec } @@ -43,11 +39,3 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (p *Pipeline) TerraformResourceName() string { return "databricks_pipeline" } - -func (p *Pipeline) Validate() error { - if p == nil || !p.DynamicValue.IsValid() { - return fmt.Errorf("pipeline is not defined") - } - - return nil -} diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go index 0d13e58fa..9160782cd 100644 --- a/bundle/config/resources/quality_monitor.go +++ b/bundle/config/resources/quality_monitor.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type QualityMonitor struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` } @@ -50,11 +44,3 @@ func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClie func (s *QualityMonitor) TerraformResourceName() string { return "databricks_quality_monitor" } - -func (s *QualityMonitor) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("quality monitor is not defined") - } - - return nil -} diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index fba643c69..6033ffdf2 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type RegisteredModel struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This represents the input args for terraform, and will get converted // to a HCL representation for CRUD *catalog.CreateRegisteredModelRequest @@ -54,11 +48,3 @@ func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceCli func (s *RegisteredModel) TerraformResourceName() string { return "databricks_registered_model" } - -func (s *RegisteredModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("registered model is not defined") - } - - return nil -} diff --git a/bundle/config/root.go b/bundle/config/root.go index cace22156..2c6fe1a4a 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -136,17 +136,6 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error { // Assign the normalized configuration tree. r.value = nv - - // At the moment the check has to be done as part of updateWithDynamicValue - // because otherwise ConfigureConfigFilePath will fail with a panic. - // In the future, we should move this check to a separate mutator in initialise phase. - err = r.Resources.VerifyAllResourcesDefined() - if err != nil { - return err - } - - // Assign config file paths after converting to typed configuration. - r.ConfigureConfigFilePath() return nil } @@ -238,15 +227,6 @@ func (r *Root) MarkMutatorExit(ctx context.Context) error { return nil } -// SetConfigFilePath configures the path that its configuration -// was loaded from in configuration leafs that require it. -func (r *Root) ConfigureConfigFilePath() { - r.Resources.ConfigureConfigFilePath() - if r.Artifacts != nil { - r.Artifacts.ConfigureConfigFilePath() - } -} - // Initializes variables using values passed from the command line flag // Input has to be a string of the form `foo=bar`. In this case the variable with // name `foo` is assigned the value `bar` diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go new file mode 100644 index 000000000..019fe48a2 --- /dev/null +++ b/bundle/config/validate/all_resources_have_values.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func AllResourcesHaveValues() bundle.Mutator { + return &allResourcesHaveValues{} +} + +type allResourcesHaveValues struct{} + +func (m *allResourcesHaveValues) Name() string { + return "validate:AllResourcesHaveValues" +} + +func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + rv := b.Config.Value().Get("resources") + + // Skip if there are no resources block defined, or the resources block is empty. + if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { + return nil + } + + _, err := dyn.MapByPattern( + rv, + dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() == dyn.KindInvalid || v.Kind() == dyn.KindNil { + // Type of the resource, stripped of the trailing 's' to make it + // singular. + rType := strings.TrimSuffix(p[0].Key(), "s") + + rName := p[1].Key() + return v, fmt.Errorf("%s %s is not defined", rType, rName) + } + return v, nil + }, + ) + return diag.FromErr(err) +} diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 034765484..6ab997e27 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -39,7 +39,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - relativePath, err := filepath.Rel(b.RootPath, job.ConfigFilePath) + l := b.Config.GetLocation("resources.jobs." + name) + relativePath, err := filepath.Rel(b.RootPath, l.File) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index ebec43d30..380d6e17d 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -29,6 +29,4 @@ func SetLocation(b *bundle.Bundle, prefix string, filePath string) { return v, dyn.ErrSkip }) }) - - b.Config.ConfigureConfigFilePath() } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 7b4dc6d41..fac3066dc 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/permissions" @@ -19,6 +20,7 @@ func Initialize() bundle.Mutator { return newPhase( "initialize", []bundle.Mutator{ + validate.AllResourcesHaveValues(), mutator.RewriteSyncPaths(), mutator.MergeJobClusters(), mutator.MergeJobParameters(), diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index c15feb424..c7bddca14 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -124,9 +123,6 @@ func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: []jobs.Task{ { diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index a18daf90c..0abeb487c 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -15,7 +15,8 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +30,8 @@ func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +44,16 @@ func TestJobAndPipelineProductionWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + pl := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(pl.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) + jl := b.Config.GetLocation("resources.jobs.pipeline_schedule") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(jl.File)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/include_test.go b/bundle/tests/include_test.go index 5b0235f60..15f8fcec1 100644 --- a/bundle/tests/include_test.go +++ b/bundle/tests/include_test.go @@ -31,7 +31,8 @@ func TestIncludeWithGlob(t *testing.T) { job := b.Config.Resources.Jobs["my_job"] assert.Equal(t, "1", job.ID) - assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(job.ConfigFilePath)) + l := b.Config.GetLocation("resources.jobs.my_job") + assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(l.File)) } func TestIncludeDefault(t *testing.T) { @@ -51,9 +52,11 @@ func TestIncludeForMultipleMatches(t *testing.T) { first := b.Config.Resources.Jobs["my_first_job"] assert.Equal(t, "1", first.ID) - assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(first.ConfigFilePath)) + fl := b.Config.GetLocation("resources.jobs.my_first_job") + assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(fl.File)) second := b.Config.Resources.Jobs["my_second_job"] assert.Equal(t, "2", second.ID) - assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(second.ConfigFilePath)) + sl := b.Config.GetLocation("resources.jobs.my_second_job") + assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(sl.File)) } diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 5e8febc33..65aa5bdc4 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -15,7 +14,6 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +27,6 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +39,12 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go index bfa1a31b4..b8b800863 100644 --- a/bundle/tests/model_serving_endpoint_test.go +++ b/bundle/tests/model_serving_endpoint_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) { - assert.Equal(t, "model_serving_endpoint/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "model-name", p.Config.ServedModels[0].ModelName) assert.Equal(t, "1", p.Config.ServedModels[0].ModelVersion) assert.Equal(t, "model-name-1", p.Config.TrafficConfig.Routes[0].ServedModelName) diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go index 920a2ac78..008db8bdd 100644 --- a/bundle/tests/registered_model_test.go +++ b/bundle/tests/registered_model_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) { - assert.Equal(t, "registered_model/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "main", p.CatalogName) assert.Equal(t, "default", p.SchemaName) assert.Equal(t, "comment", p.Comment) diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go index ed502c471..4596f2069 100644 --- a/bundle/tests/undefined_job_test.go +++ b/bundle/tests/undefined_job_test.go @@ -1,12 +1,22 @@ package config_tests import ( + "context" "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" "github.com/stretchr/testify/assert" ) func TestUndefinedJobLoadsWithError(t *testing.T) { - _, diags := loadTargetWithDiags("./undefined_job", "default") + b := load(t, "./undefined_job") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) assert.ErrorContains(t, diags.Error(), "job undefined is not defined") } + +func TestUndefinedPipelineLoadsWithError(t *testing.T) { + b := load(t, "./undefined_pipeline") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) + assert.ErrorContains(t, diags.Error(), "pipeline undefined is not defined") +} diff --git a/bundle/tests/undefined_pipeline/databricks.yml b/bundle/tests/undefined_pipeline/databricks.yml new file mode 100644 index 000000000..a52fda38c --- /dev/null +++ b/bundle/tests/undefined_pipeline/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: undefined-pipeline + +resources: + pipelines: + undefined: + test: + name: "Test Pipeline" diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 181c88cc9..839d0111a 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -9,6 +9,12 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" ) +// Populate a destination typed value from a source dynamic value. +// +// At any point while walking the destination type tree using +// reflection, if this function sees an exported field with type dyn.Value it +// will populate that field with the appropriate source dynamic value. +// see PR: https://github.com/databricks/cli/pull/1010 func ToTyped(dst any, src dyn.Value) error { dstv := reflect.ValueOf(dst) From 48ff18e5fc6c9b5a6d9cebf08b019af1e6f9ada0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 14 Aug 2024 11:03:44 +0200 Subject: [PATCH 60/88] Upload local libraries even if they don't have artifact defined (#1664) ## Changes Previously for all the libraries referenced in configuration DABs made sure that there is corresponding artifact section. But this is not really necessary and flexible, because local libraries might be built outside of dabs context. It also created difficult to follow logic in code where we back referenced libraries to artifacts which was difficult to fllow This PR does 3 things: 1. Allows all local libraries referenced in DABs config to be uploaded to remote 2. Simplifies upload and glob references expand logic by doing this in single place 3. Speed things up by uploading library only once and doing this in parallel ## Tests Added unit + integration tests + made sure that change is backward compatible (no changes in existing tests) --------- Co-authored-by: Pieter Noordhuis --- bundle/artifacts/artifacts.go | 191 ---------- bundle/artifacts/artifacts_test.go | 196 ----------- bundle/artifacts/autodetect.go | 1 - bundle/artifacts/upload.go | 38 +- bundle/artifacts/upload_test.go | 114 ------ bundle/artifacts/whl/from_libraries.go | 79 ----- bundle/config/mutator/translate_paths_jobs.go | 2 +- bundle/libraries/expand_glob_references.go | 221 ++++++++++++ .../libraries/expand_glob_references_test.go | 239 +++++++++++++ bundle/libraries/libraries.go | 4 +- bundle/libraries/local_path.go | 39 ++- bundle/libraries/local_path_test.go | 28 +- bundle/libraries/match.go | 82 ----- bundle/libraries/match_test.go | 12 +- bundle/libraries/upload.go | 238 +++++++++++++ bundle/libraries/upload_test.go | 331 ++++++++++++++++++ bundle/phases/deploy.go | 4 +- bundle/tests/enviroment_key_test.go | 2 +- .../bundle.yml | 7 - bundle/tests/python_wheel_test.go | 45 +-- internal/bundle/artifacts_test.go | 8 +- 21 files changed, 1103 insertions(+), 778 deletions(-) delete mode 100644 bundle/artifacts/artifacts_test.go delete mode 100644 bundle/artifacts/upload_test.go delete mode 100644 bundle/artifacts/whl/from_libraries.go create mode 100644 bundle/libraries/expand_glob_references.go create mode 100644 bundle/libraries/expand_glob_references_test.go delete mode 100644 bundle/libraries/match.go create mode 100644 bundle/libraries/upload.go create mode 100644 bundle/libraries/upload_test.go diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index 3060d08d9..e5e55a14d 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -1,25 +1,16 @@ package artifacts import ( - "bytes" "context" - "errors" "fmt" - "os" - "path" - "path/filepath" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts/whl" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" ) type mutatorFactory = func(name string) bundle.Mutator @@ -28,8 +19,6 @@ var buildMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactTy config.ArtifactPythonWheel: whl.Build, } -var uploadMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{} - var prepareMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{ config.ArtifactPythonWheel: whl.Prepare, } @@ -43,15 +32,6 @@ func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { return mutatorFactory(name) } -func getUploadMutator(t config.ArtifactType, name string) bundle.Mutator { - mutatorFactory, ok := uploadMutators[t] - if !ok { - mutatorFactory = BasicUpload - } - - return mutatorFactory(name) -} - func getPrepareMutator(t config.ArtifactType, name string) bundle.Mutator { mutatorFactory, ok := prepareMutators[t] if !ok { @@ -92,174 +72,3 @@ func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return nil } - -// Basic Upload defines a general upload mutator which uploads artifact as a library to workspace -type basicUpload struct { - name string -} - -func BasicUpload(name string) bundle.Mutator { - return &basicUpload{name: name} -} - -func (m *basicUpload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - uploadPath, err := getUploadBasePath(b) - if err != nil { - return diag.FromErr(err) - } - - client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) - if err != nil { - return diag.FromErr(err) - } - - err = uploadArtifact(ctx, b, artifact, uploadPath, client) - if err != nil { - return diag.Errorf("upload for %s failed, error: %v", m.name, err) - } - - return nil -} - -func getFilerForArtifacts(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { - if isVolumesPath(uploadPath) { - return filer.NewFilesClient(w, uploadPath) - } - return filer.NewWorkspaceFilesClient(w, uploadPath) -} - -func isVolumesPath(path string) bool { - return strings.HasPrefix(path, "/Volumes/") -} - -func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, uploadPath string, client filer.Filer) error { - for i := range a.Files { - f := &a.Files[i] - - filename := filepath.Base(f.Source) - cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) - - err := uploadArtifactFile(ctx, f.Source, client) - if err != nil { - return err - } - - log.Infof(ctx, "Upload succeeded") - f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source)) - remotePath := f.RemotePath - - if !strings.HasPrefix(f.RemotePath, "/Workspace/") && !strings.HasPrefix(f.RemotePath, "/Volumes/") { - wsfsBase := "/Workspace" - remotePath = path.Join(wsfsBase, f.RemotePath) - } - - for _, job := range b.Config.Resources.Jobs { - rewriteArtifactPath(b, f, job, remotePath) - } - } - - return nil -} - -func rewriteArtifactPath(b *bundle.Bundle, f *config.ArtifactFile, job *resources.Job, remotePath string) { - // Rewrite artifact path in job task libraries - for i := range job.Tasks { - task := &job.Tasks[i] - for j := range task.Libraries { - lib := &task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - - // Rewrite artifact path in job task libraries for ForEachTask - if task.ForEachTask != nil { - forEachTask := task.ForEachTask - for j := range forEachTask.Task.Libraries { - lib := &forEachTask.Task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - } - } - - // Rewrite artifact path in job environments - for i := range job.Environments { - env := &job.Environments[i] - if env.Spec == nil { - continue - } - - for j := range env.Spec.Dependencies { - lib := env.Spec.Dependencies[j] - if isArtifactMatchLibrary(f, lib, b) { - env.Spec.Dependencies[j] = remotePath - } - } - } -} - -func isArtifactMatchLibrary(f *config.ArtifactFile, libPath string, b *bundle.Bundle) bool { - if !filepath.IsAbs(libPath) { - libPath = filepath.Join(b.RootPath, libPath) - } - - // libPath can be a glob pattern, so do the match first - matches, err := filepath.Glob(libPath) - if err != nil { - return false - } - - for _, m := range matches { - if m == f.Source { - return true - } - } - - return false -} - -// Function to upload artifact file to Workspace -func uploadArtifactFile(ctx context.Context, file string, client filer.Filer) error { - raw, err := os.ReadFile(file) - if err != nil { - return fmt.Errorf("unable to read %s: %w", file, errors.Unwrap(err)) - } - - filename := filepath.Base(file) - err = client.Write(ctx, filename, bytes.NewReader(raw), filer.OverwriteIfExists, filer.CreateParentDirectories) - if err != nil { - return fmt.Errorf("unable to import %s: %w", filename, err) - } - - return nil -} - -func getUploadBasePath(b *bundle.Bundle) (string, error) { - artifactPath := b.Config.Workspace.ArtifactPath - if artifactPath == "" { - return "", fmt.Errorf("remote artifact path not configured") - } - - return path.Join(artifactPath, ".internal"), nil -} diff --git a/bundle/artifacts/artifacts_test.go b/bundle/artifacts/artifacts_test.go deleted file mode 100644 index 6d85f3af9..000000000 --- a/bundle/artifacts/artifacts_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package artifacts - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestArtifactUploadForWorkspace(t *testing.T) { - tmpDir := t.TempDir() - whlFolder := filepath.Join(tmpDir, "whl") - testutil.Touch(t, whlFolder, "source.whl") - whlLocalPath := filepath.Join(whlFolder, "source.whl") - - b := &bundle.Bundle{ - RootPath: tmpDir, - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/foo/bar/artifacts", - }, - Artifacts: config.Artifacts{ - "whl": { - Type: config.ArtifactPythonWheel, - Files: []config.ArtifactFile{ - {Source: whlLocalPath}, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - { - ForEachTask: &jobs.ForEachTask{ - Task: jobs.Task{ - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - Environments: []jobs.JobEnvironment{ - { - Spec: &compute.Environment{ - Dependencies: []string{ - filepath.Join("whl", "source.whl"), - "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - artifact := b.Config.Artifacts["whl"] - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("source.whl"), - mock.AnythingOfType("*bytes.Reader"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - err := uploadArtifact(context.Background(), b, artifact, "/foo/bar/artifacts", mockFiler) - require.NoError(t, err) - - // Test that libraries path is updated - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) -} - -func TestArtifactUploadForVolumes(t *testing.T) { - tmpDir := t.TempDir() - whlFolder := filepath.Join(tmpDir, "whl") - testutil.Touch(t, whlFolder, "source.whl") - whlLocalPath := filepath.Join(whlFolder, "source.whl") - - b := &bundle.Bundle{ - RootPath: tmpDir, - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/foo/bar/artifacts", - }, - Artifacts: config.Artifacts{ - "whl": { - Type: config.ArtifactPythonWheel, - Files: []config.ArtifactFile{ - {Source: whlLocalPath}, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Volumes/some/path/mywheel.whl", - }, - }, - }, - { - ForEachTask: &jobs.ForEachTask{ - Task: jobs.Task{ - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Volumes/some/path/mywheel.whl", - }, - }, - }, - }, - }, - }, - Environments: []jobs.JobEnvironment{ - { - Spec: &compute.Environment{ - Dependencies: []string{ - filepath.Join("whl", "source.whl"), - "/Volumes/some/path/mywheel.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - artifact := b.Config.Artifacts["whl"] - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("source.whl"), - mock.AnythingOfType("*bytes.Reader"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - err := uploadArtifact(context.Background(), b, artifact, "/Volumes/foo/bar/artifacts", mockFiler) - require.NoError(t, err) - - // Test that libraries path is updated - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) -} diff --git a/bundle/artifacts/autodetect.go b/bundle/artifacts/autodetect.go index 0e94edd82..569a480f0 100644 --- a/bundle/artifacts/autodetect.go +++ b/bundle/artifacts/autodetect.go @@ -29,6 +29,5 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return bundle.Apply(ctx, b, bundle.Seq( whl.DetectPackage(), - whl.DefineArtifactsFromLibraries(), )) } diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 3af50021e..58c006dc1 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -2,50 +2,18 @@ package artifacts import ( "context" - "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) -func UploadAll() bundle.Mutator { - return &all{ - name: "Upload", - fn: uploadArtifactByName, - } -} - func CleanUp() bundle.Mutator { return &cleanUp{} } -type upload struct { - name string -} - -func uploadArtifactByName(name string) (bundle.Mutator, error) { - return &upload{name}, nil -} - -func (m *upload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name)) -} - type cleanUp struct{} func (m *cleanUp) Name() string { @@ -53,12 +21,12 @@ func (m *cleanUp) Name() string { } func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - uploadPath, err := getUploadBasePath(b) + uploadPath, err := libraries.GetUploadBasePath(b) if err != nil { return diag.FromErr(err) } - client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) + client, err := libraries.GetFilerForLibraries(b.WorkspaceClient(), uploadPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go deleted file mode 100644 index 202086bd3..000000000 --- a/bundle/artifacts/upload_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package artifacts - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/testfile" - "github.com/stretchr/testify/require" -) - -type noop struct{} - -func (n *noop) Apply(context.Context, *bundle.Bundle) diag.Diagnostics { - return nil -} - -func (n *noop) Name() string { - return "noop" -} - -func TestExpandGlobFilesSource(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar")) - t1.Close(t) - - t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar")) - t2.Close(t) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "*.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - pm := &prepare{"test"} - prepareMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(pm, bm, u)) - require.NoError(t, diags.Error()) - - require.Equal(t, 2, len(b.Config.Artifacts["test"].Files)) - require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source) - require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source) -} - -func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "myjar.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.ErrorContains(t, diags.Error(), "no matching files") -} diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go deleted file mode 100644 index 79161a82d..000000000 --- a/bundle/artifacts/whl/from_libraries.go +++ /dev/null @@ -1,79 +0,0 @@ -package whl - -import ( - "context" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" -) - -type fromLibraries struct{} - -func DefineArtifactsFromLibraries() bundle.Mutator { - return &fromLibraries{} -} - -func (m *fromLibraries) Name() string { - return "artifacts.whl.DefineArtifactsFromLibraries" -} - -func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if len(b.Config.Artifacts) != 0 { - log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") - return nil - } - - tasks := libraries.FindTasksWithLocalLibraries(b) - for _, task := range tasks { - // Skip tasks that are not PythonWheelTasks for now, we can later support Jars too - if task.PythonWheelTask == nil { - continue - } - - for _, lib := range task.Libraries { - matchAndAdd(ctx, lib.Whl, b) - } - } - - envs := libraries.FindAllEnvironments(b) - for _, jobEnvs := range envs { - for _, env := range jobEnvs { - if env.Spec != nil { - for _, dep := range env.Spec.Dependencies { - if libraries.IsEnvironmentDependencyLocal(dep) { - matchAndAdd(ctx, dep, b) - } - } - } - } - } - - return nil -} - -func matchAndAdd(ctx context.Context, lib string, b *bundle.Bundle) { - matches, err := filepath.Glob(filepath.Join(b.RootPath, lib)) - // File referenced from libraries section does not exists, skipping - if err != nil { - return - } - - for _, match := range matches { - name := filepath.Base(match) - if b.Config.Artifacts == nil { - b.Config.Artifacts = make(map[string]*config.Artifact) - } - - log.Debugf(ctx, "Adding an artifact block for %s", match) - b.Config.Artifacts[name] = &config.Artifact{ - Files: []config.ArtifactFile{ - {Source: match}, - }, - Type: config.ArtifactPythonWheel, - } - } -} diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 60cc8bb9a..6febf4f8f 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -78,7 +78,7 @@ func (t *translateContext) jobRewritePatterns() []jobRewritePattern { ), t.translateNoOpWithPrefix, func(s string) bool { - return !libraries.IsEnvironmentDependencyLocal(s) + return !libraries.IsLibraryLocal(s) }, }, } diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go new file mode 100644 index 000000000..9e90a2a17 --- /dev/null +++ b/bundle/libraries/expand_glob_references.go @@ -0,0 +1,221 @@ +package libraries + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expand struct { +} + +func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { + return diag.Diagnostic{ + Severity: diag.Error, + Summary: message, + Paths: []dyn.Path{ + p.Append(), + }, + Locations: l, + } +} + +func getLibDetails(v dyn.Value) (string, string, bool) { + m := v.MustMap() + whl, ok := m.GetByString("whl") + if ok { + return whl.MustString(), "whl", true + } + + jar, ok := m.GetByString("jar") + if ok { + return jar.MustString(), "jar", true + } + + return "", "", false +} + +func findMatches(b *bundle.Bundle, path string) ([]string, error) { + matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) + if err != nil { + return nil, err + } + + if len(matches) == 0 { + if isGlobPattern(path) { + return nil, fmt.Errorf("no files match pattern: %s", path) + } else { + return nil, fmt.Errorf("file doesn't exist %s", path) + } + } + + // We make the matched path relative to the root path before storing it + // to allow upload mutator to distinguish between local and remote paths + for i, match := range matches { + matches[i], err = filepath.Rel(b.RootPath, match) + if err != nil { + return nil, err + } + } + + return matches, nil +} + +// Checks if the path is a glob pattern +// It can contain *, [] or ? characters +func isGlobPattern(path string) bool { + return strings.ContainsAny(path, "*?[") +} + +func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + libs := v.MustSequence() + for i, lib := range libs { + lp := p.Append(dyn.Index(i)) + path, libType, supported := getLibDetails(lib) + if !supported || !IsLibraryLocal(path) { + output = append(output, lib) + continue + } + + lp = lp.Append(dyn.Key(libType)) + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(map[string]dyn.Value{ + libType: dyn.V(match), + }, lib.Locations())) + } + } + + return diags, output +} + +func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + deps := v.MustSequence() + for i, dep := range deps { + lp := p.Append(dyn.Index(i)) + path := dep.MustString() + if !IsLibraryLocal(path) { + output = append(output, dep) + continue + } + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(match, dep.Locations())) + } + } + + return diags, output +} + +type expandPattern struct { + pattern dyn.Pattern + fn func(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) +} + +var taskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("libraries"), +) + +var forEachTaskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("for_each_task"), + dyn.Key("task"), + dyn.Key("libraries"), +) + +var envDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("dependencies"), +) + +func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + expanders := []expandPattern{ + { + pattern: taskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: forEachTaskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: envDepsPattern, + fn: expandEnvironmentDeps, + }, + } + + var diags diag.Diagnostics + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, expander := range expanders { + v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + d, output := expander.fn(b, p, lv) + diags = diags.Extend(d) + return dyn.V(output), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (e *expand) Name() string { + return "libraries.ExpandGlobReferences" +} + +// ExpandGlobReferences expands any glob references in the libraries or environments section +// to corresponding local paths. +// We only expand local paths (i.e. paths that are relative to the root path). +// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to +// distinguish between local and remote paths. +func ExpandGlobReferences() bundle.Mutator { + return &expand{} +} diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go new file mode 100644 index 000000000..34855b539 --- /dev/null +++ b/bundle/libraries/expand_glob_references_test.go @@ -0,0 +1,239 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0] + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0].ForEachTask.Task + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + EnvironmentKey: "env", + }, + }, + Environments: []jobs.JobEnvironment{ + { + EnvironmentKey: "env", + Spec: &compute.Environment{ + Dependencies: []string{ + "./whl/*.whl", + "/Workspace/path/to/whl/my.whl", + "./jar/*.jar", + "/some/local/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + env := job.JobSettings.Environments[0] + require.Equal(t, []string{ + filepath.Join("whl", "my1.whl"), + filepath.Join("whl", "my2.whl"), + "/Workspace/path/to/whl/my.whl", + filepath.Join("jar", "my1.jar"), + filepath.Join("jar", "my2.jar"), + "/some/local/path/to/whl/*.whl", + }, env.Spec.Dependencies) +} diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 72e5bcc66..33b848dd9 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -35,7 +35,7 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { } for _, l := range e.Spec.Dependencies { - if IsEnvironmentDependencyLocal(l) { + if IsLibraryLocal(l) { return true } } @@ -67,7 +67,7 @@ func FindTasksWithLocalLibraries(b *bundle.Bundle) []jobs.Task { func isTaskWithLocalLibraries(task jobs.Task) bool { for _, l := range task.Libraries { - if IsLocalLibrary(&l) { + if IsLibraryLocal(libraryPath(&l)) { return true } } diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index f1e3788f2..5b5ec6c07 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -4,8 +4,6 @@ import ( "net/url" "path" "strings" - - "github.com/databricks/databricks-sdk-go/service/compute" ) // IsLocalPath returns true if the specified path indicates that @@ -38,12 +36,12 @@ func IsLocalPath(p string) bool { return !path.IsAbs(p) } -// IsEnvironmentDependencyLocal returns true if the specified dependency +// IsLibraryLocal returns true if the specified library or environment dependency // should be interpreted as a local path. -// We use this to check if the dependency in environment spec is local. +// We use this to check if the dependency in environment spec is local or that library is local. // We can't use IsLocalPath beacuse environment dependencies can be // a pypi package name which can be misinterpreted as a local path by IsLocalPath. -func IsEnvironmentDependencyLocal(dep string) bool { +func IsLibraryLocal(dep string) bool { possiblePrefixes := []string{ ".", } @@ -54,7 +52,22 @@ func IsEnvironmentDependencyLocal(dep string) bool { } } - return false + // If the dependency is a requirements file, it's not a valid local path + if strings.HasPrefix(dep, "-r") { + return false + } + + // If the dependency has no extension, it's a PyPi package name + if isPackage(dep) { + return false + } + + return IsLocalPath(dep) +} + +func isPackage(name string) bool { + // If the dependency has no extension, it's a PyPi package name + return path.Ext(name) == "" } func isRemoteStorageScheme(path string) bool { @@ -67,16 +80,6 @@ func isRemoteStorageScheme(path string) bool { return false } - // If the path starts with scheme:/ format, it's a correct remote storage scheme - return strings.HasPrefix(path, url.Scheme+":/") -} - -// IsLocalLibrary returns true if the specified library refers to a local path. -func IsLocalLibrary(library *compute.Library) bool { - path := libraryPath(library) - if path == "" { - return false - } - - return IsLocalPath(path) + // If the path starts with scheme:/ format (not file), it's a correct remote storage scheme + return strings.HasPrefix(path, url.Scheme+":/") && url.Scheme != "file" } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index d2492d6b1..be4028d52 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -3,13 +3,13 @@ package libraries import ( "testing" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestIsLocalPath(t *testing.T) { // Relative paths, paths with the file scheme, and Windows paths. + assert.True(t, IsLocalPath("some/local/path")) assert.True(t, IsLocalPath("./some/local/path")) assert.True(t, IsLocalPath("file://path/to/package")) assert.True(t, IsLocalPath("C:\\path\\to\\package")) @@ -30,24 +30,13 @@ func TestIsLocalPath(t *testing.T) { assert.False(t, IsLocalPath("abfss://path/to/package")) } -func TestIsLocalLibrary(t *testing.T) { - // Local paths. - assert.True(t, IsLocalLibrary(&compute.Library{Whl: "./file.whl"})) - assert.True(t, IsLocalLibrary(&compute.Library{Jar: "../target/some.jar"})) - - // Non-local paths. - assert.False(t, IsLocalLibrary(&compute.Library{Whl: "/Workspace/path/to/file.whl"})) - assert.False(t, IsLocalLibrary(&compute.Library{Jar: "s3:/bucket/path/some.jar"})) - - // Empty. - assert.False(t, IsLocalLibrary(&compute.Library{})) -} - -func TestIsEnvironmentDependencyLocal(t *testing.T) { +func TestIsLibraryLocal(t *testing.T) { testCases := [](struct { path string expected bool }){ + {path: "local/*.whl", expected: true}, + {path: "local/test.whl", expected: true}, {path: "./local/*.whl", expected: true}, {path: ".\\local\\*.whl", expected: true}, {path: "./local/mypath.whl", expected: true}, @@ -58,15 +47,16 @@ func TestIsEnvironmentDependencyLocal(t *testing.T) { {path: ".\\..\\local\\*.whl", expected: true}, {path: "../../local/*.whl", expected: true}, {path: "..\\..\\local\\*.whl", expected: true}, + {path: "file://path/to/package/whl.whl", expected: true}, {path: "pypipackage", expected: false}, - {path: "pypipackage/test.whl", expected: false}, - {path: "pypipackage/*.whl", expected: false}, {path: "/Volumes/catalog/schema/volume/path.whl", expected: false}, {path: "/Workspace/my_project/dist.whl", expected: false}, {path: "-r /Workspace/my_project/requirements.txt", expected: false}, + {path: "s3://mybucket/path/to/package", expected: false}, + {path: "dbfs:/mnt/path/to/package", expected: false}, } - for _, tc := range testCases { - require.Equal(t, IsEnvironmentDependencyLocal(tc.path), tc.expected) + for i, tc := range testCases { + require.Equalf(t, tc.expected, IsLibraryLocal(tc.path), "failed case: %d, path: %s", i, tc.path) } } diff --git a/bundle/libraries/match.go b/bundle/libraries/match.go deleted file mode 100644 index 4feb4225d..000000000 --- a/bundle/libraries/match.go +++ /dev/null @@ -1,82 +0,0 @@ -package libraries - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -type match struct { -} - -func ValidateLocalLibrariesExist() bundle.Mutator { - return &match{} -} - -func (a *match) Name() string { - return "libraries.ValidateLocalLibrariesExist" -} - -func (a *match) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - for _, job := range b.Config.Resources.Jobs { - err := validateEnvironments(job.Environments, b) - if err != nil { - return diag.FromErr(err) - } - - for _, task := range job.JobSettings.Tasks { - err := validateTaskLibraries(task.Libraries, b) - if err != nil { - return diag.FromErr(err) - } - } - } - - return nil -} - -func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error { - for _, lib := range libs { - path := libraryPath(&lib) - if path == "" || !IsLocalPath(path) { - continue - } - - matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) - if err != nil { - return err - } - - if len(matches) == 0 { - return fmt.Errorf("file %s is referenced in libraries section but doesn't exist on the local file system", libraryPath(&lib)) - } - } - - return nil -} - -func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error { - for _, env := range envs { - if env.Spec == nil { - continue - } - - for _, dep := range env.Spec.Dependencies { - matches, err := filepath.Glob(filepath.Join(b.RootPath, dep)) - if err != nil { - return err - } - - if len(matches) == 0 && IsEnvironmentDependencyLocal(dep) { - return fmt.Errorf("file %s is referenced in environments section but doesn't exist on the local file system", dep) - } - } - } - - return nil -} diff --git a/bundle/libraries/match_test.go b/bundle/libraries/match_test.go index bb4b15107..e60504c84 100644 --- a/bundle/libraries/match_test.go +++ b/bundle/libraries/match_test.go @@ -42,7 +42,7 @@ func TestValidateEnvironments(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -74,9 +74,9 @@ func TestValidateEnvironmentsNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in environments section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } func TestValidateTaskLibraries(t *testing.T) { @@ -109,7 +109,7 @@ func TestValidateTaskLibraries(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -142,7 +142,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in libraries section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go new file mode 100644 index 000000000..be7cc41db --- /dev/null +++ b/bundle/libraries/upload.go @@ -0,0 +1,238 @@ +package libraries + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + + "github.com/databricks/databricks-sdk-go" + + "golang.org/x/sync/errgroup" +) + +// The Files API backend has a rate limit of 10 concurrent +// requests and 100 QPS. We limit the number of concurrent requests to 5 to +// avoid hitting the rate limit. +var maxFilesRequestsInFlight = 5 + +func Upload() bundle.Mutator { + return &upload{} +} + +func UploadWithClient(client filer.Filer) bundle.Mutator { + return &upload{ + client: client, + } +} + +type upload struct { + client filer.Filer +} + +type configLocation struct { + configPath dyn.Path + location dyn.Location +} + +// Collect all libraries from the bundle configuration and their config paths. +// By this stage all glob references are expanded and we have a list of all libraries that need to be uploaded. +// We collect them from task libraries, foreach task libraries, environment dependencies, and artifacts. +// We return a map of library source to a list of config paths and locations where the library is used. +// We use map so we don't upload the same library multiple times. +// Instead we upload it once and update all the config paths to point to the uploaded location. +func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error) { + libs := make(map[string]([]configLocation)) + + patterns := []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + envDepsPattern.Append(dyn.AnyIndex()), + } + + for _, pattern := range patterns { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + source, ok := v.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + if !IsLibraryLocal(source) { + return v, nil + } + + source = filepath.Join(b.RootPath, source) + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(), // Hack to get the copy of path + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + } + + artifactPattern := dyn.NewPattern( + dyn.Key("artifacts"), + dyn.AnyKey(), + dyn.Key("files"), + dyn.AnyIndex(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, artifactPattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + file, ok := v.AsMap() + if !ok { + return v, fmt.Errorf("expected map, got %s", v.Kind()) + } + + sv, ok := file.GetByString("source") + if !ok { + return v, nil + } + + source, ok := sv.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(dyn.Key("remote_path")), + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + + return libs, nil +} + +func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + uploadPath, err := GetUploadBasePath(b) + if err != nil { + return diag.FromErr(err) + } + + // If the client is not initialized, initialize it + // We use client field in mutator to allow for mocking client in testing + if u.client == nil { + filer, err := GetFilerForLibraries(b.WorkspaceClient(), uploadPath) + if err != nil { + return diag.FromErr(err) + } + + u.client = filer + } + + var diags diag.Diagnostics + + libs, err := collectLocalLibraries(b) + if err != nil { + return diag.FromErr(err) + } + + errs, errCtx := errgroup.WithContext(ctx) + errs.SetLimit(maxFilesRequestsInFlight) + + for source := range libs { + errs.Go(func() error { + return UploadFile(errCtx, source, u.client) + }) + } + + if err := errs.Wait(); err != nil { + return diag.FromErr(err) + } + + // Update all the config paths to point to the uploaded location + for source, locations := range libs { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + remotePath := path.Join(uploadPath, filepath.Base(source)) + + // If the remote path does not start with /Workspace or /Volumes, prepend /Workspace + if !strings.HasPrefix(remotePath, "/Workspace") && !strings.HasPrefix(remotePath, "/Volumes") { + remotePath = "/Workspace" + remotePath + } + for _, location := range locations { + v, err = dyn.SetByPath(v, location.configPath, dyn.NewValue(remotePath, []dyn.Location{location.location})) + if err != nil { + return v, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + } + + return diags +} + +func (u *upload) Name() string { + return "libraries.Upload" +} + +func GetFilerForLibraries(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { + if isVolumesPath(uploadPath) { + return filer.NewFilesClient(w, uploadPath) + } + return filer.NewWorkspaceFilesClient(w, uploadPath) +} + +func isVolumesPath(path string) bool { + return strings.HasPrefix(path, "/Volumes/") +} + +// Function to upload file (a library, artifact and etc) to Workspace or UC volume +func UploadFile(ctx context.Context, file string, client filer.Filer) error { + filename := filepath.Base(file) + cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) + + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("unable to open %s: %w", file, errors.Unwrap(err)) + } + defer f.Close() + + err = client.Write(ctx, filename, f, filer.OverwriteIfExists, filer.CreateParentDirectories) + if err != nil { + return fmt.Errorf("unable to import %s: %w", filename, err) + } + + log.Infof(ctx, "Upload succeeded") + return nil +} + +func GetUploadBasePath(b *bundle.Bundle) (string, error) { + artifactPath := b.Config.Workspace.ArtifactPath + if artifactPath == "" { + return "", fmt.Errorf("remote artifact path not configured") + } + + return path.Join(artifactPath, ".internal"), nil +} diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go new file mode 100644 index 000000000..82fe6e7c7 --- /dev/null +++ b/bundle/libraries/upload_test.go @@ -0,0 +1,331 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestArtifactUploadForWorkspace(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadForVolumes(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadWithNoLibraryReference(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Workspace/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Artifacts["whl"].Files[0].RemotePath) +} + +func TestUploadMultipleLibraries(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source1.whl") + testutil.Touch(t, whlFolder, "source2.whl") + testutil.Touch(t, whlFolder, "source3.whl") + testutil.Touch(t, whlFolder, "source4.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "*.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source1.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source2.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source3.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source4.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source1.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source2.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source3.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source4.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/Users/foo@bar.com/mywheel.whl"}) + + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source1.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source2.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source3.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source4.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/Users/foo@bar.com/mywheel.whl") +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 6929f74ba..ca967c321 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -113,9 +113,9 @@ func Deploy() bundle.Mutator { terraform.StatePull(), deploy.StatePull(), mutator.ValidateGitDetails(), - libraries.ValidateLocalLibrariesExist(), artifacts.CleanUp(), - artifacts.UploadAll(), + libraries.ExpandGlobReferences(), + libraries.Upload(), python.TransformWheelTask(), files.Upload(), deploy.StateUpdate(), diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index aed3964db..135ef1917 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -18,6 +18,6 @@ func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) - diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist()) + diags = bundle.Apply(context.Background(), b, libraries.ExpandGlobReferences()) require.Empty(t, diags) } diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml index 1bac4ebad..492861969 100644 --- a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml @@ -13,10 +13,3 @@ resources: entry_point: "run" libraries: - whl: ./package/*.whl - - task_key: TestTask2 - existing_cluster_id: "0717-aaaaa-bbbbbb" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./non-existing/*.whl diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 05e4fdfaf..c4d85703c 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -8,6 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/phases" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -23,7 +26,7 @@ func TestPythonWheelBuild(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -40,7 +43,7 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -57,7 +60,7 @@ func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -70,7 +73,7 @@ func TestPythonWheelWithDBFSLib(t *testing.T) { diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) require.NoError(t, diags.Error()) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -80,21 +83,23 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup") require.NoError(t, err) - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + b.Config.Workspace.ArtifactPath = "/foo/bar" + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + u := libraries.UploadWithClient(mockFiler) + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u)) require.NoError(t, diags.Error()) + require.Empty(t, diags) - match := libraries.ValidateLocalLibrariesExist() - diags = bundle.Apply(ctx, b, match) - require.ErrorContains(t, diags.Error(), "./non-existing/*.whl") - - require.NotZero(t, len(b.Config.Artifacts)) - - artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] - require.NotNil(t, artifact) - require.Empty(t, artifact.BuildCommand) - require.Contains(t, artifact.Files[0].Source, filepath.Join(b.RootPath, "package", - "my_test_code-0.0.1-py3-none-any.whl", - )) + require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { @@ -109,7 +114,7 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -126,7 +131,7 @@ func TestPythonWheelBuildMultiple(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -139,7 +144,7 @@ func TestPythonWheelNoBuild(t *testing.T) { diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) require.NoError(t, diags.Error()) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 46c236a4e..bae8073fc 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" "github.com/databricks/databricks-sdk-go/service/compute" @@ -74,7 +74,7 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -138,7 +138,7 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -207,7 +207,7 @@ func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. From 1225fc0c13edde6cd267d7523dd4bfa00621fa82 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 14 Aug 2024 18:31:00 +0530 Subject: [PATCH 61/88] Fix host resolution order in `auth login` (#1370) ## Changes The `auth login` command today prefers a host URL specified in a profile before selecting the one explicitly provided by a user as a command line argument. This PR fixes this bug and refactors the code to make it more linear and easy to read. Note that the same issue exists in the `auth token` command and is fixed here as well. ## Tests Unit tests, and manual testing. --- cmd/auth/auth.go | 27 ++++++------ cmd/auth/login.go | 70 +++++++++++++++++++++----------- cmd/auth/login_test.go | 68 +++++++++++++++++++++++++++++++ cmd/auth/testdata/.databrickscfg | 9 ++++ libs/auth/oauth.go | 1 - 5 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 cmd/auth/testdata/.databrickscfg diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index 79e1063b1..ceceae25c 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -2,6 +2,7 @@ package auth import ( "context" + "fmt" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" @@ -34,25 +35,23 @@ GCP: https://docs.gcp.databricks.com/dev-tools/auth/index.html`, } func promptForHost(ctx context.Context) (string, error) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Host (e.g. https://.cloud.databricks.com)" - // Validate? - host, err := prompt.Run() - if err != nil { - return "", err + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a host using --host") } - return host, nil + + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks host (e.g. https://.cloud.databricks.com)" + return prompt.Run() } func promptForAccountID(ctx context.Context) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify an account ID using --account-id") + } + prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Account ID" + prompt.Label = "Databricks account ID" prompt.Default = "" prompt.AllowEdit = true - // Validate? - accountId, err := prompt.Run() - if err != nil { - return "", err - } - return accountId, nil + return prompt.Run() } diff --git a/cmd/auth/login.go b/cmd/auth/login.go index 11cba8e5f..f87a2a027 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -17,18 +17,16 @@ import ( "github.com/spf13/cobra" ) -func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, args []string, argIndex int) error { - if len(args) > argIndex { - persistentAuth.Host = args[argIndex] - return nil +func promptForProfile(ctx context.Context, defaultValue string) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a profile using --profile") } - host, err := promptForHost(ctx) - if err != nil { - return err - } - persistentAuth.Host = host - return nil + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks profile name" + prompt.Default = defaultValue + prompt.AllowEdit = true + return prompt.Run() } const minimalDbConnectVersion = "13.1" @@ -93,23 +91,18 @@ depends on the existing profiles you have set in your configuration file cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + profileName := cmd.Flag("profile").Value.String() - var profileName string - profileFlag := cmd.Flag("profile") - if profileFlag != nil && profileFlag.Value.String() != "" { - profileName = profileFlag.Value.String() - } else if cmdio.IsInTTY(ctx) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Profile Name" - prompt.Default = persistentAuth.ProfileName() - prompt.AllowEdit = true - profile, err := prompt.Run() + // If the user has not specified a profile name, prompt for one. + if profileName == "" { + var err error + profileName, err = promptForProfile(ctx, persistentAuth.ProfileName()) if err != nil { return err } - profileName = profile } + // Set the host and account-id based on the provided arguments and flags. err := setHostAndAccountId(ctx, profileName, persistentAuth, args) if err != nil { return err @@ -167,7 +160,23 @@ depends on the existing profiles you have set in your configuration file return cmd } +// Sets the host in the persistentAuth object based on the provided arguments and flags. +// Follows the following precedence: +// 1. [HOST] (first positional argument) or --host flag. Error if both are specified. +// 2. Profile host, if available. +// 3. Prompt the user for the host. +// +// Set the account in the persistentAuth object based on the flags. +// Follows the following precedence: +// 1. --account-id flag. +// 2. account-id from the specified profile, if available. +// 3. Prompt the user for the account-id. func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error { + // If both [HOST] and --host are provided, return an error. + if len(args) > 0 && persistentAuth.Host != "" { + return fmt.Errorf("please only provide a host as an argument or a flag, not both") + } + profiler := profile.GetProfiler(ctx) // If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile. profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName)) @@ -177,17 +186,32 @@ func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth } if persistentAuth.Host == "" { - if len(profiles) > 0 && profiles[0].Host != "" { + if len(args) > 0 { + // If [HOST] is provided, set the host to the provided positional argument. + persistentAuth.Host = args[0] + } else if len(profiles) > 0 && profiles[0].Host != "" { + // If neither [HOST] nor --host are provided, and the profile has a host, use it. persistentAuth.Host = profiles[0].Host } else { - configureHost(ctx, persistentAuth, args, 0) + // If neither [HOST] nor --host are provided, and the profile does not have a host, + // then prompt the user for a host. + hostName, err := promptForHost(ctx) + if err != nil { + return err + } + persistentAuth.Host = hostName } } + + // If the account-id was not provided as a cmd line flag, try to read it from + // the specified profile. isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient() if isAccountClient && persistentAuth.AccountID == "" { if len(profiles) > 0 && profiles[0].AccountID != "" { persistentAuth.AccountID = profiles[0].AccountID } else { + // Prompt user for the account-id if it we could not get it from a + // profile. accountId, err := promptForAccountID(ctx) if err != nil { return err diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go index ce3ca5ae5..d0fa5a16b 100644 --- a/cmd/auth/login_test.go +++ b/cmd/auth/login_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { @@ -15,3 +17,69 @@ func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{}) assert.NoError(t, err) } + +func TestSetHost(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test error when both flag and argument are provided + persistentAuth.Host = "val from --host" + err := setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.EqualError(t, err, "please only provide a host as an argument or a flag, not both") + + // Test setting host from flag + persistentAuth.Host = "val from --host" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "val from --host", persistentAuth.Host) + + // Test setting host from argument + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.NoError(t, err) + assert.Equal(t, "val from [HOST]", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host1.com", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-2", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host2.com", persistentAuth.Host) + + // Test host is not set. Should prompt. + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify a host using --host") +} + +func TestSetAccountId(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test setting account-id from flag + persistentAuth.AccountID = "val from --account-id" + err := setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "val from --account-id", persistentAuth.AccountID) + + // Test setting account_id from profile + persistentAuth.AccountID = "" + err = setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + require.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "id-from-profile", persistentAuth.AccountID) + + // Neither flag nor profile account-id is set, should prompt + persistentAuth.AccountID = "" + persistentAuth.Host = "https://accounts.cloud.databricks.com" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify an account ID using --account-id") +} diff --git a/cmd/auth/testdata/.databrickscfg b/cmd/auth/testdata/.databrickscfg new file mode 100644 index 000000000..06e55224a --- /dev/null +++ b/cmd/auth/testdata/.databrickscfg @@ -0,0 +1,9 @@ +[profile-1] +host = https://www.host1.com + +[profile-2] +host = https://www.host2.com + +[account-profile] +host = https://accounts.cloud.databricks.com +account_id = id-from-profile diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go index 1f3e032de..7c1cb9576 100644 --- a/libs/auth/oauth.go +++ b/libs/auth/oauth.go @@ -105,7 +105,6 @@ func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) { } func (a *PersistentAuth) ProfileName() string { - // TODO: get profile name from interactive input if a.AccountID != "" { return fmt.Sprintf("ACCOUNT-%s", a.AccountID) } From 53041346f2360d2d07ae9c7abb899e3364100b7c Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Wed, 14 Aug 2024 15:21:40 +0200 Subject: [PATCH 62/88] Update VS Code settings to match latest value from IDE plugin (#1677) ## Changes This updates the `python.envFile` property from VS Code's settings file to use the value that is set by the latest version of the IDE plugin. This change will make it a bit easier for contributors who work on the CLI code base with the plugin enabled. --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 869465286..9697e221d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "files.trimFinalNewlines": true, - "python.envFile": "${workspaceFolder}/.databricks/.databricks.env", + "python.envFile": "${workspaceRoot}/.env", "databricks.python.envFile": "${workspaceFolder}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", From f32902dc0466118f9501e3d85c5774c6ac2c88b4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 14 Aug 2024 15:52:09 +0200 Subject: [PATCH 63/88] Use `service.NamedIdMap` to make lookup generation deterministic (#1678) ## Changes Relies on this PR from Go SDK https://github.com/databricks/databricks-sdk-go/pull/1016 See explanation there --- .codegen/lookup.go.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 7e643a90c..431709f90 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -116,12 +116,12 @@ func allResolvers() *resolvers { {{range .Services -}} {{- if in $allowlist .KebabName -}} r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) if err != nil { return "", err } - return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .List.NamedIdMap.IdPath 0).PascalName) }}), nil + return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .NamedIdMap.IdPath 0).PascalName) }}), nil } {{end -}} {{- end}} From 7aaaee2512a62a46f99820a32b64e3cd888b5b7d Mon Sep 17 00:00:00 2001 From: Renaud Hartert Date: Wed, 14 Aug 2024 17:59:55 +0200 Subject: [PATCH 64/88] [Internal] Remove dependency to the `openapi` package of the Go SDK (#1676) ## Changes This PR removes the dependency to the `databricks-sdk-go/openapi` package by copying the struct and functions that are needed in a new `schema/spec.go` file. The reason to remove this dependency is that it is being deprecated. Copying the code in the `cli` repo seems reasonable given that it only uses a couple of very small structs. ## Tests Verified that CLI code can be properly generated after this change. --- bundle/schema/docs.go | 3 +-- bundle/schema/openapi.go | 3 +-- bundle/schema/openapi_test.go | 19 +++++++++---------- bundle/schema/spec.go | 11 +++++++++++ 4 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 bundle/schema/spec.go diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index 5b960ea55..6e9289f92 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" ) // A subset of Schema struct @@ -63,7 +62,7 @@ func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) { if err != nil { return nil, err } - spec := &openapi.Specification{} + spec := &Specification{} err = json.Unmarshal(openapiSpec, spec) if err != nil { return nil, err diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go index 1756d5165..0d896b87c 100644 --- a/bundle/schema/openapi.go +++ b/bundle/schema/openapi.go @@ -6,12 +6,11 @@ import ( "strings" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" ) type OpenapiReader struct { // OpenAPI spec to read schemas from. - OpenapiSpec *openapi.Specification + OpenapiSpec *Specification // In-memory cache of schemas read from the OpenAPI spec. memo map[string]jsonschema.Schema diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go index 359b1e58a..4d393cf37 100644 --- a/bundle/schema/openapi_test.go +++ b/bundle/schema/openapi_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,7 +44,7 @@ func TestReadSchemaForObject(t *testing.T) { } } ` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -103,7 +102,7 @@ func TestReadSchemaForArray(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -149,7 +148,7 @@ func TestReadSchemaForMap(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -198,7 +197,7 @@ func TestRootReferenceIsResolved(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -248,7 +247,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -282,7 +281,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -327,7 +326,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -397,7 +396,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -460,7 +459,7 @@ func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), diff --git a/bundle/schema/spec.go b/bundle/schema/spec.go new file mode 100644 index 000000000..fdc31a4ca --- /dev/null +++ b/bundle/schema/spec.go @@ -0,0 +1,11 @@ +package schema + +import "github.com/databricks/cli/libs/jsonschema" + +type Specification struct { + Components *Components `json:"components"` +} + +type Components struct { + Schemas map[string]*jsonschema.Schema `json:"schemas,omitempty"` +} From 6b3d33a8464722001f29786f6095c54459ec7a6d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 15 Aug 2024 14:43:39 +0200 Subject: [PATCH 65/88] Upgrade TF provider to 1.50.0 (#1681) ## Changes See https://github.com/databricks/terraform-provider-databricks/pull/3900 ## Tests * Manually test on a bundle with a pipeline and a schema * Integration tests pass --- bundle/internal/tf/codegen/schema/version.go | 2 +- .../tf/schema/data_source_notebook.go | 15 +- bundle/internal/tf/schema/data_source_user.go | 1 + .../tf/schema/resource_cluster_policy.go | 2 +- .../schema/resource_metastore_data_access.go | 7 + .../tf/schema/resource_model_serving.go | 56 ++++--- .../internal/tf/schema/resource_notebook.go | 1 + .../resource_notification_destination.go | 46 ++++++ .../internal/tf/schema/resource_pipeline.go | 150 ++++++++++++++---- .../tf/schema/resource_storage_credential.go | 7 + bundle/internal/tf/schema/resources.go | 2 + bundle/internal/tf/schema/root.go | 2 +- 12 files changed, 234 insertions(+), 57 deletions(-) create mode 100644 bundle/internal/tf/schema/resource_notification_destination.go diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 39d4f66c1..efb297243 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.49.1" +const ProviderVersion = "1.50.0" diff --git a/bundle/internal/tf/schema/data_source_notebook.go b/bundle/internal/tf/schema/data_source_notebook.go index ebfbe2dfb..bf97c19a8 100644 --- a/bundle/internal/tf/schema/data_source_notebook.go +++ b/bundle/internal/tf/schema/data_source_notebook.go @@ -3,11 +3,12 @@ package schema type DataSourceNotebook struct { - Content string `json:"content,omitempty"` - Format string `json:"format"` - Id string `json:"id,omitempty"` - Language string `json:"language,omitempty"` - ObjectId int `json:"object_id,omitempty"` - ObjectType string `json:"object_type,omitempty"` - Path string `json:"path"` + Content string `json:"content,omitempty"` + Format string `json:"format"` + Id string `json:"id,omitempty"` + Language string `json:"language,omitempty"` + ObjectId int `json:"object_id,omitempty"` + ObjectType string `json:"object_type,omitempty"` + Path string `json:"path"` + WorkspacePath string `json:"workspace_path,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_user.go b/bundle/internal/tf/schema/data_source_user.go index 78981f29b..ea20c066e 100644 --- a/bundle/internal/tf/schema/data_source_user.go +++ b/bundle/internal/tf/schema/data_source_user.go @@ -4,6 +4,7 @@ package schema type DataSourceUser struct { AclPrincipalId string `json:"acl_principal_id,omitempty"` + Active bool `json:"active,omitempty"` Alphanumeric string `json:"alphanumeric,omitempty"` ApplicationId string `json:"application_id,omitempty"` DisplayName string `json:"display_name,omitempty"` diff --git a/bundle/internal/tf/schema/resource_cluster_policy.go b/bundle/internal/tf/schema/resource_cluster_policy.go index d8111fef2..7e15a7b12 100644 --- a/bundle/internal/tf/schema/resource_cluster_policy.go +++ b/bundle/internal/tf/schema/resource_cluster_policy.go @@ -33,7 +33,7 @@ type ResourceClusterPolicy struct { Description string `json:"description,omitempty"` Id string `json:"id,omitempty"` MaxClustersPerUser int `json:"max_clusters_per_user,omitempty"` - Name string `json:"name"` + Name string `json:"name,omitempty"` PolicyFamilyDefinitionOverrides string `json:"policy_family_definition_overrides,omitempty"` PolicyFamilyId string `json:"policy_family_id,omitempty"` PolicyId string `json:"policy_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_metastore_data_access.go b/bundle/internal/tf/schema/resource_metastore_data_access.go index 2e2ff4eb4..ef8c34aa7 100644 --- a/bundle/internal/tf/schema/resource_metastore_data_access.go +++ b/bundle/internal/tf/schema/resource_metastore_data_access.go @@ -20,6 +20,12 @@ type ResourceMetastoreDataAccessAzureServicePrincipal struct { DirectoryId string `json:"directory_id"` } +type ResourceMetastoreDataAccessCloudflareApiToken struct { + AccessKeyId string `json:"access_key_id"` + AccountId string `json:"account_id"` + SecretAccessKey string `json:"secret_access_key"` +} + type ResourceMetastoreDataAccessDatabricksGcpServiceAccount struct { CredentialId string `json:"credential_id,omitempty"` Email string `json:"email,omitempty"` @@ -46,6 +52,7 @@ type ResourceMetastoreDataAccess struct { AwsIamRole *ResourceMetastoreDataAccessAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceMetastoreDataAccessAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceMetastoreDataAccessAzureServicePrincipal `json:"azure_service_principal,omitempty"` + CloudflareApiToken *ResourceMetastoreDataAccessCloudflareApiToken `json:"cloudflare_api_token,omitempty"` DatabricksGcpServiceAccount *ResourceMetastoreDataAccessDatabricksGcpServiceAccount `json:"databricks_gcp_service_account,omitempty"` GcpServiceAccountKey *ResourceMetastoreDataAccessGcpServiceAccountKey `json:"gcp_service_account_key,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index f5ffbbe5e..379807a5d 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -10,43 +10,60 @@ type ResourceModelServingConfigAutoCaptureConfig struct { } type ResourceModelServingConfigServedEntitiesExternalModelAi21LabsConfig struct { - Ai21LabsApiKey string `json:"ai21labs_api_key"` + Ai21LabsApiKey string `json:"ai21labs_api_key,omitempty"` + Ai21LabsApiKeyPlaintext string `json:"ai21labs_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelAmazonBedrockConfig struct { - AwsAccessKeyId string `json:"aws_access_key_id"` - AwsRegion string `json:"aws_region"` - AwsSecretAccessKey string `json:"aws_secret_access_key"` - BedrockProvider string `json:"bedrock_provider"` + AwsAccessKeyId string `json:"aws_access_key_id,omitempty"` + AwsAccessKeyIdPlaintext string `json:"aws_access_key_id_plaintext,omitempty"` + AwsRegion string `json:"aws_region"` + AwsSecretAccessKey string `json:"aws_secret_access_key,omitempty"` + AwsSecretAccessKeyPlaintext string `json:"aws_secret_access_key_plaintext,omitempty"` + BedrockProvider string `json:"bedrock_provider"` } type ResourceModelServingConfigServedEntitiesExternalModelAnthropicConfig struct { - AnthropicApiKey string `json:"anthropic_api_key"` + AnthropicApiKey string `json:"anthropic_api_key,omitempty"` + AnthropicApiKeyPlaintext string `json:"anthropic_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelCohereConfig struct { - CohereApiKey string `json:"cohere_api_key"` + CohereApiBase string `json:"cohere_api_base,omitempty"` + CohereApiKey string `json:"cohere_api_key,omitempty"` + CohereApiKeyPlaintext string `json:"cohere_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServingConfig struct { - DatabricksApiToken string `json:"databricks_api_token"` - DatabricksWorkspaceUrl string `json:"databricks_workspace_url"` + DatabricksApiToken string `json:"databricks_api_token,omitempty"` + DatabricksApiTokenPlaintext string `json:"databricks_api_token_plaintext,omitempty"` + DatabricksWorkspaceUrl string `json:"databricks_workspace_url"` +} + +type ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct { + PrivateKey string `json:"private_key,omitempty"` + PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"` + ProjectId string `json:"project_id,omitempty"` + Region string `json:"region,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct { - MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"` - MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"` - MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"` - OpenaiApiBase string `json:"openai_api_base,omitempty"` - OpenaiApiKey string `json:"openai_api_key,omitempty"` - OpenaiApiType string `json:"openai_api_type,omitempty"` - OpenaiApiVersion string `json:"openai_api_version,omitempty"` - OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` - OpenaiOrganization string `json:"openai_organization,omitempty"` + MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"` + MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"` + MicrosoftEntraClientSecretPlaintext string `json:"microsoft_entra_client_secret_plaintext,omitempty"` + MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"` + OpenaiApiBase string `json:"openai_api_base,omitempty"` + OpenaiApiKey string `json:"openai_api_key,omitempty"` + OpenaiApiKeyPlaintext string `json:"openai_api_key_plaintext,omitempty"` + OpenaiApiType string `json:"openai_api_type,omitempty"` + OpenaiApiVersion string `json:"openai_api_version,omitempty"` + OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` + OpenaiOrganization string `json:"openai_organization,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelPalmConfig struct { - PalmApiKey string `json:"palm_api_key"` + PalmApiKey string `json:"palm_api_key,omitempty"` + PalmApiKeyPlaintext string `json:"palm_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModel struct { @@ -58,6 +75,7 @@ type ResourceModelServingConfigServedEntitiesExternalModel struct { AnthropicConfig *ResourceModelServingConfigServedEntitiesExternalModelAnthropicConfig `json:"anthropic_config,omitempty"` CohereConfig *ResourceModelServingConfigServedEntitiesExternalModelCohereConfig `json:"cohere_config,omitempty"` DatabricksModelServingConfig *ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServingConfig `json:"databricks_model_serving_config,omitempty"` + GoogleCloudVertexAiConfig *ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig `json:"google_cloud_vertex_ai_config,omitempty"` OpenaiConfig *ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig `json:"openai_config,omitempty"` PalmConfig *ResourceModelServingConfigServedEntitiesExternalModelPalmConfig `json:"palm_config,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_notebook.go b/bundle/internal/tf/schema/resource_notebook.go index 8fb5a5387..4e5d4cbc3 100644 --- a/bundle/internal/tf/schema/resource_notebook.go +++ b/bundle/internal/tf/schema/resource_notebook.go @@ -13,4 +13,5 @@ type ResourceNotebook struct { Path string `json:"path"` Source string `json:"source,omitempty"` Url string `json:"url,omitempty"` + WorkspacePath string `json:"workspace_path,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_notification_destination.go b/bundle/internal/tf/schema/resource_notification_destination.go new file mode 100644 index 000000000..0ed9cff60 --- /dev/null +++ b/bundle/internal/tf/schema/resource_notification_destination.go @@ -0,0 +1,46 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceNotificationDestinationConfigEmail struct { + Addresses []string `json:"addresses,omitempty"` +} + +type ResourceNotificationDestinationConfigGenericWebhook struct { + Password string `json:"password,omitempty"` + PasswordSet bool `json:"password_set,omitempty"` + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` + Username string `json:"username,omitempty"` + UsernameSet bool `json:"username_set,omitempty"` +} + +type ResourceNotificationDestinationConfigMicrosoftTeams struct { + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` +} + +type ResourceNotificationDestinationConfigPagerduty struct { + IntegrationKey string `json:"integration_key,omitempty"` + IntegrationKeySet bool `json:"integration_key_set,omitempty"` +} + +type ResourceNotificationDestinationConfigSlack struct { + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` +} + +type ResourceNotificationDestinationConfig struct { + Email *ResourceNotificationDestinationConfigEmail `json:"email,omitempty"` + GenericWebhook *ResourceNotificationDestinationConfigGenericWebhook `json:"generic_webhook,omitempty"` + MicrosoftTeams *ResourceNotificationDestinationConfigMicrosoftTeams `json:"microsoft_teams,omitempty"` + Pagerduty *ResourceNotificationDestinationConfigPagerduty `json:"pagerduty,omitempty"` + Slack *ResourceNotificationDestinationConfigSlack `json:"slack,omitempty"` +} + +type ResourceNotificationDestination struct { + DestinationType string `json:"destination_type,omitempty"` + DisplayName string `json:"display_name"` + Id string `json:"id,omitempty"` + Config *ResourceNotificationDestinationConfig `json:"config,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 20c25c1e2..154686463 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -3,15 +3,17 @@ package schema type ResourcePipelineClusterAutoscale struct { - MaxWorkers int `json:"max_workers,omitempty"` - MinWorkers int `json:"min_workers,omitempty"` + MaxWorkers int `json:"max_workers"` + MinWorkers int `json:"min_workers"` Mode string `json:"mode,omitempty"` } type ResourcePipelineClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -19,10 +21,16 @@ type ResourcePipelineClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourcePipelineClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourcePipelineClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourcePipelineClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourcePipelineClusterClusterLogConfDbfs struct { @@ -127,8 +135,69 @@ type ResourcePipelineFilters struct { Include []string `json:"include,omitempty"` } +type ResourcePipelineGatewayDefinition struct { + ConnectionId string `json:"connection_id,omitempty"` + GatewayStorageCatalog string `json:"gateway_storage_catalog,omitempty"` + GatewayStorageName string `json:"gateway_storage_name,omitempty"` + GatewayStorageSchema string `json:"gateway_storage_schema,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchema struct { + DestinationCatalog string `json:"destination_catalog,omitempty"` + DestinationSchema string `json:"destination_schema,omitempty"` + SourceCatalog string `json:"source_catalog,omitempty"` + SourceSchema string `json:"source_schema,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsTable struct { + DestinationCatalog string `json:"destination_catalog,omitempty"` + DestinationSchema string `json:"destination_schema,omitempty"` + DestinationTable string `json:"destination_table,omitempty"` + SourceCatalog string `json:"source_catalog,omitempty"` + SourceSchema string `json:"source_schema,omitempty"` + SourceTable string `json:"source_table,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjects struct { + Schema *ResourcePipelineIngestionDefinitionObjectsSchema `json:"schema,omitempty"` + Table *ResourcePipelineIngestionDefinitionObjectsTable `json:"table,omitempty"` +} + +type ResourcePipelineIngestionDefinitionTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinition struct { + ConnectionName string `json:"connection_name,omitempty"` + IngestionGatewayId string `json:"ingestion_gateway_id,omitempty"` + Objects []ResourcePipelineIngestionDefinitionObjects `json:"objects,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineLatestUpdates struct { + CreationTime string `json:"creation_time,omitempty"` + State string `json:"state,omitempty"` + UpdateId string `json:"update_id,omitempty"` +} + type ResourcePipelineLibraryFile struct { - Path string `json:"path"` + Path string `json:"path,omitempty"` } type ResourcePipelineLibraryMaven struct { @@ -138,7 +207,7 @@ type ResourcePipelineLibraryMaven struct { } type ResourcePipelineLibraryNotebook struct { - Path string `json:"path"` + Path string `json:"path,omitempty"` } type ResourcePipelineLibrary struct { @@ -150,28 +219,53 @@ type ResourcePipelineLibrary struct { } type ResourcePipelineNotification struct { - Alerts []string `json:"alerts"` - EmailRecipients []string `json:"email_recipients"` + Alerts []string `json:"alerts,omitempty"` + EmailRecipients []string `json:"email_recipients,omitempty"` +} + +type ResourcePipelineTriggerCron struct { + QuartzCronSchedule string `json:"quartz_cron_schedule,omitempty"` + TimezoneId string `json:"timezone_id,omitempty"` +} + +type ResourcePipelineTriggerManual struct { +} + +type ResourcePipelineTrigger struct { + Cron *ResourcePipelineTriggerCron `json:"cron,omitempty"` + Manual *ResourcePipelineTriggerManual `json:"manual,omitempty"` } type ResourcePipeline struct { - AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` - Catalog string `json:"catalog,omitempty"` - Channel string `json:"channel,omitempty"` - Configuration map[string]string `json:"configuration,omitempty"` - Continuous bool `json:"continuous,omitempty"` - Development bool `json:"development,omitempty"` - Edition string `json:"edition,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Photon bool `json:"photon,omitempty"` - Serverless bool `json:"serverless,omitempty"` - Storage string `json:"storage,omitempty"` - Target string `json:"target,omitempty"` - Url string `json:"url,omitempty"` - Cluster []ResourcePipelineCluster `json:"cluster,omitempty"` - Deployment *ResourcePipelineDeployment `json:"deployment,omitempty"` - Filters *ResourcePipelineFilters `json:"filters,omitempty"` - Library []ResourcePipelineLibrary `json:"library,omitempty"` - Notification []ResourcePipelineNotification `json:"notification,omitempty"` + AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` + Catalog string `json:"catalog,omitempty"` + Cause string `json:"cause,omitempty"` + Channel string `json:"channel,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + Configuration map[string]string `json:"configuration,omitempty"` + Continuous bool `json:"continuous,omitempty"` + CreatorUserName string `json:"creator_user_name,omitempty"` + Development bool `json:"development,omitempty"` + Edition string `json:"edition,omitempty"` + ExpectedLastModified int `json:"expected_last_modified,omitempty"` + Health string `json:"health,omitempty"` + Id string `json:"id,omitempty"` + LastModified int `json:"last_modified,omitempty"` + Name string `json:"name,omitempty"` + Photon bool `json:"photon,omitempty"` + RunAsUserName string `json:"run_as_user_name,omitempty"` + Serverless bool `json:"serverless,omitempty"` + State string `json:"state,omitempty"` + Storage string `json:"storage,omitempty"` + Target string `json:"target,omitempty"` + Url string `json:"url,omitempty"` + Cluster []ResourcePipelineCluster `json:"cluster,omitempty"` + Deployment *ResourcePipelineDeployment `json:"deployment,omitempty"` + Filters *ResourcePipelineFilters `json:"filters,omitempty"` + GatewayDefinition *ResourcePipelineGatewayDefinition `json:"gateway_definition,omitempty"` + IngestionDefinition *ResourcePipelineIngestionDefinition `json:"ingestion_definition,omitempty"` + LatestUpdates []ResourcePipelineLatestUpdates `json:"latest_updates,omitempty"` + Library []ResourcePipelineLibrary `json:"library,omitempty"` + Notification []ResourcePipelineNotification `json:"notification,omitempty"` + Trigger *ResourcePipelineTrigger `json:"trigger,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_storage_credential.go b/bundle/internal/tf/schema/resource_storage_credential.go index 1c62cf8df..7278c2193 100644 --- a/bundle/internal/tf/schema/resource_storage_credential.go +++ b/bundle/internal/tf/schema/resource_storage_credential.go @@ -20,6 +20,12 @@ type ResourceStorageCredentialAzureServicePrincipal struct { DirectoryId string `json:"directory_id"` } +type ResourceStorageCredentialCloudflareApiToken struct { + AccessKeyId string `json:"access_key_id"` + AccountId string `json:"account_id"` + SecretAccessKey string `json:"secret_access_key"` +} + type ResourceStorageCredentialDatabricksGcpServiceAccount struct { CredentialId string `json:"credential_id,omitempty"` Email string `json:"email,omitempty"` @@ -46,6 +52,7 @@ type ResourceStorageCredential struct { AwsIamRole *ResourceStorageCredentialAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceStorageCredentialAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceStorageCredentialAzureServicePrincipal `json:"azure_service_principal,omitempty"` + CloudflareApiToken *ResourceStorageCredentialCloudflareApiToken `json:"cloudflare_api_token,omitempty"` DatabricksGcpServiceAccount *ResourceStorageCredentialDatabricksGcpServiceAccount `json:"databricks_gcp_service_account,omitempty"` GcpServiceAccountKey *ResourceStorageCredentialGcpServiceAccountKey `json:"gcp_service_account_key,omitempty"` } diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79c1b32b5..737b77a2a 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -59,6 +59,7 @@ type Resources struct { MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` Notebook map[string]any `json:"databricks_notebook,omitempty"` + NotificationDestination map[string]any `json:"databricks_notification_destination,omitempty"` OboToken map[string]any `json:"databricks_obo_token,omitempty"` OnlineTable map[string]any `json:"databricks_online_table,omitempty"` PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` @@ -160,6 +161,7 @@ func NewResources() *Resources { MwsVpcEndpoint: make(map[string]any), MwsWorkspaces: make(map[string]any), Notebook: make(map[string]any), + NotificationDestination: make(map[string]any), OboToken: make(map[string]any), OnlineTable: make(map[string]any), PermissionAssignment: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 171128350..ebdb7f095 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.49.1" +const ProviderVersion = "1.50.0" func NewRoot() *Root { return &Root{ From a6eb673d55ad7cd6163050b6c3cc845c67ac52a5 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 15 Aug 2024 18:23:02 +0530 Subject: [PATCH 66/88] Print text logs in `import-dir` and `export-dir` commands (#1682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes In https://github.com/databricks/cli/pull/1202 the semantics of `cmdio.RenderJson` was changes to always render the JSON object. Before we would only render it if `--output json` was specified. This PR fixes the logs to print human-readable log lines instead of a JSON object. This PR also removes the now unused `cmdio.Render` method. ## Tests Manually: ``` ➜ bundle-playground git:(master) ✗ cli workspace import-dir ./tmp /Users/shreyas.goenka@databricks.com/test-import-1 -p aws-prod-ucws Importing files from ./tmp a -> /Users/shreyas.goenka@databricks.com/test-import-1/a Import complete. The files are available at /Users/shreyas.goenka@databricks.com/test-import-1 ``` ``` ➜ bundle-playground git:(master) ✗ cli workspace export-dir /Users/shreyas.goenka@databricks.com/test-export-1 ./tmp-2 -p aws-prod-ucws Exporting files from /Users/shreyas.goenka@databricks.com/test-export-1 /Users/shreyas.goenka@databricks.com/test-export-1/b -> tmp-2/b Exported complete. The files are available at ./tmp-2 ``` --- cmd/workspace/workspace/export_dir.go | 5 +-- cmd/workspace/workspace/import_dir.go | 5 +-- internal/workspace_test.go | 53 ++++++++++++++++++--------- libs/cmdio/render.go | 8 ---- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/cmd/workspace/workspace/export_dir.go b/cmd/workspace/workspace/export_dir.go index 0b53666f9..0046f46ef 100644 --- a/cmd/workspace/workspace/export_dir.go +++ b/cmd/workspace/workspace/export_dir.go @@ -110,8 +110,7 @@ func newExportDir() *cobra.Command { } workspaceFS := filer.NewFS(ctx, workspaceFiler) - // TODO: print progress events on stderr instead: https://github.com/databricks/cli/issues/448 - err = cmdio.RenderJson(ctx, newExportStartedEvent(opts.sourceDir)) + err = cmdio.RenderWithTemplate(ctx, newExportStartedEvent(opts.sourceDir), "", "Exporting files from {{.SourcePath}}\n") if err != nil { return err } @@ -120,7 +119,7 @@ func newExportDir() *cobra.Command { if err != nil { return err } - return cmdio.RenderJson(ctx, newExportCompletedEvent(opts.targetDir)) + return cmdio.RenderWithTemplate(ctx, newExportCompletedEvent(opts.targetDir), "", "Export complete\n") } return cmd diff --git a/cmd/workspace/workspace/import_dir.go b/cmd/workspace/workspace/import_dir.go index 19d9a0a17..a197d7dd9 100644 --- a/cmd/workspace/workspace/import_dir.go +++ b/cmd/workspace/workspace/import_dir.go @@ -134,8 +134,7 @@ Notebooks will have their extensions (one of .scala, .py, .sql, .ipynb, .r) stri return err } - // TODO: print progress events on stderr instead: https://github.com/databricks/cli/issues/448 - err = cmdio.RenderJson(ctx, newImportStartedEvent(opts.sourceDir)) + err = cmdio.RenderWithTemplate(ctx, newImportStartedEvent(opts.sourceDir), "", "Importing files from {{.SourcePath}}\n") if err != nil { return err } @@ -145,7 +144,7 @@ Notebooks will have their extensions (one of .scala, .py, .sql, .ipynb, .r) stri if err != nil { return err } - return cmdio.RenderJson(ctx, newImportCompletedEvent(opts.targetDir)) + return cmdio.RenderWithTemplate(ctx, newImportCompletedEvent(opts.targetDir), "", "Import complete\n") } return cmd diff --git a/internal/workspace_test.go b/internal/workspace_test.go index bc354914f..445361654 100644 --- a/internal/workspace_test.go +++ b/internal/workspace_test.go @@ -3,18 +3,17 @@ package internal import ( "context" "encoding/base64" - "errors" + "fmt" "io" - "net/http" "os" "path" "path/filepath" "strings" "testing" + "github.com/databricks/cli/internal/acc" "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -63,21 +62,12 @@ func TestAccWorkpaceExportPrintsContents(t *testing.T) { } func setupWorkspaceImportExportTest(t *testing.T) (context.Context, filer.Filer, string) { - t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + ctx, wt := acc.WorkspaceTest(t) - ctx := context.Background() - w := databricks.Must(databricks.NewWorkspaceClient()) - tmpdir := TemporaryWorkspaceDir(t, w) - f, err := filer.NewWorkspaceFilesClient(w, tmpdir) + tmpdir := TemporaryWorkspaceDir(t, wt.W) + f, err := filer.NewWorkspaceFilesClient(wt.W, tmpdir) require.NoError(t, err) - // Check if we can use this API here, skip test if we cannot. - _, err = f.Read(ctx, "we_use_this_call_to_test_if_this_api_is_enabled") - var aerr *apierr.APIError - if errors.As(err, &aerr) && aerr.StatusCode == http.StatusBadRequest { - t.Skip(aerr.Message) - } - return ctx, f, tmpdir } @@ -122,8 +112,21 @@ func TestAccExportDir(t *testing.T) { err = f.Write(ctx, "a/b/c/file-b", strings.NewReader("def"), filer.CreateParentDirectories) require.NoError(t, err) + expectedLogs := strings.Join([]string{ + fmt.Sprintf("Exporting files from %s", sourceDir), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "a/b/c/file-b"), filepath.Join(targetDir, "a/b/c/file-b")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "file-a"), filepath.Join(targetDir, "file-a")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "pyNotebook"), filepath.Join(targetDir, "pyNotebook.py")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "rNotebook"), filepath.Join(targetDir, "rNotebook.r")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "scalaNotebook"), filepath.Join(targetDir, "scalaNotebook.scala")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "sqlNotebook"), filepath.Join(targetDir, "sqlNotebook.sql")), + "Export complete\n", + }, "\n") + // Run Export - RequireSuccessfulRun(t, "workspace", "export-dir", sourceDir, targetDir) + stdout, stderr := RequireSuccessfulRun(t, "workspace", "export-dir", sourceDir, targetDir) + assert.Equal(t, expectedLogs, stdout.String()) + assert.Equal(t, "", stderr.String()) // Assert files were exported assertLocalFileContents(t, filepath.Join(targetDir, "file-a"), "abc") @@ -176,10 +179,24 @@ func TestAccExportDirWithOverwriteFlag(t *testing.T) { assertLocalFileContents(t, filepath.Join(targetDir, "file-a"), "content from workspace") } -// TODO: Add assertions on progress logs for workspace import-dir command. https://github.com/databricks/cli/issues/455 func TestAccImportDir(t *testing.T) { ctx, workspaceFiler, targetDir := setupWorkspaceImportExportTest(t) - RequireSuccessfulRun(t, "workspace", "import-dir", "./testdata/import_dir", targetDir, "--log-level=debug") + stdout, stderr := RequireSuccessfulRun(t, "workspace", "import-dir", "./testdata/import_dir", targetDir, "--log-level=debug") + + expectedLogs := strings.Join([]string{ + fmt.Sprintf("Importing files from %s", "./testdata/import_dir"), + fmt.Sprintf("%s -> %s", filepath.FromSlash("a/b/c/file-b"), path.Join(targetDir, "a/b/c/file-b")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("file-a"), path.Join(targetDir, "file-a")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("jupyterNotebook.ipynb"), path.Join(targetDir, "jupyterNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("pyNotebook.py"), path.Join(targetDir, "pyNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("rNotebook.r"), path.Join(targetDir, "rNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("scalaNotebook.scala"), path.Join(targetDir, "scalaNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("sqlNotebook.sql"), path.Join(targetDir, "sqlNotebook")), + "Import complete\n", + }, "\n") + + assert.Equal(t, expectedLogs, stdout.String()) + assert.Equal(t, "", stderr.String()) // Assert files are imported assertFilerFileContents(t, ctx, workspaceFiler, "file-a", "hello, world") diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index ec851b8ff..4114db5ca 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -280,14 +280,6 @@ func RenderIteratorWithTemplate[T any](ctx context.Context, i listing.Iterator[T return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, headerTemplate, template) } -func RenderJson(ctx context.Context, v any) error { - c := fromContext(ctx) - if _, ok := v.(listingInterface); ok { - panic("use RenderIteratorJson instead") - } - return renderWithTemplate(newRenderer(v), ctx, flags.OutputJSON, c.out, c.headerTemplate, c.template) -} - func RenderIteratorJson[T any](ctx context.Context, i listing.Iterator[T]) error { c := fromContext(ctx) return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, c.headerTemplate, c.template) From 54799a1918e4eca026090626539928d3d886736e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 15 Aug 2024 15:23:07 +0200 Subject: [PATCH 67/88] Upgrade Go SDK to 0.44.0 (#1679) ## Changes Upgrade Go SDK to 0.44.0 --------- Co-authored-by: Pieter Noordhuis --- .codegen/_openapi_sha | 2 +- .gitattributes | 6 + bundle/config/variable/lookup.go | 4 +- bundle/run/pipeline.go | 4 +- bundle/run/progress/pipeline.go | 6 +- bundle/schema/docs/bundle_descriptions.json | 266 ++++++++-- cmd/account/budgets/budgets.go | 125 ++--- cmd/account/cmd.go | 4 +- .../custom-app-integration.go | 55 +- .../o-auth-published-apps.go | 2 +- .../published-app-integration.go | 43 +- .../usage-dashboards/usage-dashboards.go | 164 ++++++ .../workspace-assignment.go | 8 +- cmd/cmd.go | 2 + cmd/labs/project/installer_test.go | 11 +- cmd/root/auth_test.go | 8 + cmd/workspace/alerts-legacy/alerts-legacy.go | 388 ++++++++++++++ cmd/workspace/alerts/alerts.go | 144 +++-- cmd/workspace/apps/apps.go | 325 ++++++++++-- .../cluster-policies/cluster-policies.go | 60 +-- cmd/workspace/clusters/clusters.go | 142 ++++- cmd/workspace/cmd.go | 10 + .../consumer-fulfillments.go | 3 - .../consumer-installations.go | 3 - .../consumer-listings/consumer-listings.go | 7 - .../consumer-personalization-requests.go | 3 - .../consumer-providers/consumer-providers.go | 3 - cmd/workspace/data-sources/data-sources.go | 12 +- cmd/workspace/genie/genie.go | 437 +++++++++++++++ cmd/workspace/groups.go | 4 + cmd/workspace/jobs/jobs.go | 1 + cmd/workspace/lakeview/lakeview.go | 2 +- .../model-versions/model-versions.go | 3 + .../notification-destinations.go | 342 ++++++++++++ .../permission-migration.go | 16 +- cmd/workspace/permissions/permissions.go | 30 +- .../policy-families/policy-families.go | 13 +- .../provider-exchange-filters.go | 3 - .../provider-exchanges/provider-exchanges.go | 3 - .../provider-files/provider-files.go | 3 - .../provider-listings/provider-listings.go | 3 - .../provider-personalization-requests.go | 3 - .../provider-provider-analytics-dashboards.go | 3 - .../provider-providers/provider-providers.go | 3 - cmd/workspace/providers/providers.go | 5 + .../queries-legacy/queries-legacy.go | 500 ++++++++++++++++++ cmd/workspace/queries/queries.go | 227 ++++---- cmd/workspace/query-history/query-history.go | 23 +- .../query-visualizations-legacy.go | 253 +++++++++ .../query-visualizations.go | 72 ++- cmd/workspace/recipients/recipients.go | 7 + .../registered-models/registered-models.go | 1 + cmd/workspace/schemas/schemas.go | 2 + cmd/workspace/shares/shares.go | 23 +- .../system-schemas/system-schemas.go | 3 + .../workspace-bindings/workspace-bindings.go | 29 +- go.mod | 8 +- go.sum | 16 +- libs/databrickscfg/cfgpickers/clusters.go | 4 +- .../databrickscfg/cfgpickers/clusters_test.go | 8 +- 60 files changed, 3251 insertions(+), 609 deletions(-) create mode 100755 cmd/account/usage-dashboards/usage-dashboards.go create mode 100755 cmd/workspace/alerts-legacy/alerts-legacy.go create mode 100755 cmd/workspace/genie/genie.go create mode 100755 cmd/workspace/notification-destinations/notification-destinations.go create mode 100755 cmd/workspace/queries-legacy/queries-legacy.go create mode 100755 cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index c4b47ca14..fef6f268b 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -7437dabb9dadee402c1fc060df4c1ce8cc5369f0 \ No newline at end of file +f98c07f9c71f579de65d2587bb0292f83d10e55d \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index c11257e9e..bdb3f3982 100755 --- a/.gitattributes +++ b/.gitattributes @@ -24,10 +24,12 @@ cmd/account/service-principals/service-principals.go linguist-generated=true cmd/account/settings/settings.go linguist-generated=true cmd/account/storage-credentials/storage-credentials.go linguist-generated=true cmd/account/storage/storage.go linguist-generated=true +cmd/account/usage-dashboards/usage-dashboards.go linguist-generated=true cmd/account/users/users.go linguist-generated=true cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true cmd/account/workspaces/workspaces.go linguist-generated=true +cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true cmd/workspace/alerts/alerts.go linguist-generated=true cmd/workspace/apps/apps.go linguist-generated=true cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true @@ -54,6 +56,7 @@ cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go lingu cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true cmd/workspace/functions/functions.go linguist-generated=true +cmd/workspace/genie/genie.go linguist-generated=true cmd/workspace/git-credentials/git-credentials.go linguist-generated=true cmd/workspace/global-init-scripts/global-init-scripts.go linguist-generated=true cmd/workspace/grants/grants.go linguist-generated=true @@ -67,6 +70,7 @@ cmd/workspace/libraries/libraries.go linguist-generated=true cmd/workspace/metastores/metastores.go linguist-generated=true cmd/workspace/model-registry/model-registry.go linguist-generated=true cmd/workspace/model-versions/model-versions.go linguist-generated=true +cmd/workspace/notification-destinations/notification-destinations.go linguist-generated=true cmd/workspace/online-tables/online-tables.go linguist-generated=true cmd/workspace/permission-migration/permission-migration.go linguist-generated=true cmd/workspace/permissions/permissions.go linguist-generated=true @@ -81,8 +85,10 @@ cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics cmd/workspace/provider-providers/provider-providers.go linguist-generated=true cmd/workspace/providers/providers.go linguist-generated=true cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true +cmd/workspace/queries-legacy/queries-legacy.go linguist-generated=true cmd/workspace/queries/queries.go linguist-generated=true cmd/workspace/query-history/query-history.go linguist-generated=true +cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go linguist-generated=true cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true cmd/workspace/recipient-activation/recipient-activation.go linguist-generated=true cmd/workspace/recipients/recipients.go linguist-generated=true diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 56d2ca810..9c85e2a71 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -220,7 +220,7 @@ type resolvers struct { func allResolvers() *resolvers { r := &resolvers{} r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.Alerts.GetByName(ctx, name) + entity, err := w.Alerts.GetByDisplayName(ctx, name) if err != nil { return "", err } @@ -284,7 +284,7 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PipelineId), nil } r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.Queries.GetByName(ctx, name) + entity, err := w.Queries.GetByDisplayName(ctx, name) if err != nil { return "", err } diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 4e29b9f3f..d684f8388 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -53,7 +53,7 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId string, u // Otherwise for long lived pipelines, there can be a lot of unnecessary // latency due to multiple pagination API calls needed underneath the hood for // ListPipelineEventsAll - res, err := w.Pipelines.Impl().ListPipelineEvents(ctx, pipelines.ListPipelineEventsRequest{ + events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ Filter: `level='ERROR'`, MaxResults: 100, PipelineId: pipelineId, @@ -61,7 +61,7 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId string, u if err != nil { return err } - updateEvents := filterEventsByUpdateId(res.Events, updateId) + updateEvents := filterEventsByUpdateId(events, updateId) // The events API returns most recent events first. We iterate in a reverse order // to print the events chronologically for i := len(updateEvents) - 1; i >= 0; i-- { diff --git a/bundle/run/progress/pipeline.go b/bundle/run/progress/pipeline.go index fb076f680..4a256e76c 100644 --- a/bundle/run/progress/pipeline.go +++ b/bundle/run/progress/pipeline.go @@ -78,7 +78,7 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { } // we only check the most recent 100 events for progress - response, err := l.w.Pipelines.Impl().ListPipelineEvents(ctx, pipelines.ListPipelineEventsRequest{ + events, err := l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ PipelineId: l.PipelineId, MaxResults: 100, Filter: filter, @@ -89,8 +89,8 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { result := make([]ProgressEvent, 0) // we iterate in reverse to return events in chronological order - for i := len(response.Events) - 1; i >= 0; i-- { - event := response.Events[i] + for i := len(events) - 1; i >= 0; i-- { + event := events[i] // filter to only include update_progress and flow_progress events if event.EventType == "flow_progress" || event.EventType == "update_progress" { result = append(result, ProgressEvent(event)) diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json index 380be0545..d888b3663 100644 --- a/bundle/schema/docs/bundle_descriptions.json +++ b/bundle/schema/docs/bundle_descriptions.json @@ -218,7 +218,7 @@ } }, "description": { - "description": "An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding." + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." }, "edit_mode": { "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." @@ -935,7 +935,7 @@ } }, "egg": { - "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." }, "jar": { "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." @@ -1827,13 +1827,16 @@ } }, "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. When an external_model is present, the served\nentities list can only have one served_entity object. For an existing endpoint with external_model, it can not be updated to an endpoint without external_model.\nIf the endpoint is created without external_model, users cannot update it to add external_model later.\n", + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", "properties": { "ai21labs_config": { "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", "properties": { "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21Labs API key." + "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." + }, + "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." } } }, @@ -1841,13 +1844,19 @@ "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", "properties": { "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS Access Key ID with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." }, "aws_region": { "description": "The AWS region to use. Bedrock has to be enabled there." }, "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS Secret Access Key paired with the access key ID, with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." }, "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." @@ -1858,15 +1867,24 @@ "description": "Anthropic Config. Only required if the provider is 'anthropic'.", "properties": { "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key." + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." } } }, "cohere_config": { "description": "Cohere Config. Only required if the provider is 'cohere'.", "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + }, "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key." + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." } } }, @@ -1874,13 +1892,33 @@ "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", "properties": { "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\n" + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" }, "databricks_workspace_url": { "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" } } }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "properties": { + "private_key": { + "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" + }, + "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." + }, + "project_id": { + "description": "This is the Google Cloud project id that the service account is associated with." + }, + "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." + } + } + }, "name": { "description": "The name of the external model." }, @@ -1891,16 +1929,22 @@ "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" }, "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" + }, + "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" }, "microsoft_entra_tenant_id": { "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" }, "openai_api_base": { - "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" + "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" }, "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI or Azure OpenAI API key." + "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." + }, + "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." }, "openai_api_type": { "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" @@ -1920,12 +1964,15 @@ "description": "PaLM Config. Only required if the provider is 'palm'.", "properties": { "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key." + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." + }, + "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." } } }, "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'openai', and 'palm'.\",\n" + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" }, "task": { "description": "The task type of the external model." @@ -2331,6 +2378,9 @@ "driver_node_type_id": { "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster." + }, "gcp_attributes": { "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", "properties": { @@ -2525,7 +2575,7 @@ "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." }, "gateway_storage_name": { - "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" }, "gateway_storage_schema": { "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." @@ -2565,7 +2615,7 @@ "description": "Required. Schema name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -2605,7 +2655,7 @@ "description": "Required. Table name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -2685,6 +2735,9 @@ "description": "The absolute path of the notebook." } } + }, + "whl": { + "description": "URI of the whl to be installed." } } } @@ -2955,6 +3008,49 @@ } } } + }, + "schemas": { + "description": "", + "additionalproperties": { + "description": "", + "properties": { + "catalog_name": { + "description": "" + }, + "comment": { + "description": "" + }, + "grants": { + "description": "", + "items": { + "description": "", + "properties": { + "principal": { + "description": "" + }, + "privileges": { + "description": "", + "items": { + "description": "" + } + } + } + } + }, + "name": { + "description": "" + }, + "properties": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "storage_root": { + "description": "" + } + } + } } } }, @@ -3194,7 +3290,7 @@ } }, "description": { - "description": "An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding." + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." }, "edit_mode": { "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." @@ -3911,7 +4007,7 @@ } }, "egg": { - "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." }, "jar": { "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." @@ -4803,13 +4899,16 @@ } }, "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. When an external_model is present, the served\nentities list can only have one served_entity object. For an existing endpoint with external_model, it can not be updated to an endpoint without external_model.\nIf the endpoint is created without external_model, users cannot update it to add external_model later.\n", + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", "properties": { "ai21labs_config": { "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", "properties": { "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21Labs API key." + "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." + }, + "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." } } }, @@ -4817,13 +4916,19 @@ "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", "properties": { "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS Access Key ID with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." }, "aws_region": { "description": "The AWS region to use. Bedrock has to be enabled there." }, "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS Secret Access Key paired with the access key ID, with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." }, "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." @@ -4834,15 +4939,24 @@ "description": "Anthropic Config. Only required if the provider is 'anthropic'.", "properties": { "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key." + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." } } }, "cohere_config": { "description": "Cohere Config. Only required if the provider is 'cohere'.", "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + }, "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key." + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." } } }, @@ -4850,13 +4964,33 @@ "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", "properties": { "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\n" + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" }, "databricks_workspace_url": { "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" } } }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "properties": { + "private_key": { + "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" + }, + "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." + }, + "project_id": { + "description": "This is the Google Cloud project id that the service account is associated with." + }, + "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." + } + } + }, "name": { "description": "The name of the external model." }, @@ -4867,16 +5001,22 @@ "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" }, "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" + }, + "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" }, "microsoft_entra_tenant_id": { "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" }, "openai_api_base": { - "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" + "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" }, "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI or Azure OpenAI API key." + "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." + }, + "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." }, "openai_api_type": { "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" @@ -4896,12 +5036,15 @@ "description": "PaLM Config. Only required if the provider is 'palm'.", "properties": { "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key." + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." + }, + "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." } } }, "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'openai', and 'palm'.\",\n" + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" }, "task": { "description": "The task type of the external model." @@ -5307,6 +5450,9 @@ "driver_node_type_id": { "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster." + }, "gcp_attributes": { "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", "properties": { @@ -5501,7 +5647,7 @@ "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." }, "gateway_storage_name": { - "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" }, "gateway_storage_schema": { "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." @@ -5541,7 +5687,7 @@ "description": "Required. Schema name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -5581,7 +5727,7 @@ "description": "Required. Table name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -5661,6 +5807,9 @@ "description": "The absolute path of the notebook." } } + }, + "whl": { + "description": "URI of the whl to be installed." } } } @@ -5931,6 +6080,49 @@ } } } + }, + "schemas": { + "description": "", + "additionalproperties": { + "description": "", + "properties": { + "catalog_name": { + "description": "" + }, + "comment": { + "description": "" + }, + "grants": { + "description": "", + "items": { + "description": "", + "properties": { + "principal": { + "description": "" + }, + "privileges": { + "description": "", + "items": { + "description": "" + } + } + } + } + }, + "name": { + "description": "" + }, + "properties": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "storage_root": { + "description": "" + } + } + } } } }, @@ -6010,6 +6202,9 @@ "description": "" } } + }, + "type": { + "description": "" } } } @@ -6115,6 +6310,9 @@ "description": "" } } + }, + "type": { + "description": "" } } } diff --git a/cmd/account/budgets/budgets.go b/cmd/account/budgets/budgets.go index 82f7b9f01..6b47bb32c 100755 --- a/cmd/account/budgets/budgets.go +++ b/cmd/account/budgets/budgets.go @@ -19,16 +19,15 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "budgets", - Short: `These APIs manage budget configuration including notifications for exceeding a budget for a period.`, - Long: `These APIs manage budget configuration including notifications for exceeding a - budget for a period. They can also retrieve the status of each budget.`, + Short: `These APIs manage budget configurations for this account.`, + Long: `These APIs manage budget configurations for this account. Budgets enable you + to monitor usage across your account. You can set up budgets to either track + account-wide spending, or apply filters to track the spending of specific + teams, projects, or workspaces.`, GroupID: "billing", Annotations: map[string]string{ "package": "billing", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods @@ -52,23 +51,24 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *billing.WrappedBudget, + *billing.CreateBudgetConfigurationRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq billing.WrappedBudget + var createReq billing.CreateBudgetConfigurationRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "create" - cmd.Short = `Create a new budget.` - cmd.Long = `Create a new budget. + cmd.Short = `Create new budget.` + cmd.Long = `Create new budget. - Creates a new budget in the specified account.` + Create a new budget configuration for an account. For full details, see + https://docs.databricks.com/en/admin/account-settings/budgets.html.` cmd.Annotations = make(map[string]string) @@ -111,13 +111,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *billing.DeleteBudgetRequest, + *billing.DeleteBudgetConfigurationRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq billing.DeleteBudgetRequest + var deleteReq billing.DeleteBudgetConfigurationRequest // TODO: short flags @@ -125,35 +125,24 @@ func newDelete() *cobra.Command { cmd.Short = `Delete budget.` cmd.Long = `Delete budget. - Deletes the budget specified by its UUID. + Deletes a budget configuration for an account. Both account and budget + configuration are specified by ID. This cannot be undone. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No BUDGET_ID argument specified. Loading names for Budgets drop-down." - names, err := a.Budgets.BudgetWithStatusNameToBudgetIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Budgets drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Budget ID") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have budget id") - } deleteReq.BudgetId = args[0] err = a.Budgets.Delete(ctx, deleteReq) @@ -181,50 +170,38 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *billing.GetBudgetRequest, + *billing.GetBudgetConfigurationRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq billing.GetBudgetRequest + var getReq billing.GetBudgetConfigurationRequest // TODO: short flags cmd.Use = "get BUDGET_ID" - cmd.Short = `Get budget and its status.` - cmd.Long = `Get budget and its status. + cmd.Short = `Get budget.` + cmd.Long = `Get budget. - Gets the budget specified by its UUID, including noncumulative status for each - day that the budget is configured to include. + Gets a budget configuration for an account. Both account and budget + configuration are specified by ID. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No BUDGET_ID argument specified. Loading names for Budgets drop-down." - names, err := a.Budgets.BudgetWithStatusNameToBudgetIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Budgets drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Budget ID") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have budget id") - } getReq.BudgetId = args[0] response, err := a.Budgets.Get(ctx, getReq) @@ -252,25 +229,37 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *billing.ListBudgetConfigurationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq billing.ListBudgetConfigurationsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token received from a previous get all budget configurations call.`) + cmd.Use = "list" cmd.Short = `Get all budgets.` cmd.Long = `Get all budgets. - Gets all budgets associated with this account, including noncumulative status - for each day that the budget is configured to include.` + Gets all budgets associated with this account.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.Budgets.List(ctx) + + response := a.Budgets.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -280,7 +269,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -292,13 +281,13 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *billing.WrappedBudget, + *billing.UpdateBudgetConfigurationRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq billing.WrappedBudget + var updateReq billing.UpdateBudgetConfigurationRequest var updateJson flags.JsonFlag // TODO: short flags @@ -308,11 +297,11 @@ func newUpdate() *cobra.Command { cmd.Short = `Modify budget.` cmd.Long = `Modify budget. - Modifies a budget in this account. Budget properties are completely - overwritten. + Updates a budget configuration for an account. Both account and budget + configuration are specified by ID. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) @@ -336,11 +325,11 @@ func newUpdate() *cobra.Command { } updateReq.BudgetId = args[0] - err = a.Budgets.Update(ctx, updateReq) + response, err := a.Budgets.Update(ctx, updateReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. @@ -355,4 +344,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service Budgets +// end service budgets diff --git a/cmd/account/cmd.go b/cmd/account/cmd.go index 627d6d590..9b4bb8139 100644 --- a/cmd/account/cmd.go +++ b/cmd/account/cmd.go @@ -26,6 +26,7 @@ import ( account_settings "github.com/databricks/cli/cmd/account/settings" storage "github.com/databricks/cli/cmd/account/storage" account_storage_credentials "github.com/databricks/cli/cmd/account/storage-credentials" + usage_dashboards "github.com/databricks/cli/cmd/account/usage-dashboards" account_users "github.com/databricks/cli/cmd/account/users" vpc_endpoints "github.com/databricks/cli/cmd/account/vpc-endpoints" workspace_assignment "github.com/databricks/cli/cmd/account/workspace-assignment" @@ -40,7 +41,6 @@ func New() *cobra.Command { cmd.AddCommand(account_access_control.New()) cmd.AddCommand(billable_usage.New()) - cmd.AddCommand(budgets.New()) cmd.AddCommand(credentials.New()) cmd.AddCommand(custom_app_integration.New()) cmd.AddCommand(encryption_keys.New()) @@ -59,10 +59,12 @@ func New() *cobra.Command { cmd.AddCommand(account_settings.New()) cmd.AddCommand(storage.New()) cmd.AddCommand(account_storage_credentials.New()) + cmd.AddCommand(usage_dashboards.New()) cmd.AddCommand(account_users.New()) cmd.AddCommand(vpc_endpoints.New()) cmd.AddCommand(workspace_assignment.New()) cmd.AddCommand(workspaces.New()) + cmd.AddCommand(budgets.New()) // Register all groups with the parent command. groups := Groups() diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index ca9f69a35..5cdf422d7 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -3,8 +3,6 @@ package custom_app_integration import ( - "fmt" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" @@ -19,8 +17,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "custom-app-integration", - Short: `These APIs enable administrators to manage custom oauth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, - Long: `These APIs enable administrators to manage custom oauth app integrations, + Short: `These APIs enable administrators to manage custom OAuth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, + Long: `These APIs enable administrators to manage custom OAuth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, GroupID: "oauth2", @@ -62,7 +60,9 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&createReq.Confidential, "confidential", createReq.Confidential, `indicates if an oauth client-secret should be generated.`) + cmd.Flags().BoolVar(&createReq.Confidential, "confidential", createReq.Confidential, `This field indicates whether an OAuth client secret is required to authenticate this client.`) + cmd.Flags().StringVar(&createReq.Name, "name", createReq.Name, `Name of the custom OAuth app.`) + // TODO: array: redirect_urls // TODO: array: scopes // TODO: complex arg: token_access_policy @@ -72,11 +72,16 @@ func newCreate() *cobra.Command { Create Custom OAuth App Integration. - You can retrieve the custom oauth app integration via + You can retrieve the custom OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -87,8 +92,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := a.CustomAppIntegration.Create(ctx, createReq) @@ -131,10 +134,7 @@ func newDelete() *cobra.Command { cmd.Long = `Delete Custom OAuth App Integration. Delete an existing Custom OAuth App Integration. You can retrieve the custom - oauth app integration via :method:CustomAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -189,10 +189,7 @@ func newGet() *cobra.Command { cmd.Short = `Get OAuth Custom App Integration.` cmd.Long = `Get OAuth Custom App Integration. - Gets the Custom OAuth App Integration for the given integration id. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + Gets the Custom OAuth App Integration for the given integration id.` cmd.Annotations = make(map[string]string) @@ -233,25 +230,40 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *oauth2.ListCustomAppIntegrationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq oauth2.ListCustomAppIntegrationsRequest + + // TODO: short flags + + cmd.Flags().BoolVar(&listReq.IncludeCreatorUsername, "include-creator-username", listReq.IncludeCreatorUsername, ``) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" cmd.Short = `Get custom oauth app integrations.` cmd.Long = `Get custom oauth app integrations. - Get the list of custom oauth app integrations for the specified Databricks + Get the list of custom OAuth app integrations for the specified Databricks account` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.CustomAppIntegration.List(ctx) + + response := a.CustomAppIntegration.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -261,7 +273,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -293,10 +305,7 @@ func newUpdate() *cobra.Command { cmd.Long = `Updates Custom OAuth App Integration. Updates an existing custom OAuth App Integration. You can retrieve the custom - oauth app integration via :method:CustomAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/o-auth-published-apps/o-auth-published-apps.go b/cmd/account/o-auth-published-apps/o-auth-published-apps.go index 6573b0529..f1af17d2e 100755 --- a/cmd/account/o-auth-published-apps/o-auth-published-apps.go +++ b/cmd/account/o-auth-published-apps/o-auth-published-apps.go @@ -54,7 +54,7 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().Int64Var(&listReq.PageSize, "page-size", listReq.PageSize, `The max number of OAuth published apps to return.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The max number of OAuth published apps to return in one page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) cmd.Use = "list" diff --git a/cmd/account/published-app-integration/published-app-integration.go b/cmd/account/published-app-integration/published-app-integration.go index 32fed5cd0..5143d53cc 100755 --- a/cmd/account/published-app-integration/published-app-integration.go +++ b/cmd/account/published-app-integration/published-app-integration.go @@ -17,8 +17,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "published-app-integration", - Short: `These APIs enable administrators to manage published oauth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, - Long: `These APIs enable administrators to manage published oauth app integrations, + Short: `These APIs enable administrators to manage published OAuth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, + Long: `These APIs enable administrators to manage published OAuth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, GroupID: "oauth2", @@ -60,7 +60,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&createReq.AppId, "app-id", createReq.AppId, `app_id of the oauth published app integration.`) + cmd.Flags().StringVar(&createReq.AppId, "app-id", createReq.AppId, `App id of the OAuth published app integration.`) // TODO: complex arg: token_access_policy cmd.Use = "create" @@ -69,7 +69,7 @@ func newCreate() *cobra.Command { Create Published OAuth App Integration. - You can retrieve the published oauth app integration via + You can retrieve the published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -131,10 +131,7 @@ func newDelete() *cobra.Command { cmd.Long = `Delete Published OAuth App Integration. Delete an existing Published OAuth App Integration. You can retrieve the - published oauth app integration via :method:PublishedAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -189,10 +186,7 @@ func newGet() *cobra.Command { cmd.Short = `Get OAuth Published App Integration.` cmd.Long = `Get OAuth Published App Integration. - Gets the Published OAuth App Integration for the given integration id. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + Gets the Published OAuth App Integration for the given integration id.` cmd.Annotations = make(map[string]string) @@ -233,25 +227,39 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *oauth2.ListPublishedAppIntegrationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq oauth2.ListPublishedAppIntegrationsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" cmd.Short = `Get published oauth app integrations.` cmd.Long = `Get published oauth app integrations. - Get the list of published oauth app integrations for the specified Databricks + Get the list of published OAuth app integrations for the specified Databricks account` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.PublishedAppIntegration.List(ctx) + + response := a.PublishedAppIntegration.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -261,7 +269,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -292,10 +300,7 @@ func newUpdate() *cobra.Command { cmd.Long = `Updates Published OAuth App Integration. Updates an existing published OAuth App Integration. You can retrieve the - published oauth app integration via :method:PublishedAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/usage-dashboards/usage-dashboards.go b/cmd/account/usage-dashboards/usage-dashboards.go new file mode 100755 index 000000000..8a1c32476 --- /dev/null +++ b/cmd/account/usage-dashboards/usage-dashboards.go @@ -0,0 +1,164 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package usage_dashboards + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/billing" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "usage-dashboards", + Short: `These APIs manage usage dashboards for this account.`, + Long: `These APIs manage usage dashboards for this account. Usage dashboards enable + you to gain insights into your usage with pre-built dashboards: visualize + breakdowns, analyze tag attributions, and identify cost drivers.`, + GroupID: "billing", + Annotations: map[string]string{ + "package": "billing", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newGet()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *billing.CreateBillingUsageDashboardRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq billing.CreateBillingUsageDashboardRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().Var(&createReq.DashboardType, "dashboard-type", `Workspace level usage dashboard shows usage data for the specified workspace ID. Supported values: [USAGE_DASHBOARD_TYPE_GLOBAL, USAGE_DASHBOARD_TYPE_WORKSPACE]`) + cmd.Flags().Int64Var(&createReq.WorkspaceId, "workspace-id", createReq.WorkspaceId, `The workspace ID of the workspace in which the usage dashboard is created.`) + + cmd.Use = "create" + cmd.Short = `Create new usage dashboard.` + cmd.Long = `Create new usage dashboard. + + Create a usage dashboard specified by workspaceId, accountId, and dashboard + type.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } + + response, err := a.UsageDashboards.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *billing.GetBillingUsageDashboardRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq billing.GetBillingUsageDashboardRequest + + // TODO: short flags + + cmd.Flags().Var(&getReq.DashboardType, "dashboard-type", `Workspace level usage dashboard shows usage data for the specified workspace ID. Supported values: [USAGE_DASHBOARD_TYPE_GLOBAL, USAGE_DASHBOARD_TYPE_WORKSPACE]`) + cmd.Flags().Int64Var(&getReq.WorkspaceId, "workspace-id", getReq.WorkspaceId, `The workspace ID of the workspace in which the usage dashboard is created.`) + + cmd.Use = "get" + cmd.Short = `Get usage dashboard.` + cmd.Long = `Get usage dashboard. + + Get a usage dashboard specified by workspaceId, accountId, and dashboard type.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.UsageDashboards.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// end service UsageDashboards diff --git a/cmd/account/workspace-assignment/workspace-assignment.go b/cmd/account/workspace-assignment/workspace-assignment.go index b965d31ad..58468d09f 100755 --- a/cmd/account/workspace-assignment/workspace-assignment.go +++ b/cmd/account/workspace-assignment/workspace-assignment.go @@ -66,7 +66,7 @@ func newDelete() *cobra.Command { for the specified principal. Arguments: - WORKSPACE_ID: The workspace ID. + WORKSPACE_ID: The workspace ID for the account. PRINCIPAL_ID: The ID of the user, service principal, or group.` cmd.Annotations = make(map[string]string) @@ -247,6 +247,8 @@ func newUpdate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: permissions + cmd.Use = "update WORKSPACE_ID PRINCIPAL_ID" cmd.Short = `Create or update permissions assignment.` cmd.Long = `Create or update permissions assignment. @@ -255,7 +257,7 @@ func newUpdate() *cobra.Command { workspace for the specified principal. Arguments: - WORKSPACE_ID: The workspace ID. + WORKSPACE_ID: The workspace ID for the account. PRINCIPAL_ID: The ID of the user, service principal, or group.` cmd.Annotations = make(map[string]string) @@ -275,8 +277,6 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) if err != nil { diff --git a/cmd/cmd.go b/cmd/cmd.go index 5d835409f..5b53a4ae5 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/cmd/sync" "github.com/databricks/cli/cmd/version" "github.com/databricks/cli/cmd/workspace" + "github.com/databricks/cli/cmd/workspace/apps" "github.com/spf13/cobra" ) @@ -67,6 +68,7 @@ func New(ctx context.Context) *cobra.Command { // Add other subcommands. cli.AddCommand(api.New()) + cli.AddCommand(apps.New()) cli.AddCommand(auth.New()) cli.AddCommand(bundle.New()) cli.AddCommand(configure.New()) diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index 8754a560b..1e45fafe6 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -182,7 +182,7 @@ func TestInstallerWorksForReleases(t *testing.T) { w.Write(raw) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) @@ -249,8 +249,9 @@ func TestInstallerWorksForDevelopment(t *testing.T) { Path: filepath.Dir(t.TempDir()), }) }() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/2.0/clusters/list" { + if r.URL.Path == "/api/2.1/clusters/list" { respondWithJSON(t, w, compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -278,7 +279,7 @@ func TestInstallerWorksForDevelopment(t *testing.T) { }) return } - if r.URL.Path == "/api/2.0/clusters/spark-versions" { + if r.URL.Path == "/api/2.1/clusters/spark-versions" { respondWithJSON(t, w, compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { @@ -289,7 +290,7 @@ func TestInstallerWorksForDevelopment(t *testing.T) { }) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) @@ -387,7 +388,7 @@ func TestUpgraderWorksForReleases(t *testing.T) { w.Write(raw) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) diff --git a/cmd/root/auth_test.go b/cmd/root/auth_test.go index 486f587ef..9ba2a8fa9 100644 --- a/cmd/root/auth_test.go +++ b/cmd/root/auth_test.go @@ -111,6 +111,10 @@ func TestAccountClientOrPrompt(t *testing.T) { expectPrompts(t, accountPromptFn, &config.Config{ Host: "https://accounts.azuredatabricks.net/", AccountID: "1234", + + // Force SDK to not try and lookup the tenant ID from the host. + // The host above is invalid and will not be reachable. + AzureTenantID: "nonempty", }) }) @@ -165,6 +169,10 @@ func TestWorkspaceClientOrPrompt(t *testing.T) { t.Run("Prompt if no credential provider can be configured", func(t *testing.T) { expectPrompts(t, workspacePromptFn, &config.Config{ Host: "https://adb-1111.11.azuredatabricks.net/", + + // Force SDK to not try and lookup the tenant ID from the host. + // The host above is invalid and will not be reachable. + AzureTenantID: "nonempty", }) }) diff --git a/cmd/workspace/alerts-legacy/alerts-legacy.go b/cmd/workspace/alerts-legacy/alerts-legacy.go new file mode 100755 index 000000000..1046b1124 --- /dev/null +++ b/cmd/workspace/alerts-legacy/alerts-legacy.go @@ -0,0 +1,388 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package alerts_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "alerts-legacy", + Short: `The alerts API can be used to perform CRUD operations on alerts.`, + Long: `The alerts API can be used to perform CRUD operations on alerts. An alert is a + Databricks SQL object that periodically runs a query, evaluates a condition of + its result, and notifies one or more users and/or notification destinations if + the condition was met. Alerts can be scheduled using the sql_task type of + the Jobs API, e.g. :method:jobs/create. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.CreateAlert, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.CreateAlert + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createReq.Parent, "parent", createReq.Parent, `The identifier of the workspace folder containing the object.`) + cmd.Flags().IntVar(&createReq.Rearm, "rearm", createReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + + cmd.Use = "create" + cmd.Short = `Create an alert.` + cmd.Long = `Create an alert. + + Creates an alert. An alert is a Databricks SQL object that periodically runs a + query, evaluates a condition of its result, and notifies users or notification + destinations if the condition was met. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.AlertsLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteAlertsLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteAlertsLegacyRequest + + // TODO: short flags + + cmd.Use = "delete ALERT_ID" + cmd.Short = `Delete an alert.` + cmd.Long = `Delete an alert. + + Deletes an alert. Deleted alerts are no longer accessible and cannot be + restored. **Note**: Unlike queries and dashboards, alerts cannot be moved to + the trash. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts Legacy drop-down." + names, err := w.AlertsLegacy.LegacyAlertNameToIdMap(ctx) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Alerts Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + deleteReq.AlertId = args[0] + + err = w.AlertsLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *sql.GetAlertsLegacyRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq sql.GetAlertsLegacyRequest + + // TODO: short flags + + cmd.Use = "get ALERT_ID" + cmd.Short = `Get an alert.` + cmd.Long = `Get an alert. + + Gets an alert. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/get instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts Legacy drop-down." + names, err := w.AlertsLegacy.LegacyAlertNameToIdMap(ctx) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Alerts Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + getReq.AlertId = args[0] + + response, err := w.AlertsLegacy.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + cmd.Use = "list" + cmd.Short = `Get alerts.` + cmd.Long = `Get alerts. + + Gets a list of alerts. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/list instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + response, err := w.AlertsLegacy.List(ctx) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.EditAlert, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.EditAlert + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().IntVar(&updateReq.Rearm, "rearm", updateReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + + cmd.Use = "update ALERT_ID" + cmd.Short = `Update an alert.` + cmd.Long = `Update an alert. + + Updates an alert. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + updateReq.AlertId = args[0] + + err = w.AlertsLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service AlertsLegacy diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go index 61c1e0eab..cfaa3f55f 100755 --- a/cmd/workspace/alerts/alerts.go +++ b/cmd/workspace/alerts/alerts.go @@ -24,12 +24,7 @@ func New() *cobra.Command { Databricks SQL object that periodically runs a query, evaluates a condition of its result, and notifies one or more users and/or notification destinations if the condition was met. Alerts can be scheduled using the sql_task type of - the Jobs API, e.g. :method:jobs/create. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + the Jobs API, e.g. :method:jobs/create.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -57,36 +52,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.CreateAlert, + *sql.CreateAlertRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.CreateAlert + var createReq sql.CreateAlertRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&createReq.Parent, "parent", createReq.Parent, `The identifier of the workspace folder containing the object.`) - cmd.Flags().IntVar(&createReq.Rearm, "rearm", createReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + // TODO: complex arg: alert cmd.Use = "create" cmd.Short = `Create an alert.` cmd.Long = `Create an alert. - Creates an alert. An alert is a Databricks SQL object that periodically runs a - query, evaluates a condition of its result, and notifies users or notification - destinations if the condition was met. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Creates an alert.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -97,8 +89,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.Alerts.Create(ctx, createReq) @@ -126,28 +116,23 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteAlertRequest, + *sql.TrashAlertRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteAlertRequest + var deleteReq sql.TrashAlertRequest // TODO: short flags - cmd.Use = "delete ALERT_ID" + cmd.Use = "delete ID" cmd.Short = `Delete an alert.` cmd.Long = `Delete an alert. - Deletes an alert. Deleted alerts are no longer accessible and cannot be - restored. **Note**: Unlike queries and dashboards, alerts cannot be moved to - the trash. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Moves an alert to the trash. Trashed alerts immediately disappear from + searches and list views, and can no longer trigger. You can restore a trashed + alert through the UI. A trashed alert is permanently deleted after 30 days.` cmd.Annotations = make(map[string]string) @@ -158,8 +143,8 @@ func newDelete() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts drop-down." - names, err := w.Alerts.AlertNameToIdMap(ctx) + promptSpinner <- "No ID argument specified. Loading names for Alerts drop-down." + names, err := w.Alerts.ListAlertsResponseAlertDisplayNameToIdMap(ctx, sql.ListAlertsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Alerts drop-down. Please manually specify required arguments. Original error: %w", err) @@ -173,7 +158,7 @@ func newDelete() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - deleteReq.AlertId = args[0] + deleteReq.Id = args[0] err = w.Alerts.Delete(ctx, deleteReq) if err != nil { @@ -210,16 +195,11 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get ALERT_ID" + cmd.Use = "get ID" cmd.Short = `Get an alert.` cmd.Long = `Get an alert. - Gets an alert. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets an alert.` cmd.Annotations = make(map[string]string) @@ -230,8 +210,8 @@ func newGet() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts drop-down." - names, err := w.Alerts.AlertNameToIdMap(ctx) + promptSpinner <- "No ID argument specified. Loading names for Alerts drop-down." + names, err := w.Alerts.ListAlertsResponseAlertDisplayNameToIdMap(ctx, sql.ListAlertsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Alerts drop-down. Please manually specify required arguments. Original error: %w", err) @@ -245,7 +225,7 @@ func newGet() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - getReq.AlertId = args[0] + getReq.Id = args[0] response, err := w.Alerts.Get(ctx, getReq) if err != nil { @@ -272,33 +252,41 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *sql.ListAlertsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq sql.ListAlertsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" - cmd.Short = `Get alerts.` - cmd.Long = `Get alerts. + cmd.Short = `List alerts.` + cmd.Long = `List alerts. - Gets a list of alerts. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of alerts accessible to the user, ordered by creation time. + **Warning:** Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Alerts.List(ctx) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + + response := w.Alerts.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -307,7 +295,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -319,35 +307,44 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.EditAlert, + *sql.UpdateAlertRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.EditAlert + var updateReq sql.UpdateAlertRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().IntVar(&updateReq.Rearm, "rearm", updateReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + // TODO: complex arg: alert - cmd.Use = "update ALERT_ID" + cmd.Use = "update ID UPDATE_MASK" cmd.Short = `Update an alert.` cmd.Long = `Update an alert. Updates an alert. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + + Arguments: + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) return check(cmd, args) } @@ -361,16 +358,17 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } - updateReq.AlertId = args[0] + updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } - err = w.Alerts.Update(ctx, updateReq) + response, err := w.Alerts.Update(ctx, updateReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index 1572d4f4b..bc3fbe920 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -24,9 +24,9 @@ func New() *cobra.Command { Long: `Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`, - GroupID: "serving", + GroupID: "apps", Annotations: map[string]string{ - "package": "serving", + "package": "apps", }, // This service is being previewed; hide from help output. @@ -39,12 +39,15 @@ func New() *cobra.Command { cmd.AddCommand(newDeploy()) cmd.AddCommand(newGet()) cmd.AddCommand(newGetDeployment()) - cmd.AddCommand(newGetEnvironment()) + cmd.AddCommand(newGetPermissionLevels()) + cmd.AddCommand(newGetPermissions()) cmd.AddCommand(newList()) cmd.AddCommand(newListDeployments()) + cmd.AddCommand(newSetPermissions()) cmd.AddCommand(newStart()) cmd.AddCommand(newStop()) cmd.AddCommand(newUpdate()) + cmd.AddCommand(newUpdatePermissions()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -60,13 +63,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *serving.CreateAppRequest, + *apps.CreateAppRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq serving.CreateAppRequest + var createReq apps.CreateAppRequest var createJson flags.JsonFlag var createSkipWait bool @@ -126,7 +129,7 @@ func newCreate() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *serving.App) { + info, err := wait.OnProgress(func(i *apps.App) { if i.Status == nil { return } @@ -162,13 +165,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *serving.DeleteAppRequest, + *apps.DeleteAppRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq serving.DeleteAppRequest + var deleteReq apps.DeleteAppRequest // TODO: short flags @@ -220,13 +223,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deployOverrides []func( *cobra.Command, - *serving.CreateAppDeploymentRequest, + *apps.CreateAppDeploymentRequest, ) func newDeploy() *cobra.Command { cmd := &cobra.Command{} - var deployReq serving.CreateAppDeploymentRequest + var deployReq apps.CreateAppDeploymentRequest var deployJson flags.JsonFlag var deploySkipWait bool @@ -237,7 +240,9 @@ func newDeploy() *cobra.Command { // TODO: short flags cmd.Flags().Var(&deployJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH MODE" + cmd.Flags().Var(&deployReq.Mode, "mode", `The mode of which the deployment will manage the source code. Supported values: [AUTO_SYNC, SNAPSHOT]`) + + cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH" cmd.Short = `Create an app deployment.` cmd.Long = `Create an app deployment. @@ -251,8 +256,7 @@ func newDeploy() *cobra.Command { deployed app. The former refers to the original source code location of the app in the workspace during deployment creation, whereas the latter provides a system generated stable snapshotted source code path used by - the deployment. - MODE: The mode of which the deployment will manage the source code.` + the deployment.` cmd.Annotations = make(map[string]string) @@ -260,11 +264,11 @@ func newDeploy() *cobra.Command { if cmd.Flags().Changed("json") { err := root.ExactArgs(1)(cmd, args) if err != nil { - return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path', 'mode' in your JSON input") + return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input") } return nil } - check := root.ExactArgs(3) + check := root.ExactArgs(2) return check(cmd, args) } @@ -283,12 +287,6 @@ func newDeploy() *cobra.Command { if !cmd.Flags().Changed("json") { deployReq.SourceCodePath = args[1] } - if !cmd.Flags().Changed("json") { - _, err = fmt.Sscan(args[2], &deployReq.Mode) - if err != nil { - return fmt.Errorf("invalid MODE: %s", args[2]) - } - } wait, err := w.Apps.Deploy(ctx, deployReq) if err != nil { @@ -298,7 +296,7 @@ func newDeploy() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *serving.AppDeployment) { + info, err := wait.OnProgress(func(i *apps.AppDeployment) { if i.Status == nil { return } @@ -334,13 +332,13 @@ func newDeploy() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *serving.GetAppRequest, + *apps.GetAppRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq serving.GetAppRequest + var getReq apps.GetAppRequest // TODO: short flags @@ -392,13 +390,13 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getDeploymentOverrides []func( *cobra.Command, - *serving.GetAppDeploymentRequest, + *apps.GetAppDeploymentRequest, ) func newGetDeployment() *cobra.Command { cmd := &cobra.Command{} - var getDeploymentReq serving.GetAppDeploymentRequest + var getDeploymentReq apps.GetAppDeploymentRequest // TODO: short flags @@ -447,30 +445,30 @@ func newGetDeployment() *cobra.Command { return cmd } -// start get-environment command +// start get-permission-levels command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getEnvironmentOverrides []func( +var getPermissionLevelsOverrides []func( *cobra.Command, - *serving.GetAppEnvironmentRequest, + *apps.GetAppPermissionLevelsRequest, ) -func newGetEnvironment() *cobra.Command { +func newGetPermissionLevels() *cobra.Command { cmd := &cobra.Command{} - var getEnvironmentReq serving.GetAppEnvironmentRequest + var getPermissionLevelsReq apps.GetAppPermissionLevelsRequest // TODO: short flags - cmd.Use = "get-environment NAME" - cmd.Short = `Get app environment.` - cmd.Long = `Get app environment. + cmd.Use = "get-permission-levels APP_NAME" + cmd.Short = `Get app permission levels.` + cmd.Long = `Get app permission levels. - Retrieves app environment. + Gets the permission levels that a user can have on an object. Arguments: - NAME: The name of the app.` + APP_NAME: The app for which to get or manage permissions.` cmd.Annotations = make(map[string]string) @@ -484,9 +482,9 @@ func newGetEnvironment() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getEnvironmentReq.Name = args[0] + getPermissionLevelsReq.AppName = args[0] - response, err := w.Apps.GetEnvironment(ctx, getEnvironmentReq) + response, err := w.Apps.GetPermissionLevels(ctx, getPermissionLevelsReq) if err != nil { return err } @@ -498,8 +496,67 @@ func newGetEnvironment() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getEnvironmentOverrides { - fn(cmd, &getEnvironmentReq) + for _, fn := range getPermissionLevelsOverrides { + fn(cmd, &getPermissionLevelsReq) + } + + return cmd +} + +// start get-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getPermissionsOverrides []func( + *cobra.Command, + *apps.GetAppPermissionsRequest, +) + +func newGetPermissions() *cobra.Command { + cmd := &cobra.Command{} + + var getPermissionsReq apps.GetAppPermissionsRequest + + // TODO: short flags + + cmd.Use = "get-permissions APP_NAME" + cmd.Short = `Get app permissions.` + cmd.Long = `Get app permissions. + + Gets the permissions of an app. Apps can inherit permissions from their root + object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getPermissionsReq.AppName = args[0] + + response, err := w.Apps.GetPermissions(ctx, getPermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getPermissionsOverrides { + fn(cmd, &getPermissionsReq) } return cmd @@ -511,13 +568,13 @@ func newGetEnvironment() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, - *serving.ListAppsRequest, + *apps.ListAppsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} - var listReq serving.ListAppsRequest + var listReq apps.ListAppsRequest // TODO: short flags @@ -564,13 +621,13 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listDeploymentsOverrides []func( *cobra.Command, - *serving.ListAppDeploymentsRequest, + *apps.ListAppDeploymentsRequest, ) func newListDeployments() *cobra.Command { cmd := &cobra.Command{} - var listDeploymentsReq serving.ListAppDeploymentsRequest + var listDeploymentsReq apps.ListAppDeploymentsRequest // TODO: short flags @@ -616,20 +673,94 @@ func newListDeployments() *cobra.Command { return cmd } +// start set-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var setPermissionsOverrides []func( + *cobra.Command, + *apps.AppPermissionsRequest, +) + +func newSetPermissions() *cobra.Command { + cmd := &cobra.Command{} + + var setPermissionsReq apps.AppPermissionsRequest + var setPermissionsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&setPermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: access_control_list + + cmd.Use = "set-permissions APP_NAME" + cmd.Short = `Set app permissions.` + cmd.Long = `Set app permissions. + + Sets permissions on an app. Apps can inherit permissions from their root + object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = setPermissionsJson.Unmarshal(&setPermissionsReq) + if err != nil { + return err + } + } + setPermissionsReq.AppName = args[0] + + response, err := w.Apps.SetPermissions(ctx, setPermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range setPermissionsOverrides { + fn(cmd, &setPermissionsReq) + } + + return cmd +} + // start start command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var startOverrides []func( *cobra.Command, - *serving.StartAppRequest, + *apps.StartAppRequest, ) func newStart() *cobra.Command { cmd := &cobra.Command{} - var startReq serving.StartAppRequest + var startReq apps.StartAppRequest + var startSkipWait bool + var startTimeout time.Duration + + cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach SUCCEEDED state`) + cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`) // TODO: short flags cmd.Use = "start NAME" @@ -655,11 +786,30 @@ func newStart() *cobra.Command { startReq.Name = args[0] - response, err := w.Apps.Start(ctx, startReq) + wait, err := w.Apps.Start(ctx, startReq) if err != nil { return err } - return cmdio.Render(ctx, response) + if startSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *apps.AppDeployment) { + if i.Status == nil { + return + } + status := i.Status.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.Status != nil { + statusMessage = i.Status.Message + } + spinner <- statusMessage + }).GetWithTimeout(startTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) } // Disable completions since they are not applicable. @@ -680,13 +830,13 @@ func newStart() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var stopOverrides []func( *cobra.Command, - *serving.StopAppRequest, + *apps.StopAppRequest, ) func newStop() *cobra.Command { cmd := &cobra.Command{} - var stopReq serving.StopAppRequest + var stopReq apps.StopAppRequest // TODO: short flags @@ -738,13 +888,13 @@ func newStop() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *serving.UpdateAppRequest, + *apps.UpdateAppRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq serving.UpdateAppRequest + var updateReq apps.UpdateAppRequest var updateJson flags.JsonFlag // TODO: short flags @@ -801,4 +951,73 @@ func newUpdate() *cobra.Command { return cmd } +// start update-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updatePermissionsOverrides []func( + *cobra.Command, + *apps.AppPermissionsRequest, +) + +func newUpdatePermissions() *cobra.Command { + cmd := &cobra.Command{} + + var updatePermissionsReq apps.AppPermissionsRequest + var updatePermissionsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updatePermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: access_control_list + + cmd.Use = "update-permissions APP_NAME" + cmd.Short = `Update app permissions.` + cmd.Long = `Update app permissions. + + Updates the permissions on an app. Apps can inherit permissions from their + root object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if err != nil { + return err + } + } + updatePermissionsReq.AppName = args[0] + + response, err := w.Apps.UpdatePermissions(ctx, updatePermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updatePermissionsOverrides { + fn(cmd, &updatePermissionsReq) + } + + return cmd +} + // end service Apps diff --git a/cmd/workspace/cluster-policies/cluster-policies.go b/cmd/workspace/cluster-policies/cluster-policies.go index 8129db477..830d44ca3 100755 --- a/cmd/workspace/cluster-policies/cluster-policies.go +++ b/cmd/workspace/cluster-policies/cluster-policies.go @@ -90,30 +90,20 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `Additional human-readable description of the cluster policy.`) // TODO: array: libraries cmd.Flags().Int64Var(&createReq.MaxClustersPerUser, "max-clusters-per-user", createReq.MaxClustersPerUser, `Max number of clusters per user that can be active using this policy.`) + cmd.Flags().StringVar(&createReq.Name, "name", createReq.Name, `Cluster Policy name requested by the user.`) cmd.Flags().StringVar(&createReq.PolicyFamilyDefinitionOverrides, "policy-family-definition-overrides", createReq.PolicyFamilyDefinitionOverrides, `Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).`) cmd.Flags().StringVar(&createReq.PolicyFamilyId, "policy-family-id", createReq.PolicyFamilyId, `ID of the policy family.`) - cmd.Use = "create NAME" + cmd.Use = "create" cmd.Short = `Create a new policy.` cmd.Long = `Create a new policy. - Creates a new policy with prescribed settings. - - Arguments: - NAME: Cluster Policy name requested by the user. This has to be unique. Length - must be between 1 and 100 characters.` + Creates a new policy with prescribed settings.` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(0)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input") - } - return nil - } - check := root.ExactArgs(1) + check := root.ExactArgs(0) return check(cmd, args) } @@ -128,9 +118,6 @@ func newCreate() *cobra.Command { return err } } - if !cmd.Flags().Changed("json") { - createReq.Name = args[0] - } response, err := w.ClusterPolicies.Create(ctx, createReq) if err != nil { @@ -264,10 +251,11 @@ func newEdit() *cobra.Command { cmd.Flags().StringVar(&editReq.Description, "description", editReq.Description, `Additional human-readable description of the cluster policy.`) // TODO: array: libraries cmd.Flags().Int64Var(&editReq.MaxClustersPerUser, "max-clusters-per-user", editReq.MaxClustersPerUser, `Max number of clusters per user that can be active using this policy.`) + cmd.Flags().StringVar(&editReq.Name, "name", editReq.Name, `Cluster Policy name requested by the user.`) cmd.Flags().StringVar(&editReq.PolicyFamilyDefinitionOverrides, "policy-family-definition-overrides", editReq.PolicyFamilyDefinitionOverrides, `Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).`) cmd.Flags().StringVar(&editReq.PolicyFamilyId, "policy-family-id", editReq.PolicyFamilyId, `ID of the policy family.`) - cmd.Use = "edit POLICY_ID NAME" + cmd.Use = "edit POLICY_ID" cmd.Short = `Update a cluster policy.` cmd.Long = `Update a cluster policy. @@ -275,9 +263,7 @@ func newEdit() *cobra.Command { governed by the previous policy invalid. Arguments: - POLICY_ID: The ID of the policy to update. - NAME: Cluster Policy name requested by the user. This has to be unique. Length - must be between 1 and 100 characters.` + POLICY_ID: The ID of the policy to update.` cmd.Annotations = make(map[string]string) @@ -285,12 +271,11 @@ func newEdit() *cobra.Command { if cmd.Flags().Changed("json") { err := root.ExactArgs(0)(cmd, args) if err != nil { - return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id', 'name' in your JSON input") + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id' in your JSON input") } return nil } - check := root.ExactArgs(2) - return check(cmd, args) + return nil } cmd.PreRunE = root.MustWorkspaceClient @@ -303,13 +288,26 @@ func newEdit() *cobra.Command { if err != nil { return err } - } - if !cmd.Flags().Changed("json") { + } else { + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No POLICY_ID argument specified. Loading names for Cluster Policies drop-down." + names, err := w.ClusterPolicies.PolicyNameToPolicyIdMap(ctx, compute.ListClusterPoliciesRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Cluster Policies drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "The ID of the policy to update") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have the id of the policy to update") + } editReq.PolicyId = args[0] } - if !cmd.Flags().Changed("json") { - editReq.Name = args[1] - } err = w.ClusterPolicies.Edit(ctx, editReq) if err != nil { @@ -353,7 +351,7 @@ func newGet() *cobra.Command { Get a cluster policy entity. Creation and editing is available to admins only. Arguments: - POLICY_ID: Canonical unique identifier for the cluster policy.` + POLICY_ID: Canonical unique identifier for the Cluster Policy.` cmd.Annotations = make(map[string]string) @@ -370,7 +368,7 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Cluster Policies drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "Canonical unique identifier for the cluster policy") + id, err := cmdio.Select(ctx, names, "Canonical unique identifier for the Cluster Policy") if err != nil { return err } diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index abde1bb71..a64a6ab7c 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -43,11 +43,10 @@ func New() *cobra.Command { manually terminate and restart an all-purpose cluster. Multiple users can share such clusters to do collaborative interactive analysis. - IMPORTANT: Databricks retains cluster configuration information for up to 200 - all-purpose clusters terminated in the last 30 days and up to 30 job clusters - recently terminated by the job scheduler. To keep an all-purpose cluster - configuration even after it has been terminated for more than 30 days, an - administrator can pin a cluster to the cluster list.`, + IMPORTANT: Databricks retains cluster configuration information for terminated + clusters for 30 days. To keep an all-purpose cluster configuration even after + it has been terminated for more than 30 days, an administrator can pin a + cluster to the cluster list.`, GroupID: "compute", Annotations: map[string]string{ "package": "compute", @@ -74,6 +73,7 @@ func New() *cobra.Command { cmd.AddCommand(newSparkVersions()) cmd.AddCommand(newStart()) cmd.AddCommand(newUnpin()) + cmd.AddCommand(newUpdate()) cmd.AddCommand(newUpdatePermissions()) // Apply optional overrides to this command. @@ -885,21 +885,18 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().StringVar(&listReq.CanUseClient, "can-use-client", listReq.CanUseClient, `Filter clusters based on what type of client it can be used for.`) + // TODO: complex arg: filter_by + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Use this field to specify the maximum number of results to be returned by the server.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Use next_page_token or prev_page_token returned from the previous request to list the next or previous page of clusters respectively.`) + // TODO: complex arg: sort_by cmd.Use = "list" - cmd.Short = `List all clusters.` - cmd.Long = `List all clusters. + cmd.Short = `List clusters.` + cmd.Long = `List clusters. - Return information about all pinned clusters, active clusters, up to 200 of - the most recently terminated all-purpose clusters in the past 30 days, and up - to 30 of the most recently terminated job clusters in the past 30 days. - - For example, if there is 1 pinned cluster, 4 active clusters, 45 terminated - all-purpose clusters in the past 30 days, and 50 terminated job clusters in - the past 30 days, then this API returns the 1 pinned cluster, 4 active - clusters, all 45 terminated all-purpose clusters, and the 30 most recently - terminated job clusters.` + Return information about all pinned and active clusters, and all clusters + terminated within the last 30 days. Clusters terminated prior to this period + are not included.` cmd.Annotations = make(map[string]string) @@ -1753,6 +1750,117 @@ func newUnpin() *cobra.Command { return cmd } +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *compute.UpdateCluster, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq compute.UpdateCluster + var updateJson flags.JsonFlag + + var updateSkipWait bool + var updateTimeout time.Duration + + cmd.Flags().BoolVar(&updateSkipWait, "no-wait", updateSkipWait, `do not wait to reach RUNNING state`) + cmd.Flags().DurationVar(&updateTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach RUNNING state`) + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: cluster + + cmd.Use = "update CLUSTER_ID UPDATE_MASK" + cmd.Short = `Update cluster configuration (partial).` + cmd.Long = `Update cluster configuration (partial). + + Updates the configuration of a cluster to match the partial set of attributes + and size. Denote which fields to update using the update_mask field in the + request body. A cluster can be updated if it is in a RUNNING or TERMINATED + state. If a cluster is updated while in a RUNNING state, it will be + restarted so that the new attributes can take effect. If a cluster is updated + while in a TERMINATED state, it will remain TERMINATED. The updated + attributes will take effect the next time the cluster is started using the + clusters/start API. Attempts to update a cluster in any other state will be + rejected with an INVALID_STATE error code. Clusters created by the + Databricks Jobs service cannot be updated. + + Arguments: + CLUSTER_ID: ID of the cluster. + UPDATE_MASK: Specifies which fields of the cluster will be updated. This is required in + the POST request. The update mask should be supplied as a single string. + To specify multiple fields, separate them with commas (no spaces). To + delete a field from a cluster configuration, add it to the update_mask + string but omit it from the cluster object.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'cluster_id', 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + if !cmd.Flags().Changed("json") { + updateReq.ClusterId = args[0] + } + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } + + wait, err := w.Clusters.Update(ctx, updateReq) + if err != nil { + return err + } + if updateSkipWait { + return nil + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *compute.ClusterDetails) { + statusMessage := i.StateMessage + spinner <- statusMessage + }).GetWithTimeout(updateTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + // start update-permissions command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 7ad9389a8..75664c79c 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -4,6 +4,7 @@ package workspace import ( alerts "github.com/databricks/cli/cmd/workspace/alerts" + alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy" apps "github.com/databricks/cli/cmd/workspace/apps" artifact_allowlists "github.com/databricks/cli/cmd/workspace/artifact-allowlists" catalogs "github.com/databricks/cli/cmd/workspace/catalogs" @@ -24,6 +25,7 @@ import ( experiments "github.com/databricks/cli/cmd/workspace/experiments" external_locations "github.com/databricks/cli/cmd/workspace/external-locations" functions "github.com/databricks/cli/cmd/workspace/functions" + genie "github.com/databricks/cli/cmd/workspace/genie" git_credentials "github.com/databricks/cli/cmd/workspace/git-credentials" global_init_scripts "github.com/databricks/cli/cmd/workspace/global-init-scripts" grants "github.com/databricks/cli/cmd/workspace/grants" @@ -37,6 +39,7 @@ import ( metastores "github.com/databricks/cli/cmd/workspace/metastores" model_registry "github.com/databricks/cli/cmd/workspace/model-registry" model_versions "github.com/databricks/cli/cmd/workspace/model-versions" + notification_destinations "github.com/databricks/cli/cmd/workspace/notification-destinations" online_tables "github.com/databricks/cli/cmd/workspace/online-tables" permission_migration "github.com/databricks/cli/cmd/workspace/permission-migration" permissions "github.com/databricks/cli/cmd/workspace/permissions" @@ -52,8 +55,10 @@ import ( providers "github.com/databricks/cli/cmd/workspace/providers" quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors" queries "github.com/databricks/cli/cmd/workspace/queries" + queries_legacy "github.com/databricks/cli/cmd/workspace/queries-legacy" query_history "github.com/databricks/cli/cmd/workspace/query-history" query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations" + query_visualizations_legacy "github.com/databricks/cli/cmd/workspace/query-visualizations-legacy" recipient_activation "github.com/databricks/cli/cmd/workspace/recipient-activation" recipients "github.com/databricks/cli/cmd/workspace/recipients" registered_models "github.com/databricks/cli/cmd/workspace/registered-models" @@ -85,6 +90,7 @@ func All() []*cobra.Command { var out []*cobra.Command out = append(out, alerts.New()) + out = append(out, alerts_legacy.New()) out = append(out, apps.New()) out = append(out, artifact_allowlists.New()) out = append(out, catalogs.New()) @@ -105,6 +111,7 @@ func All() []*cobra.Command { out = append(out, experiments.New()) out = append(out, external_locations.New()) out = append(out, functions.New()) + out = append(out, genie.New()) out = append(out, git_credentials.New()) out = append(out, global_init_scripts.New()) out = append(out, grants.New()) @@ -118,6 +125,7 @@ func All() []*cobra.Command { out = append(out, metastores.New()) out = append(out, model_registry.New()) out = append(out, model_versions.New()) + out = append(out, notification_destinations.New()) out = append(out, online_tables.New()) out = append(out, permission_migration.New()) out = append(out, permissions.New()) @@ -133,8 +141,10 @@ func All() []*cobra.Command { out = append(out, providers.New()) out = append(out, quality_monitors.New()) out = append(out, queries.New()) + out = append(out, queries_legacy.New()) out = append(out, query_history.New()) out = append(out, query_visualizations.New()) + out = append(out, query_visualizations_legacy.New()) out = append(out, recipient_activation.New()) out = append(out, recipients.New()) out = append(out, registered_models.New()) diff --git a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go index 6f3ba4b42..46fd27c6f 100755 --- a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go +++ b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go @@ -22,9 +22,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go index d176e5b39..92f61789f 100755 --- a/cmd/workspace/consumer-installations/consumer-installations.go +++ b/cmd/workspace/consumer-installations/consumer-installations.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go index 18f3fb39e..5a8f76e36 100755 --- a/cmd/workspace/consumer-listings/consumer-listings.go +++ b/cmd/workspace/consumer-listings/consumer-listings.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods @@ -186,14 +183,12 @@ func newList() *cobra.Command { // TODO: array: assets // TODO: array: categories - cmd.Flags().BoolVar(&listReq.IsAscending, "is-ascending", listReq.IsAscending, ``) cmd.Flags().BoolVar(&listReq.IsFree, "is-free", listReq.IsFree, `Filters each listing based on if it is free.`) cmd.Flags().BoolVar(&listReq.IsPrivateExchange, "is-private-exchange", listReq.IsPrivateExchange, `Filters each listing based on if it is a private exchange.`) cmd.Flags().BoolVar(&listReq.IsStaffPick, "is-staff-pick", listReq.IsStaffPick, `Filters each listing based on whether it is a staff pick.`) cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) // TODO: array: provider_ids - cmd.Flags().Var(&listReq.SortBy, "sort-by", `Criteria for sorting the resulting set of listings. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`) // TODO: array: tags cmd.Use = "list" @@ -249,13 +244,11 @@ func newSearch() *cobra.Command { // TODO: array: assets // TODO: array: categories - cmd.Flags().BoolVar(&searchReq.IsAscending, "is-ascending", searchReq.IsAscending, ``) cmd.Flags().BoolVar(&searchReq.IsFree, "is-free", searchReq.IsFree, ``) cmd.Flags().BoolVar(&searchReq.IsPrivateExchange, "is-private-exchange", searchReq.IsPrivateExchange, ``) cmd.Flags().IntVar(&searchReq.PageSize, "page-size", searchReq.PageSize, ``) cmd.Flags().StringVar(&searchReq.PageToken, "page-token", searchReq.PageToken, ``) // TODO: array: provider_ids - cmd.Flags().Var(&searchReq.SortBy, "sort-by", `. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`) cmd.Use = "search QUERY" cmd.Short = `Search listings.` diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go index c55ca4ee1..8b0af3cc6 100755 --- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go +++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-providers/consumer-providers.go b/cmd/workspace/consumer-providers/consumer-providers.go index 579a89516..ab84249e9 100755 --- a/cmd/workspace/consumer-providers/consumer-providers.go +++ b/cmd/workspace/consumer-providers/consumer-providers.go @@ -24,9 +24,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/data-sources/data-sources.go b/cmd/workspace/data-sources/data-sources.go index f310fe50a..9f8a9dcd7 100755 --- a/cmd/workspace/data-sources/data-sources.go +++ b/cmd/workspace/data-sources/data-sources.go @@ -27,10 +27,10 @@ func New() *cobra.Command { grep to search the response from this API for the name of your SQL warehouse as it appears in Databricks SQL. - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] + **Note**: A new version of the Databricks SQL API is now available. [Learn + more] - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -67,10 +67,10 @@ func newList() *cobra.Command { fields that appear in this API response are enumerated for clarity. However, you need only a SQL warehouse's id to create new queries against it. - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:warehouses/list instead. [Learn more] - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go new file mode 100755 index 000000000..e4a059091 --- /dev/null +++ b/cmd/workspace/genie/genie.go @@ -0,0 +1,437 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package genie + +import ( + "fmt" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie", + Short: `Genie provides a no-code experience for business users, powered by AI/BI.`, + Long: `Genie provides a no-code experience for business users, powered by AI/BI. + Analysts set up spaces that business users can use to ask questions using + natural language. Genie uses data registered to Unity Catalog and requires at + least CAN USE permission on a Pro or Serverless SQL warehouse. Also, + Databricks Assistant must be enabled.`, + GroupID: "dashboards", + Annotations: map[string]string{ + "package": "dashboards", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCreateMessage()) + cmd.AddCommand(newExecuteMessageQuery()) + cmd.AddCommand(newGetMessage()) + cmd.AddCommand(newGetMessageQueryResult()) + cmd.AddCommand(newStartConversation()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create-message command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createMessageOverrides []func( + *cobra.Command, + *dashboards.GenieCreateConversationMessageRequest, +) + +func newCreateMessage() *cobra.Command { + cmd := &cobra.Command{} + + var createMessageReq dashboards.GenieCreateConversationMessageRequest + var createMessageJson flags.JsonFlag + + var createMessageSkipWait bool + var createMessageTimeout time.Duration + + cmd.Flags().BoolVar(&createMessageSkipWait, "no-wait", createMessageSkipWait, `do not wait to reach COMPLETED state`) + cmd.Flags().DurationVar(&createMessageTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach COMPLETED state`) + // TODO: short flags + cmd.Flags().Var(&createMessageJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create-message SPACE_ID CONVERSATION_ID CONTENT" + cmd.Short = `Create conversation message.` + cmd.Long = `Create conversation message. + + Create new message in [conversation](:method:genie/startconversation). The AI + response uses all previously created messages in the conversation to respond. + + Arguments: + SPACE_ID: The ID associated with the Genie space where the conversation is started. + CONVERSATION_ID: The ID associated with the conversation. + CONTENT: User message content.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only SPACE_ID, CONVERSATION_ID as positional arguments. Provide 'content' in your JSON input") + } + return nil + } + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createMessageJson.Unmarshal(&createMessageReq) + if err != nil { + return err + } + } + createMessageReq.SpaceId = args[0] + createMessageReq.ConversationId = args[1] + if !cmd.Flags().Changed("json") { + createMessageReq.Content = args[2] + } + + wait, err := w.Genie.CreateMessage(ctx, createMessageReq) + if err != nil { + return err + } + if createMessageSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *dashboards.GenieMessage) { + status := i.Status + statusMessage := fmt.Sprintf("current status: %s", status) + spinner <- statusMessage + }).GetWithTimeout(createMessageTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createMessageOverrides { + fn(cmd, &createMessageReq) + } + + return cmd +} + +// start execute-message-query command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var executeMessageQueryOverrides []func( + *cobra.Command, + *dashboards.ExecuteMessageQueryRequest, +) + +func newExecuteMessageQuery() *cobra.Command { + cmd := &cobra.Command{} + + var executeMessageQueryReq dashboards.ExecuteMessageQueryRequest + + // TODO: short flags + + cmd.Use = "execute-message-query SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Execute SQL query in a conversation message.` + cmd.Long = `Execute SQL query in a conversation message. + + Execute the SQL query in the message. + + Arguments: + SPACE_ID: Genie space ID + CONVERSATION_ID: Conversation ID + MESSAGE_ID: Message ID` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + executeMessageQueryReq.SpaceId = args[0] + executeMessageQueryReq.ConversationId = args[1] + executeMessageQueryReq.MessageId = args[2] + + response, err := w.Genie.ExecuteMessageQuery(ctx, executeMessageQueryReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range executeMessageQueryOverrides { + fn(cmd, &executeMessageQueryReq) + } + + return cmd +} + +// start get-message command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getMessageOverrides []func( + *cobra.Command, + *dashboards.GenieGetConversationMessageRequest, +) + +func newGetMessage() *cobra.Command { + cmd := &cobra.Command{} + + var getMessageReq dashboards.GenieGetConversationMessageRequest + + // TODO: short flags + + cmd.Use = "get-message SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Get conversation message.` + cmd.Long = `Get conversation message. + + Get message from conversation. + + Arguments: + SPACE_ID: The ID associated with the Genie space where the target conversation is + located. + CONVERSATION_ID: The ID associated with the target conversation. + MESSAGE_ID: The ID associated with the target message from the identified + conversation.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getMessageReq.SpaceId = args[0] + getMessageReq.ConversationId = args[1] + getMessageReq.MessageId = args[2] + + response, err := w.Genie.GetMessage(ctx, getMessageReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getMessageOverrides { + fn(cmd, &getMessageReq) + } + + return cmd +} + +// start get-message-query-result command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getMessageQueryResultOverrides []func( + *cobra.Command, + *dashboards.GenieGetMessageQueryResultRequest, +) + +func newGetMessageQueryResult() *cobra.Command { + cmd := &cobra.Command{} + + var getMessageQueryResultReq dashboards.GenieGetMessageQueryResultRequest + + // TODO: short flags + + cmd.Use = "get-message-query-result SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Get conversation message SQL query result.` + cmd.Long = `Get conversation message SQL query result. + + Get the result of SQL query if the message has a query attachment. This is + only available if a message has a query attachment and the message status is + EXECUTING_QUERY. + + Arguments: + SPACE_ID: Genie space ID + CONVERSATION_ID: Conversation ID + MESSAGE_ID: Message ID` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getMessageQueryResultReq.SpaceId = args[0] + getMessageQueryResultReq.ConversationId = args[1] + getMessageQueryResultReq.MessageId = args[2] + + response, err := w.Genie.GetMessageQueryResult(ctx, getMessageQueryResultReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getMessageQueryResultOverrides { + fn(cmd, &getMessageQueryResultReq) + } + + return cmd +} + +// start start-conversation command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var startConversationOverrides []func( + *cobra.Command, + *dashboards.GenieStartConversationMessageRequest, +) + +func newStartConversation() *cobra.Command { + cmd := &cobra.Command{} + + var startConversationReq dashboards.GenieStartConversationMessageRequest + var startConversationJson flags.JsonFlag + + var startConversationSkipWait bool + var startConversationTimeout time.Duration + + cmd.Flags().BoolVar(&startConversationSkipWait, "no-wait", startConversationSkipWait, `do not wait to reach COMPLETED state`) + cmd.Flags().DurationVar(&startConversationTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach COMPLETED state`) + // TODO: short flags + cmd.Flags().Var(&startConversationJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "start-conversation SPACE_ID CONTENT" + cmd.Short = `Start conversation.` + cmd.Long = `Start conversation. + + Start a new conversation. + + Arguments: + SPACE_ID: The ID associated with the Genie space where you want to start a + conversation. + CONTENT: The text of the message that starts the conversation.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only SPACE_ID as positional arguments. Provide 'content' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = startConversationJson.Unmarshal(&startConversationReq) + if err != nil { + return err + } + } + startConversationReq.SpaceId = args[0] + if !cmd.Flags().Changed("json") { + startConversationReq.Content = args[1] + } + + wait, err := w.Genie.StartConversation(ctx, startConversationReq) + if err != nil { + return err + } + if startConversationSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *dashboards.GenieMessage) { + status := i.Status + statusMessage := fmt.Sprintf("current status: %s", status) + spinner <- statusMessage + }).GetWithTimeout(startConversationTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range startConversationOverrides { + fn(cmd, &startConversationReq) + } + + return cmd +} + +// end service Genie diff --git a/cmd/workspace/groups.go b/cmd/workspace/groups.go index d8a4dec4f..98e474d33 100644 --- a/cmd/workspace/groups.go +++ b/cmd/workspace/groups.go @@ -68,5 +68,9 @@ func Groups() []cobra.Group { ID: "marketplace", Title: "Marketplace", }, + { + ID: "apps", + Title: "Apps", + }, } } diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 50a045921..2d422fa8c 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -817,6 +817,7 @@ func newGetRun() *cobra.Command { cmd.Flags().BoolVar(&getRunReq.IncludeHistory, "include-history", getRunReq.IncludeHistory, `Whether to include the repair history in the response.`) cmd.Flags().BoolVar(&getRunReq.IncludeResolvedValues, "include-resolved-values", getRunReq.IncludeResolvedValues, `Whether to include resolved parameter values in the response.`) + cmd.Flags().StringVar(&getRunReq.PageToken, "page-token", getRunReq.PageToken, `To list the next page or the previous page of job tasks, set this field to the value of the next_page_token or prev_page_token returned in the GetJob response.`) cmd.Use = "get-run RUN_ID" cmd.Short = `Get a single job run.` diff --git a/cmd/workspace/lakeview/lakeview.go b/cmd/workspace/lakeview/lakeview.go index 36eab0e7f..ef2d6845b 100755 --- a/cmd/workspace/lakeview/lakeview.go +++ b/cmd/workspace/lakeview/lakeview.go @@ -666,7 +666,7 @@ func newList() *cobra.Command { cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The number of dashboards to return per page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token, received from a previous ListDashboards call.`) cmd.Flags().BoolVar(&listReq.ShowTrashed, "show-trashed", listReq.ShowTrashed, `The flag to include dashboards located in the trash.`) - cmd.Flags().Var(&listReq.View, "view", `Indicates whether to include all metadata from the dashboard in the response. Supported values: [DASHBOARD_VIEW_BASIC, DASHBOARD_VIEW_FULL]`) + cmd.Flags().Var(&listReq.View, "view", `DASHBOARD_VIEW_BASIConly includes summary metadata from the dashboard. Supported values: [DASHBOARD_VIEW_BASIC]`) cmd.Use = "list" cmd.Short = `List dashboards.` diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go index 034cea2df..d2f054045 100755 --- a/cmd/workspace/model-versions/model-versions.go +++ b/cmd/workspace/model-versions/model-versions.go @@ -133,6 +133,7 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getReq.IncludeAliases, "include-aliases", getReq.IncludeAliases, `Whether to include aliases associated with the model version in the response.`) cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include model versions in the response for which the principal can only access selective metadata for.`) cmd.Use = "get FULL_NAME VERSION" @@ -203,6 +204,8 @@ func newGetByAlias() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getByAliasReq.IncludeAliases, "include-aliases", getByAliasReq.IncludeAliases, `Whether to include aliases associated with the model version in the response.`) + cmd.Use = "get-by-alias FULL_NAME ALIAS" cmd.Short = `Get Model Version By Alias.` cmd.Long = `Get Model Version By Alias. diff --git a/cmd/workspace/notification-destinations/notification-destinations.go b/cmd/workspace/notification-destinations/notification-destinations.go new file mode 100755 index 000000000..5ad47cc95 --- /dev/null +++ b/cmd/workspace/notification-destinations/notification-destinations.go @@ -0,0 +1,342 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package notification_destinations + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "notification-destinations", + Short: `The notification destinations API lets you programmatically manage a workspace's notification destinations.`, + Long: `The notification destinations API lets you programmatically manage a + workspace's notification destinations. Notification destinations are used to + send notifications for query alerts and jobs to destinations outside of + Databricks. Only workspace admins can create, update, and delete notification + destinations.`, + GroupID: "settings", + Annotations: map[string]string{ + "package": "settings", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *settings.CreateNotificationDestinationRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq settings.CreateNotificationDestinationRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: config + cmd.Flags().StringVar(&createReq.DisplayName, "display-name", createReq.DisplayName, `The display name for the notification destination.`) + + cmd.Use = "create" + cmd.Short = `Create a notification destination.` + cmd.Long = `Create a notification destination. + + Creates a notification destination. Requires workspace admin permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } + + response, err := w.NotificationDestinations.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteNotificationDestinationRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteNotificationDestinationRequest + + // TODO: short flags + + cmd.Use = "delete ID" + cmd.Short = `Delete a notification destination.` + cmd.Long = `Delete a notification destination. + + Deletes a notification destination. Requires workspace admin permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + deleteReq.Id = args[0] + + err = w.NotificationDestinations.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetNotificationDestinationRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetNotificationDestinationRequest + + // TODO: short flags + + cmd.Use = "get ID" + cmd.Short = `Get a notification destination.` + cmd.Long = `Get a notification destination. + + Gets a notification destination.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getReq.Id = args[0] + + response, err := w.NotificationDestinations.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, + *settings.ListNotificationDestinationsRequest, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + var listReq settings.ListNotificationDestinationsRequest + + // TODO: short flags + + cmd.Flags().Int64Var(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + + cmd.Use = "list" + cmd.Short = `List notification destinations.` + cmd.Long = `List notification destinations. + + Lists notification destinations.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response := w.NotificationDestinations.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd, &listReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateNotificationDestinationRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateNotificationDestinationRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: config + cmd.Flags().StringVar(&updateReq.DisplayName, "display-name", updateReq.DisplayName, `The display name for the notification destination.`) + + cmd.Use = "update ID" + cmd.Short = `Update a notification destination.` + cmd.Long = `Update a notification destination. + + Updates a notification destination. Requires workspace admin permissions. At + least one field is required in the request body.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + updateReq.Id = args[0] + + response, err := w.NotificationDestinations.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service NotificationDestinations diff --git a/cmd/workspace/permission-migration/permission-migration.go b/cmd/workspace/permission-migration/permission-migration.go index 40d3f9a3b..2e50b1231 100755 --- a/cmd/workspace/permission-migration/permission-migration.go +++ b/cmd/workspace/permission-migration/permission-migration.go @@ -19,9 +19,9 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "permission-migration", - Short: `This spec contains undocumented permission migration APIs used in https://github.com/databrickslabs/ucx.`, - Long: `This spec contains undocumented permission migration APIs used in - https://github.com/databrickslabs/ucx.`, + Short: `APIs for migrating acl permissions, used only by the ucx tool: https://github.com/databrickslabs/ucx.`, + Long: `APIs for migrating acl permissions, used only by the ucx tool: + https://github.com/databrickslabs/ucx`, GroupID: "iam", Annotations: map[string]string{ "package": "iam", @@ -48,13 +48,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var migratePermissionsOverrides []func( *cobra.Command, - *iam.PermissionMigrationRequest, + *iam.MigratePermissionsRequest, ) func newMigratePermissions() *cobra.Command { cmd := &cobra.Command{} - var migratePermissionsReq iam.PermissionMigrationRequest + var migratePermissionsReq iam.MigratePermissionsRequest var migratePermissionsJson flags.JsonFlag // TODO: short flags @@ -65,14 +65,10 @@ func newMigratePermissions() *cobra.Command { cmd.Use = "migrate-permissions WORKSPACE_ID FROM_WORKSPACE_GROUP_NAME TO_ACCOUNT_GROUP_NAME" cmd.Short = `Migrate Permissions.` cmd.Long = `Migrate Permissions. - - Migrate a batch of permissions from a workspace local group to an account - group. Arguments: WORKSPACE_ID: WorkspaceId of the associated workspace where the permission migration - will occur. Both workspace group and account group must be in this - workspace. + will occur. FROM_WORKSPACE_GROUP_NAME: The name of the workspace group that permissions will be migrated from. TO_ACCOUNT_GROUP_NAME: The name of the account group that permissions will be migrated to.` diff --git a/cmd/workspace/permissions/permissions.go b/cmd/workspace/permissions/permissions.go index 57a7d1e5e..fd9c1a468 100755 --- a/cmd/workspace/permissions/permissions.go +++ b/cmd/workspace/permissions/permissions.go @@ -21,6 +21,9 @@ func New() *cobra.Command { Long: `Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints. + * **[Apps permissions](:service:apps)** — Manage which users can manage or + use apps. + * **[Cluster permissions](:service:clusters)** — Manage which users can manage, restart, or attach to clusters. @@ -59,7 +62,8 @@ func New() *cobra.Command { create or use tokens. * **[Workspace object permissions](:service:workspace)** — Manage which - users can read, run, edit, or manage directories, files, and notebooks. + users can read, run, edit, or manage alerts, dbsql-dashboards, directories, + files, notebooks and queries. For the mapping of the required permissions for specific actions or abilities and other important information, see [Access Control]. @@ -112,10 +116,10 @@ func newGet() *cobra.Command { parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -240,10 +244,10 @@ func newSet() *cobra.Command { parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -314,10 +318,10 @@ func newUpdate() *cobra.Command { their parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/policy-families/policy-families.go b/cmd/workspace/policy-families/policy-families.go index beee6e963..cac23405b 100755 --- a/cmd/workspace/policy-families/policy-families.go +++ b/cmd/workspace/policy-families/policy-families.go @@ -60,11 +60,17 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().Int64Var(&getReq.Version, "version", getReq.Version, `The version number for the family to fetch.`) + cmd.Use = "get POLICY_FAMILY_ID" cmd.Short = `Get policy family information.` cmd.Long = `Get policy family information. - Retrieve the information for an policy family based on its identifier.` + Retrieve the information for an policy family based on its identifier and + version + + Arguments: + POLICY_FAMILY_ID: The family ID about which to retrieve information.` cmd.Annotations = make(map[string]string) @@ -115,14 +121,15 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().Int64Var(&listReq.MaxResults, "max-results", listReq.MaxResults, `The max number of policy families to return.`) + cmd.Flags().Int64Var(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of policy families to return.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) cmd.Use = "list" cmd.Short = `List policy families.` cmd.Long = `List policy families. - Retrieve a list of policy families. This API is paginated.` + Returns the list of policy definition types available to use at their latest + version. This API is paginated.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go index 4ab36b5d0..a3f746214 100755 --- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go +++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go index 7ff73e0d1..b92403755 100755 --- a/cmd/workspace/provider-exchanges/provider-exchanges.go +++ b/cmd/workspace/provider-exchanges/provider-exchanges.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go index 25e1addf5..62dcb6de9 100755 --- a/cmd/workspace/provider-files/provider-files.go +++ b/cmd/workspace/provider-files/provider-files.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go index 0abdf51d8..18c99c53d 100755 --- a/cmd/workspace/provider-listings/provider-listings.go +++ b/cmd/workspace/provider-listings/provider-listings.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go index a38d9f420..d18e2e578 100755 --- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go +++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go index 8cee6e4eb..bb3ca9666 100755 --- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go +++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go @@ -23,9 +23,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go index b7273a344..94d12d6f0 100755 --- a/cmd/workspace/provider-providers/provider-providers.go +++ b/cmd/workspace/provider-providers/provider-providers.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index 7305191c8..af2737a0f 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -291,6 +291,8 @@ func newList() *cobra.Command { // TODO: short flags cmd.Flags().StringVar(&listReq.DataProviderGlobalMetastoreId, "data-provider-global-metastore-id", listReq.DataProviderGlobalMetastoreId, `If not provided, all providers will be returned.`) + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of providers to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "list" cmd.Short = `List providers.` @@ -345,6 +347,9 @@ func newListShares() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&listSharesReq.MaxResults, "max-results", listSharesReq.MaxResults, `Maximum number of shares to return.`) + cmd.Flags().StringVar(&listSharesReq.PageToken, "page-token", listSharesReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list-shares NAME" cmd.Short = `List shares by Provider.` cmd.Long = `List shares by Provider. diff --git a/cmd/workspace/queries-legacy/queries-legacy.go b/cmd/workspace/queries-legacy/queries-legacy.go new file mode 100755 index 000000000..fa78bb2b0 --- /dev/null +++ b/cmd/workspace/queries-legacy/queries-legacy.go @@ -0,0 +1,500 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package queries_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "queries-legacy", + Short: `These endpoints are used for CRUD operations on query definitions.`, + Long: `These endpoints are used for CRUD operations on query definitions. Query + definitions include the target SQL warehouse, query text, name, description, + tags, parameters, and visualizations. Queries can be scheduled using the + sql_task type of the Jobs API, e.g. :method:jobs/create. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newRestore()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.QueryPostContent, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.QueryPostContent + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create" + cmd.Short = `Create a new query definition.` + cmd.Long = `Create a new query definition. + + Creates a new query definition. Queries created with this endpoint belong to + the authenticated user making the request. + + The data_source_id field specifies the ID of the SQL warehouse to run this + query against. You can use the Data Sources API to see a complete list of + available SQL warehouses. Or you can copy the data_source_id from an + existing query. + + **Note**: You cannot add a visualization until you create the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.QueriesLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteQueriesLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "delete QUERY_ID" + cmd.Short = `Delete a query.` + cmd.Long = `Delete a query. + + Moves a query to the trash. Trashed queries immediately disappear from + searches and list views, and they cannot be used for alerts. The trash is + deleted after 30 days. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + deleteReq.QueryId = args[0] + + err = w.QueriesLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *sql.GetQueriesLegacyRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq sql.GetQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "get QUERY_ID" + cmd.Short = `Get a query definition.` + cmd.Long = `Get a query definition. + + Retrieve a query object definition along with contextual permissions + information about the currently authenticated user. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/get instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + getReq.QueryId = args[0] + + response, err := w.QueriesLegacy.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, + *sql.ListQueriesLegacyRequest, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + var listReq sql.ListQueriesLegacyRequest + + // TODO: short flags + + cmd.Flags().StringVar(&listReq.Order, "order", listReq.Order, `Name of query attribute to order by.`) + cmd.Flags().IntVar(&listReq.Page, "page", listReq.Page, `Page number to retrieve.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Number of queries to return per page.`) + cmd.Flags().StringVar(&listReq.Q, "q", listReq.Q, `Full text search term.`) + + cmd.Use = "list" + cmd.Short = `Get a list of queries.` + cmd.Long = `Get a list of queries. + + Gets a list of queries. Optionally, this list can be filtered by a search + term. + + **Warning**: Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/list instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response := w.QueriesLegacy.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd, &listReq) + } + + return cmd +} + +// start restore command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var restoreOverrides []func( + *cobra.Command, + *sql.RestoreQueriesLegacyRequest, +) + +func newRestore() *cobra.Command { + cmd := &cobra.Command{} + + var restoreReq sql.RestoreQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "restore QUERY_ID" + cmd.Short = `Restore a query.` + cmd.Long = `Restore a query. + + Restore a query that has been moved to the trash. A restored query appears in + list views and searches. You can use restored queries for alerts. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + restoreReq.QueryId = args[0] + + err = w.QueriesLegacy.Restore(ctx, restoreReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range restoreOverrides { + fn(cmd, &restoreReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.QueryEditContent, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.QueryEditContent + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&updateReq.DataSourceId, "data-source-id", updateReq.DataSourceId, `Data source ID maps to the ID of the data source used by the resource and is distinct from the warehouse ID.`) + cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `General description that conveys additional information about this query such as usage notes.`) + cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this query that appears in list views, widget headings, and on the query page.`) + // TODO: any: options + cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`) + cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) + // TODO: array: tags + + cmd.Use = "update QUERY_ID" + cmd.Short = `Change a query definition.` + cmd.Long = `Change a query definition. + + Modify this query definition. + + **Note**: You cannot undo this operation. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + updateReq.QueryId = args[0] + + response, err := w.QueriesLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service QueriesLegacy diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index 650131974..fea01451a 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -19,16 +19,11 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "queries", - Short: `These endpoints are used for CRUD operations on query definitions.`, - Long: `These endpoints are used for CRUD operations on query definitions. Query - definitions include the target SQL warehouse, query text, name, description, - tags, parameters, and visualizations. Queries can be scheduled using the - sql_task type of the Jobs API, e.g. :method:jobs/create. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + Short: `The queries API can be used to perform CRUD operations on queries.`, + Long: `The queries API can be used to perform CRUD operations on queries. A query is + a Databricks SQL object that includes the target SQL warehouse, query text, + name, description, tags, and parameters. Queries can be scheduled using the + sql_task type of the Jobs API, e.g. :method:jobs/create.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -40,7 +35,7 @@ func New() *cobra.Command { cmd.AddCommand(newDelete()) cmd.AddCommand(newGet()) cmd.AddCommand(newList()) - cmd.AddCommand(newRestore()) + cmd.AddCommand(newListVisualizations()) cmd.AddCommand(newUpdate()) // Apply optional overrides to this command. @@ -57,39 +52,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.QueryPostContent, + *sql.CreateQueryRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.QueryPostContent + var createReq sql.CreateQueryRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: query + cmd.Use = "create" - cmd.Short = `Create a new query definition.` - cmd.Long = `Create a new query definition. + cmd.Short = `Create a query.` + cmd.Long = `Create a query. - Creates a new query definition. Queries created with this endpoint belong to - the authenticated user making the request. - - The data_source_id field specifies the ID of the SQL warehouse to run this - query against. You can use the Data Sources API to see a complete list of - available SQL warehouses. Or you can copy the data_source_id from an - existing query. - - **Note**: You cannot add a visualization until you create the query. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Creates a query.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -100,8 +89,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.Queries.Create(ctx, createReq) @@ -129,28 +116,24 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteQueryRequest, + *sql.TrashQueryRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteQueryRequest + var deleteReq sql.TrashQueryRequest // TODO: short flags - cmd.Use = "delete QUERY_ID" + cmd.Use = "delete ID" cmd.Short = `Delete a query.` cmd.Long = `Delete a query. Moves a query to the trash. Trashed queries immediately disappear from - searches and list views, and they cannot be used for alerts. The trash is - deleted after 30 days. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + searches and list views, and cannot be used for alerts. You can restore a + trashed query through the UI. A trashed query is permanently deleted after 30 + days.` cmd.Annotations = make(map[string]string) @@ -161,8 +144,8 @@ func newDelete() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -176,7 +159,7 @@ func newDelete() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - deleteReq.QueryId = args[0] + deleteReq.Id = args[0] err = w.Queries.Delete(ctx, deleteReq) if err != nil { @@ -213,17 +196,11 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get QUERY_ID" - cmd.Short = `Get a query definition.` - cmd.Long = `Get a query definition. + cmd.Use = "get ID" + cmd.Short = `Get a query.` + cmd.Long = `Get a query. - Retrieve a query object definition along with contextual permissions - information about the currently authenticated user. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a query.` cmd.Annotations = make(map[string]string) @@ -234,8 +211,8 @@ func newGet() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -249,7 +226,7 @@ func newGet() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - getReq.QueryId = args[0] + getReq.Id = args[0] response, err := w.Queries.Get(ctx, getReq) if err != nil { @@ -286,25 +263,16 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().StringVar(&listReq.Order, "order", listReq.Order, `Name of query attribute to order by.`) - cmd.Flags().IntVar(&listReq.Page, "page", listReq.Page, `Page number to retrieve.`) - cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Number of queries to return per page.`) - cmd.Flags().StringVar(&listReq.Q, "q", listReq.Q, `Full text search term.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) cmd.Use = "list" - cmd.Short = `Get a list of queries.` - cmd.Long = `Get a list of queries. + cmd.Short = `List queries.` + cmd.Long = `List queries. - Gets a list of queries. Optionally, this list can be filtered by a search - term. - - **Warning**: Calling this API concurrently 10 or more times could result in - throttling, service degradation, or a temporary ban. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of queries accessible to the user, ordered by creation time. + **Warning:** Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban.` cmd.Annotations = make(map[string]string) @@ -334,33 +302,33 @@ func newList() *cobra.Command { return cmd } -// start restore command +// start list-visualizations command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var restoreOverrides []func( +var listVisualizationsOverrides []func( *cobra.Command, - *sql.RestoreQueryRequest, + *sql.ListVisualizationsForQueryRequest, ) -func newRestore() *cobra.Command { +func newListVisualizations() *cobra.Command { cmd := &cobra.Command{} - var restoreReq sql.RestoreQueryRequest + var listVisualizationsReq sql.ListVisualizationsForQueryRequest // TODO: short flags - cmd.Use = "restore QUERY_ID" - cmd.Short = `Restore a query.` - cmd.Long = `Restore a query. + cmd.Flags().IntVar(&listVisualizationsReq.PageSize, "page-size", listVisualizationsReq.PageSize, ``) + cmd.Flags().StringVar(&listVisualizationsReq.PageToken, "page-token", listVisualizationsReq.PageToken, ``) + + cmd.Use = "list-visualizations ID" + cmd.Short = `List visualizations on a query.` + cmd.Long = `List visualizations on a query. - Restore a query that has been moved to the trash. A restored query appears in - list views and searches. You can use restored queries for alerts. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of visualizations on a query.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true cmd.Annotations = make(map[string]string) @@ -371,8 +339,8 @@ func newRestore() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -386,13 +354,10 @@ func newRestore() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - restoreReq.QueryId = args[0] + listVisualizationsReq.Id = args[0] - err = w.Queries.Restore(ctx, restoreReq) - if err != nil { - return err - } - return nil + response := w.Queries.ListVisualizations(ctx, listVisualizationsReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -400,8 +365,8 @@ func newRestore() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range restoreOverrides { - fn(cmd, &restoreReq) + for _, fn := range listVisualizationsOverrides { + fn(cmd, &listVisualizationsReq) } return cmd @@ -413,41 +378,47 @@ func newRestore() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.QueryEditContent, + *sql.UpdateQueryRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.QueryEditContent + var updateReq sql.UpdateQueryRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateReq.DataSourceId, "data-source-id", updateReq.DataSourceId, `Data source ID maps to the ID of the data source used by the resource and is distinct from the warehouse ID.`) - cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `General description that conveys additional information about this query such as usage notes.`) - cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this query that appears in list views, widget headings, and on the query page.`) - // TODO: any: options - cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`) - cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) - // TODO: array: tags + // TODO: complex arg: query - cmd.Use = "update QUERY_ID" - cmd.Short = `Change a query definition.` - cmd.Long = `Change a query definition. + cmd.Use = "update ID UPDATE_MASK" + cmd.Short = `Update a query.` + cmd.Long = `Update a query. - Modify this query definition. - - **Note**: You cannot undo this operation. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Updates a query. + + Arguments: + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -459,24 +430,10 @@ func newUpdate() *cobra.Command { return err } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "") - if err != nil { - return err - } - args = append(args, id) + updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] } - if len(args) != 1 { - return fmt.Errorf("expected to have ") - } - updateReq.QueryId = args[0] response, err := w.Queries.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/query-history/query-history.go b/cmd/workspace/query-history/query-history.go index 60d6004d9..5155b5cc0 100755 --- a/cmd/workspace/query-history/query-history.go +++ b/cmd/workspace/query-history/query-history.go @@ -15,9 +15,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "query-history", - Short: `Access the history of queries through SQL warehouses.`, - Long: `Access the history of queries through SQL warehouses.`, + Use: "query-history", + Short: `A service responsible for storing and retrieving the list of queries run against SQL endpoints, serverless compute, and DLT.`, + Long: `A service responsible for storing and retrieving the list of queries run + against SQL endpoints, serverless compute, and DLT.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -52,7 +53,6 @@ func newList() *cobra.Command { // TODO: short flags // TODO: complex arg: filter_by - cmd.Flags().BoolVar(&listReq.IncludeMetrics, "include-metrics", listReq.IncludeMetrics, `Whether to include metrics about query.`) cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Limit the number of results returned in one page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) @@ -60,9 +60,13 @@ func newList() *cobra.Command { cmd.Short = `List Queries.` cmd.Long = `List Queries. - List the history of queries through SQL warehouses. + List the history of queries through SQL warehouses, serverless compute, and + DLT. - You can filter by user ID, warehouse ID, status, and time range.` + You can filter by user ID, warehouse ID, status, and time range. Most recently + started queries are returned first (up to max_results in request). The + pagination token returned in response can be used to list subsequent query + statuses.` cmd.Annotations = make(map[string]string) @@ -76,8 +80,11 @@ func newList() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response := w.QueryHistory.List(ctx, listReq) - return cmdio.RenderIterator(ctx, response) + response, err := w.QueryHistory.List(ctx, listReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go new file mode 100755 index 000000000..4f45ab23e --- /dev/null +++ b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go @@ -0,0 +1,253 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package query_visualizations_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "query-visualizations-legacy", + Short: `This is an evolving API that facilitates the addition and removal of vizualisations from existing queries within the Databricks Workspace.`, + Long: `This is an evolving API that facilitates the addition and removal of + vizualisations from existing queries within the Databricks Workspace. Data + structures may change over time. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.CreateQueryVisualizationsLegacyRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.CreateQueryVisualizationsLegacyRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create" + cmd.Short = `Add visualization to a query.` + cmd.Long = `Add visualization to a query. + + Creates visualization in the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.QueryVisualizationsLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteQueryVisualizationsLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteQueryVisualizationsLegacyRequest + + // TODO: short flags + + cmd.Use = "delete ID" + cmd.Short = `Remove visualization.` + cmd.Long = `Remove visualization. + + Removes a visualization from the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + + Arguments: + ID: Widget ID returned by :method:queryvizualisations/create` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + deleteReq.Id = args[0] + + err = w.QueryVisualizationsLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.LegacyVisualization, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.LegacyVisualization + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update ID" + cmd.Short = `Edit existing visualization.` + cmd.Long = `Edit existing visualization. + + Updates visualization in the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + + Arguments: + ID: The UUID for this visualization.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + updateReq.Id = args[0] + + response, err := w.QueryVisualizationsLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service QueryVisualizationsLegacy diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go index c94d83a82..042594529 100755 --- a/cmd/workspace/query-visualizations/query-visualizations.go +++ b/cmd/workspace/query-visualizations/query-visualizations.go @@ -19,10 +19,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "query-visualizations", - Short: `This is an evolving API that facilitates the addition and removal of vizualisations from existing queries within the Databricks Workspace.`, + Short: `This is an evolving API that facilitates the addition and removal of visualizations from existing queries in the Databricks Workspace.`, Long: `This is an evolving API that facilitates the addition and removal of - vizualisations from existing queries within the Databricks Workspace. Data - structures may change over time.`, + visualizations from existing queries in the Databricks Workspace. Data + structures can change over time.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -51,24 +51,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.CreateQueryVisualizationRequest, + *sql.CreateVisualizationRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.CreateQueryVisualizationRequest + var createReq sql.CreateVisualizationRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: visualization + cmd.Use = "create" - cmd.Short = `Add visualization to a query.` - cmd.Long = `Add visualization to a query.` + cmd.Short = `Add a visualization to a query.` + cmd.Long = `Add a visualization to a query. + + Adds a visualization to a query.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -79,8 +88,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.QueryVisualizations.Create(ctx, createReq) @@ -108,22 +115,21 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteQueryVisualizationRequest, + *sql.DeleteVisualizationRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteQueryVisualizationRequest + var deleteReq sql.DeleteVisualizationRequest // TODO: short flags cmd.Use = "delete ID" - cmd.Short = `Remove visualization.` - cmd.Long = `Remove visualization. - - Arguments: - ID: Widget ID returned by :method:queryvizualisations/create` + cmd.Short = `Remove a visualization.` + cmd.Long = `Remove a visualization. + + Removes a visualization.` cmd.Annotations = make(map[string]string) @@ -164,29 +170,44 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.Visualization, + *sql.UpdateVisualizationRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.Visualization + var updateReq sql.UpdateVisualizationRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Use = "update ID" - cmd.Short = `Edit existing visualization.` - cmd.Long = `Edit existing visualization. + // TODO: complex arg: visualization + + cmd.Use = "update ID UPDATE_MASK" + cmd.Short = `Update a visualization.` + cmd.Long = `Update a visualization. + + Updates a visualization. Arguments: - ID: The UUID for this visualization.` + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) return check(cmd, args) } @@ -200,10 +221,11 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } response, err := w.QueryVisualizations.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index c21d8a8c0..f4472cf37 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -80,6 +80,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the recipient.`) cmd.Flags().StringVar(&createReq.DataRecipientGlobalMetastoreId, "data-recipient-global-metastore-id", createReq.DataRecipientGlobalMetastoreId, `The global Unity Catalog metastore id provided by the data recipient.`) + cmd.Flags().Int64Var(&createReq.ExpirationTime, "expiration-time", createReq.ExpirationTime, `Expiration timestamp of the token, in epoch milliseconds.`) // TODO: complex arg: ip_access_list cmd.Flags().StringVar(&createReq.Owner, "owner", createReq.Owner, `Username of the recipient owner.`) // TODO: complex arg: properties_kvpairs @@ -311,6 +312,8 @@ func newList() *cobra.Command { // TODO: short flags cmd.Flags().StringVar(&listReq.DataRecipientGlobalMetastoreId, "data-recipient-global-metastore-id", listReq.DataRecipientGlobalMetastoreId, `If not provided, all recipients will be returned.`) + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of recipients to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "list" cmd.Short = `List share recipients.` @@ -449,6 +452,9 @@ func newSharePermissions() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&sharePermissionsReq.MaxResults, "max-results", sharePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&sharePermissionsReq.PageToken, "page-token", sharePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "share-permissions NAME" cmd.Short = `Get recipient share permissions.` cmd.Long = `Get recipient share permissions. @@ -523,6 +529,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the recipient.`) + cmd.Flags().Int64Var(&updateReq.ExpirationTime, "expiration-time", updateReq.ExpirationTime, `Expiration timestamp of the token, in epoch milliseconds.`) // TODO: complex arg: ip_access_list cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the recipient.`) cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of the recipient owner.`) diff --git a/cmd/workspace/registered-models/registered-models.go b/cmd/workspace/registered-models/registered-models.go index 08e11d686..5aa6cdf15 100755 --- a/cmd/workspace/registered-models/registered-models.go +++ b/cmd/workspace/registered-models/registered-models.go @@ -326,6 +326,7 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getReq.IncludeAliases, "include-aliases", getReq.IncludeAliases, `Whether to include registered model aliases in the response.`) cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include registered models in the response for which the principal can only access selective metadata for.`) cmd.Use = "get FULL_NAME" diff --git a/cmd/workspace/schemas/schemas.go b/cmd/workspace/schemas/schemas.go index 710141913..3a398251f 100755 --- a/cmd/workspace/schemas/schemas.go +++ b/cmd/workspace/schemas/schemas.go @@ -147,6 +147,8 @@ func newDelete() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&deleteReq.Force, "force", deleteReq.Force, `Force deletion even if the schema is not empty.`) + cmd.Use = "delete FULL_NAME" cmd.Short = `Delete a schema.` cmd.Long = `Delete a schema. diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index c2fd779a7..67f870177 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -254,11 +254,19 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *sharing.ListSharesRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq sharing.ListSharesRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of shares to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list" cmd.Short = `List shares.` cmd.Long = `List shares. @@ -269,11 +277,17 @@ func newList() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response := w.Shares.List(ctx) + + response := w.Shares.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -283,7 +297,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -305,6 +319,9 @@ func newSharePermissions() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&sharePermissionsReq.MaxResults, "max-results", sharePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&sharePermissionsReq.PageToken, "page-token", sharePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "share-permissions NAME" cmd.Short = `Get permissions.` cmd.Long = `Get permissions. @@ -455,6 +472,8 @@ func newUpdatePermissions() *cobra.Command { cmd.Flags().Var(&updatePermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: changes + cmd.Flags().IntVar(&updatePermissionsReq.MaxResults, "max-results", updatePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&updatePermissionsReq.PageToken, "page-token", updatePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "update-permissions NAME" cmd.Short = `Update permissions.` diff --git a/cmd/workspace/system-schemas/system-schemas.go b/cmd/workspace/system-schemas/system-schemas.go index 3fe0580d7..292afbe84 100755 --- a/cmd/workspace/system-schemas/system-schemas.go +++ b/cmd/workspace/system-schemas/system-schemas.go @@ -177,6 +177,9 @@ func newList() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of schemas to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list METASTORE_ID" cmd.Short = `List system schemas.` cmd.Long = `List system schemas. diff --git a/cmd/workspace/workspace-bindings/workspace-bindings.go b/cmd/workspace/workspace-bindings/workspace-bindings.go index b7e0614ea..4993f1aff 100755 --- a/cmd/workspace/workspace-bindings/workspace-bindings.go +++ b/cmd/workspace/workspace-bindings/workspace-bindings.go @@ -3,6 +3,8 @@ package workspace_bindings import ( + "fmt" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" @@ -35,7 +37,8 @@ func New() *cobra.Command { (/api/2.1/unity-catalog/bindings/{securable_type}/{securable_name}) which introduces the ability to bind a securable in READ_ONLY mode (catalogs only). - Securables that support binding: - catalog`, + Securable types that support binding: - catalog - storage_credential - + external_location`, GroupID: "catalog", Annotations: map[string]string{ "package": "catalog", @@ -131,6 +134,9 @@ func newGetBindings() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&getBindingsReq.MaxResults, "max-results", getBindingsReq.MaxResults, `Maximum number of workspace bindings to return.`) + cmd.Flags().StringVar(&getBindingsReq.PageToken, "page-token", getBindingsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "get-bindings SECURABLE_TYPE SECURABLE_NAME" cmd.Short = `Get securable workspace bindings.` cmd.Long = `Get securable workspace bindings. @@ -139,7 +145,7 @@ func newGetBindings() *cobra.Command { or an owner of the securable. Arguments: - SECURABLE_TYPE: The type of the securable. + SECURABLE_TYPE: The type of the securable to bind to a workspace. SECURABLE_NAME: The name of the securable.` cmd.Annotations = make(map[string]string) @@ -154,14 +160,14 @@ func newGetBindings() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getBindingsReq.SecurableType = args[0] + _, err = fmt.Sscan(args[0], &getBindingsReq.SecurableType) + if err != nil { + return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) + } getBindingsReq.SecurableName = args[1] - response, err := w.WorkspaceBindings.GetBindings(ctx, getBindingsReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + response := w.WorkspaceBindings.GetBindings(ctx, getBindingsReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -275,7 +281,7 @@ func newUpdateBindings() *cobra.Command { admin or an owner of the securable. Arguments: - SECURABLE_TYPE: The type of the securable. + SECURABLE_TYPE: The type of the securable to bind to a workspace. SECURABLE_NAME: The name of the securable.` cmd.Annotations = make(map[string]string) @@ -296,7 +302,10 @@ func newUpdateBindings() *cobra.Command { return err } } - updateBindingsReq.SecurableType = args[0] + _, err = fmt.Sscan(args[0], &updateBindingsReq.SecurableType) + if err != nil { + return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) + } updateBindingsReq.SecurableName = args[1] response, err := w.WorkspaceBindings.UpdateBindings(ctx, updateBindingsReq) diff --git a/go.mod b/go.mod index 3f5af0815..1457a4d67 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.43.2 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.44.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause @@ -60,13 +60,13 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index f33a9562a..b2985955c 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.43.2 h1:4B+sHAYO5kFqwZNQRmsF70eecqsFX6i/0KfXoDFQT/E= -github.com/databricks/databricks-sdk-go v0.43.2/go.mod h1:nlzeOEgJ1Tmb5HyknBJ3GEorCZKWqEBoHprvPmTSNq8= +github.com/databricks/databricks-sdk-go v0.44.0 h1:9/FZACv4EFQIOYxfwYVKnY7v46xio9FKCw9tpKB2O/s= +github.com/databricks/databricks-sdk-go v0.44.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -176,8 +176,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= @@ -192,8 +192,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -244,8 +244,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/libs/databrickscfg/cfgpickers/clusters.go b/libs/databrickscfg/cfgpickers/clusters.go index d955be35b..cac1b08a7 100644 --- a/libs/databrickscfg/cfgpickers/clusters.go +++ b/libs/databrickscfg/cfgpickers/clusters.go @@ -134,9 +134,7 @@ func loadInteractiveClusters(ctx context.Context, w *databricks.WorkspaceClient, promptSpinner := cmdio.Spinner(ctx) promptSpinner <- "Loading list of clusters to select from" defer close(promptSpinner) - all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{ - CanUseClient: "NOTEBOOKS", - }) + all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{}) if err != nil { return nil, fmt.Errorf("list clusters: %w", err) } diff --git a/libs/databrickscfg/cfgpickers/clusters_test.go b/libs/databrickscfg/cfgpickers/clusters_test.go index 2e62f93a8..d17e86d4a 100644 --- a/libs/databrickscfg/cfgpickers/clusters_test.go +++ b/libs/databrickscfg/cfgpickers/clusters_test.go @@ -70,7 +70,7 @@ func TestFirstCompatibleCluster(t *testing.T) { cfg, server := qa.HTTPFixtures{ { Method: "GET", - Resource: "/api/2.0/clusters/list?can_use_client=NOTEBOOKS", + Resource: "/api/2.1/clusters/list?", Response: compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -100,7 +100,7 @@ func TestFirstCompatibleCluster(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/clusters/spark-versions", + Resource: "/api/2.1/clusters/spark-versions", Response: compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { @@ -125,7 +125,7 @@ func TestNoCompatibleClusters(t *testing.T) { cfg, server := qa.HTTPFixtures{ { Method: "GET", - Resource: "/api/2.0/clusters/list?can_use_client=NOTEBOOKS", + Resource: "/api/2.1/clusters/list?", Response: compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -147,7 +147,7 @@ func TestNoCompatibleClusters(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/clusters/spark-versions", + Resource: "/api/2.1/clusters/spark-versions", Response: compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { From 7c5b650111b176ab61bce1241d514ff641815218 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 16 Aug 2024 10:32:38 +0200 Subject: [PATCH 68/88] Fix integration tests after Go SDK bump (#1686) ## Changes These 2 tests failed `TestAccAlertsCreateErrWhenNoArguments ` -> switched to legacy command for now, new one does not have a required request body (might be an OpenAPI spec issue https://github.com/databricks/databricks-sdk-go/blob/main/service/sql/model.go#L595), will follow up later `TestAccClustersList` -> increased channel size because new clusters API returns more clusters ## Tests Tests are green now --- internal/alerts_test.go | 2 +- internal/helpers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/alerts_test.go b/internal/alerts_test.go index f34b404de..6d7544074 100644 --- a/internal/alerts_test.go +++ b/internal/alerts_test.go @@ -9,6 +9,6 @@ import ( func TestAccAlertsCreateErrWhenNoArguments(t *testing.T) { t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) - _, _, err := RequireErrorRun(t, "alerts", "create") + _, _, err := RequireErrorRun(t, "alerts-legacy", "create") assert.Equal(t, "please provide command input in JSON format by specifying the --json flag", err.Error()) } diff --git a/internal/helpers.go b/internal/helpers.go index 5d9aead1f..269030183 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -87,7 +87,7 @@ type cobraTestRunner struct { } func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan string { - ch := make(chan string, 1000) + ch := make(chan string, 10000) wg.Add(1) go func() { defer close(ch) From f99335e87145f1b18d9b8bdd6e376fb365be2c13 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 19 Aug 2024 12:00:21 +0200 Subject: [PATCH 69/88] Increased chan size for clusters test to pass (#1691) ## Changes Increased chan size for clusters test to pass --- internal/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/helpers.go b/internal/helpers.go index 269030183..419fa419c 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -87,7 +87,7 @@ type cobraTestRunner struct { } func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan string { - ch := make(chan string, 10000) + ch := make(chan string, 30000) wg.Add(1) go func() { defer close(ch) From beced9f1b5ec5bde8665f0a63c223829655a33f5 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 19 Aug 2024 13:27:05 +0200 Subject: [PATCH 70/88] [Release] Release v0.226.0 (#1683) CLI: * Add command line autocomplete to the fs commands ([#1622](https://github.com/databricks/cli/pull/1622)). * Add trailing slash to directory to produce completions for ([#1666](https://github.com/databricks/cli/pull/1666)). * Fix ability to import the CLI repository as module ([#1671](https://github.com/databricks/cli/pull/1671)). * Fix host resolution order in `auth login` ([#1370](https://github.com/databricks/cli/pull/1370)). * Print text logs in `import-dir` and `export-dir` commands ([#1682](https://github.com/databricks/cli/pull/1682)). Bundles: * Expand and upload local wheel libraries for all task types ([#1649](https://github.com/databricks/cli/pull/1649)). * Clarify file format required for the `config-file` flag in `bundle init` ([#1651](https://github.com/databricks/cli/pull/1651)). * Fixed incorrectly cleaning up python wheel dist folder ([#1656](https://github.com/databricks/cli/pull/1656)). * Merge job parameters based on their name ([#1659](https://github.com/databricks/cli/pull/1659)). * Fix glob expansion after running a generic build command ([#1662](https://github.com/databricks/cli/pull/1662)). * Upload local libraries even if they don't have artifact defined ([#1664](https://github.com/databricks/cli/pull/1664)). Internal: * Fix python wheel task integration tests ([#1648](https://github.com/databricks/cli/pull/1648)). * Skip pushing Terraform state after destroy ([#1667](https://github.com/databricks/cli/pull/1667)). * Enable Spark JAR task test ([#1658](https://github.com/databricks/cli/pull/1658)). * Run Spark JAR task test on multiple DBR versions ([#1665](https://github.com/databricks/cli/pull/1665)). * Stop tracking file path locations in bundle resources ([#1673](https://github.com/databricks/cli/pull/1673)). * Update VS Code settings to match latest value from IDE plugin ([#1677](https://github.com/databricks/cli/pull/1677)). * Use `service.NamedIdMap` to make lookup generation deterministic ([#1678](https://github.com/databricks/cli/pull/1678)). * [Internal] Remove dependency to the `openapi` package of the Go SDK ([#1676](https://github.com/databricks/cli/pull/1676)). * Upgrade TF provider to 1.50.0 ([#1681](https://github.com/databricks/cli/pull/1681)). * Upgrade Go SDK to 0.44.0 ([#1679](https://github.com/databricks/cli/pull/1679)). API Changes: * Changed `databricks account budgets create` command . New request type is . * Changed `databricks account budgets create` command to return . * Changed `databricks account budgets delete` command . New request type is . * Changed `databricks account budgets delete` command to return . * Changed `databricks account budgets get` command . New request type is . * Changed `databricks account budgets get` command to return . * Changed `databricks account budgets list` command to require request of . * Changed `databricks account budgets list` command to return . * Changed `databricks account budgets update` command . New request type is . * Changed `databricks account budgets update` command to return . * Added `databricks account usage-dashboards` command group. * Changed `databricks model-versions get` command to return . * Changed `databricks cluster-policies create` command with new required argument order. * Changed `databricks cluster-policies edit` command with new required argument order. * Added `databricks clusters update` command. * Added `databricks genie` command group. * Changed `databricks permission-migration migrate-permissions` command . New request type is . * Changed `databricks permission-migration migrate-permissions` command to return . * Changed `databricks account workspace-assignment delete` command to return . * Changed `databricks account workspace-assignment update` command with new required argument order. * Changed `databricks account custom-app-integration create` command with new required argument order. * Changed `databricks account custom-app-integration list` command to require request of . * Changed `databricks account published-app-integration list` command to require request of . * Removed `databricks apps` command group. * Added `databricks notification-destinations` command group. * Changed `databricks shares list` command to require request of . * Changed `databricks alerts create` command . New request type is . * Changed `databricks alerts delete` command . New request type is . * Changed `databricks alerts delete` command to return . * Changed `databricks alerts get` command with new required argument order. * Changed `databricks alerts list` command to require request of . * Changed `databricks alerts list` command to return . * Changed `databricks alerts update` command . New request type is . * Changed `databricks alerts update` command to return . * Changed `databricks queries create` command . New request type is . * Changed `databricks queries delete` command . New request type is . * Changed `databricks queries delete` command to return . * Changed `databricks queries get` command with new required argument order. * Changed `databricks queries list` command to return . * Removed `databricks queries restore` command. * Changed `databricks queries update` command . New request type is . * Added `databricks queries list-visualizations` command. * Changed `databricks query-visualizations create` command . New request type is . * Changed `databricks query-visualizations delete` command . New request type is . * Changed `databricks query-visualizations delete` command to return . * Changed `databricks query-visualizations update` command . New request type is . * Changed `databricks statement-execution execute-statement` command to return . * Changed `databricks statement-execution get-statement` command to return . * Added `databricks alerts-legacy` command group. * Added `databricks queries-legacy` command group. * Added `databricks query-visualizations-legacy` command group. OpenAPI commit f98c07f9c71f579de65d2587bb0292f83d10e55d (2024-08-12) Dependency updates: * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.8.0 ([#1652](https://github.com/databricks/cli/pull/1652)). * Bump golang.org/x/sync from 0.7.0 to 0.8.0 ([#1655](https://github.com/databricks/cli/pull/1655)). * Bump golang.org/x/mod from 0.19.0 to 0.20.0 ([#1654](https://github.com/databricks/cli/pull/1654)). * Bump golang.org/x/oauth2 from 0.21.0 to 0.22.0 ([#1653](https://github.com/databricks/cli/pull/1653)). * Bump golang.org/x/text from 0.16.0 to 0.17.0 ([#1670](https://github.com/databricks/cli/pull/1670)). * Bump golang.org/x/term from 0.22.0 to 0.23.0 ([#1669](https://github.com/databricks/cli/pull/1669)). --- CHANGELOG.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1e0b9a5a..39960e308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,96 @@ # Version changelog +## [Release] Release v0.226.0 + +CLI: + * Add command line autocomplete to the fs commands ([#1622](https://github.com/databricks/cli/pull/1622)). + * Add trailing slash to directory to produce completions for ([#1666](https://github.com/databricks/cli/pull/1666)). + * Fix ability to import the CLI repository as module ([#1671](https://github.com/databricks/cli/pull/1671)). + * Fix host resolution order in `auth login` ([#1370](https://github.com/databricks/cli/pull/1370)). + * Print text logs in `import-dir` and `export-dir` commands ([#1682](https://github.com/databricks/cli/pull/1682)). + +Bundles: + * Expand and upload local wheel libraries for all task types ([#1649](https://github.com/databricks/cli/pull/1649)). + * Clarify file format required for the `config-file` flag in `bundle init` ([#1651](https://github.com/databricks/cli/pull/1651)). + * Fixed incorrectly cleaning up python wheel dist folder ([#1656](https://github.com/databricks/cli/pull/1656)). + * Merge job parameters based on their name ([#1659](https://github.com/databricks/cli/pull/1659)). + * Fix glob expansion after running a generic build command ([#1662](https://github.com/databricks/cli/pull/1662)). + * Upload local libraries even if they don't have artifact defined ([#1664](https://github.com/databricks/cli/pull/1664)). + +Internal: + * Fix python wheel task integration tests ([#1648](https://github.com/databricks/cli/pull/1648)). + * Skip pushing Terraform state after destroy ([#1667](https://github.com/databricks/cli/pull/1667)). + * Enable Spark JAR task test ([#1658](https://github.com/databricks/cli/pull/1658)). + * Run Spark JAR task test on multiple DBR versions ([#1665](https://github.com/databricks/cli/pull/1665)). + * Stop tracking file path locations in bundle resources ([#1673](https://github.com/databricks/cli/pull/1673)). + * Update VS Code settings to match latest value from IDE plugin ([#1677](https://github.com/databricks/cli/pull/1677)). + * Use `service.NamedIdMap` to make lookup generation deterministic ([#1678](https://github.com/databricks/cli/pull/1678)). + * [Internal] Remove dependency to the `openapi` package of the Go SDK ([#1676](https://github.com/databricks/cli/pull/1676)). + * Upgrade TF provider to 1.50.0 ([#1681](https://github.com/databricks/cli/pull/1681)). + * Upgrade Go SDK to 0.44.0 ([#1679](https://github.com/databricks/cli/pull/1679)). + +API Changes: + * Changed `databricks account budgets create` command . New request type is . + * Changed `databricks account budgets create` command to return . + * Changed `databricks account budgets delete` command . New request type is . + * Changed `databricks account budgets delete` command to return . + * Changed `databricks account budgets get` command . New request type is . + * Changed `databricks account budgets get` command to return . + * Changed `databricks account budgets list` command to require request of . + * Changed `databricks account budgets list` command to return . + * Changed `databricks account budgets update` command . New request type is . + * Changed `databricks account budgets update` command to return . + * Added `databricks account usage-dashboards` command group. + * Changed `databricks model-versions get` command to return . + * Changed `databricks cluster-policies create` command with new required argument order. + * Changed `databricks cluster-policies edit` command with new required argument order. + * Added `databricks clusters update` command. + * Added `databricks genie` command group. + * Changed `databricks permission-migration migrate-permissions` command . New request type is . + * Changed `databricks permission-migration migrate-permissions` command to return . + * Changed `databricks account workspace-assignment delete` command to return . + * Changed `databricks account workspace-assignment update` command with new required argument order. + * Changed `databricks account custom-app-integration create` command with new required argument order. + * Changed `databricks account custom-app-integration list` command to require request of . + * Changed `databricks account published-app-integration list` command to require request of . + * Removed `databricks apps` command group. + * Added `databricks notification-destinations` command group. + * Changed `databricks shares list` command to require request of . + * Changed `databricks alerts create` command . New request type is . + * Changed `databricks alerts delete` command . New request type is . + * Changed `databricks alerts delete` command to return . + * Changed `databricks alerts get` command with new required argument order. + * Changed `databricks alerts list` command to require request of . + * Changed `databricks alerts list` command to return . + * Changed `databricks alerts update` command . New request type is . + * Changed `databricks alerts update` command to return . + * Changed `databricks queries create` command . New request type is . + * Changed `databricks queries delete` command . New request type is . + * Changed `databricks queries delete` command to return . + * Changed `databricks queries get` command with new required argument order. + * Changed `databricks queries list` command to return . + * Removed `databricks queries restore` command. + * Changed `databricks queries update` command . New request type is . + * Added `databricks queries list-visualizations` command. + * Changed `databricks query-visualizations create` command . New request type is . + * Changed `databricks query-visualizations delete` command . New request type is . + * Changed `databricks query-visualizations delete` command to return . + * Changed `databricks query-visualizations update` command . New request type is . + * Changed `databricks statement-execution execute-statement` command to return . + * Changed `databricks statement-execution get-statement` command to return . + * Added `databricks alerts-legacy` command group. + * Added `databricks queries-legacy` command group. + * Added `databricks query-visualizations-legacy` command group. + +OpenAPI commit f98c07f9c71f579de65d2587bb0292f83d10e55d (2024-08-12) +Dependency updates: + * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.8.0 ([#1652](https://github.com/databricks/cli/pull/1652)). + * Bump golang.org/x/sync from 0.7.0 to 0.8.0 ([#1655](https://github.com/databricks/cli/pull/1655)). + * Bump golang.org/x/mod from 0.19.0 to 0.20.0 ([#1654](https://github.com/databricks/cli/pull/1654)). + * Bump golang.org/x/oauth2 from 0.21.0 to 0.22.0 ([#1653](https://github.com/databricks/cli/pull/1653)). + * Bump golang.org/x/text from 0.16.0 to 0.17.0 ([#1670](https://github.com/databricks/cli/pull/1670)). + * Bump golang.org/x/term from 0.22.0 to 0.23.0 ([#1669](https://github.com/databricks/cli/pull/1669)). + ## 0.225.0 Bundles: From ab4e8099fb71af95b566f2d78e9d53523bdaa5c5 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Mon, 19 Aug 2024 15:24:56 +0200 Subject: [PATCH 71/88] Add `import` option for PyDABs (#1693) ## Changes Add 'import' option for PyDABs ## Tests Manually --- bundle/config/experimental.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 12048a322..66e975820 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -39,6 +39,12 @@ type PyDABs struct { // Required if PyDABs is enabled. PyDABs will load the code in the specified // environment. VEnvPath string `json:"venv_path,omitempty"` + + // Import contains a list Python packages with PyDABs code. + // + // These packages are imported to discover resources, resource generators, and mutators. + // This list can include namespace packages, which causes the import of nested packages. + Import []string `json:"import,omitempty"` } type Command string From 7de7583b37a84ee6d4a4f163ad4ba1d87207850f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 19 Aug 2024 17:15:14 +0200 Subject: [PATCH 72/88] Make fileset take optional list of paths to list (#1684) ## Changes Before this change, the fileset library would take a single root path and list all files in it. To support an allowlist of paths to list (much like a Git `pathspec` without patterns; see [pathspec](pathspec)), this change introduces an optional argument to `fileset.New` where the caller can specify paths to list. If not specified, this argument defaults to list `.` (i.e. list all files in the root). The motivation for this change is that we wish to expose this pattern in bundles. Users should be able to specify which paths to synchronize instead of always only synchronizing the bundle root directory. [pathspec]: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec ## Tests New and existing unit tests. --- .../config/validate/validate_sync_patterns.go | 2 +- bundle/deploy/state_test.go | 4 +- bundle/deploy/state_update_test.go | 2 +- libs/fileset/fileset.go | 95 +++++++++--- libs/fileset/fileset_test.go | 144 ++++++++++++++++++ libs/fileset/glob_test.go | 18 ++- libs/fileset/testdata/dir1/a | 0 libs/fileset/testdata/dir1/b | 0 libs/fileset/testdata/dir2/a | 0 libs/fileset/testdata/dir2/b | 0 libs/fileset/testdata/dir3/a | 1 + libs/git/fileset.go | 10 +- libs/git/fileset_test.go | 4 +- libs/sync/snapshot_state_test.go | 2 +- libs/sync/snapshot_test.go | 28 ++-- libs/sync/sync.go | 6 +- 16 files changed, 257 insertions(+), 59 deletions(-) create mode 100644 libs/fileset/fileset_test.go create mode 100644 libs/fileset/testdata/dir1/a create mode 100644 libs/fileset/testdata/dir1/b create mode 100644 libs/fileset/testdata/dir2/a create mode 100644 libs/fileset/testdata/dir2/b create mode 120000 libs/fileset/testdata/dir3/a diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index fd011bf78..52f06835c 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -63,7 +63,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di return err } - all, err := fs.All() + all, err := fs.Files() if err != nil { return err } diff --git a/bundle/deploy/state_test.go b/bundle/deploy/state_test.go index 5e1e54230..d149b0efa 100644 --- a/bundle/deploy/state_test.go +++ b/bundle/deploy/state_test.go @@ -18,7 +18,7 @@ func TestFromSlice(t *testing.T) { testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") - files, err := fileset.All() + files, err := fileset.Files() require.NoError(t, err) f, err := FromSlice(files) @@ -38,7 +38,7 @@ func TestToSlice(t *testing.T) { testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") - files, err := fileset.All() + files, err := fileset.Files() require.NoError(t, err) f, err := FromSlice(files) diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index 2982546d5..72096d142 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -23,7 +23,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle { testutil.Touch(t, tmpDir, "test1.py") testutil.TouchNotebook(t, tmpDir, "test2.py") - files, err := fileset.New(vfs.MustNew(tmpDir)).All() + files, err := fileset.New(vfs.MustNew(tmpDir)).Files() require.NoError(t, err) return &bundle.Bundle{ diff --git a/libs/fileset/fileset.go b/libs/fileset/fileset.go index d0f00f97a..00c6dcfa4 100644 --- a/libs/fileset/fileset.go +++ b/libs/fileset/fileset.go @@ -3,25 +3,56 @@ package fileset import ( "fmt" "io/fs" - "os" + pathlib "path" + "path/filepath" + "slices" "github.com/databricks/cli/libs/vfs" ) -// FileSet facilitates fast recursive file listing of a path. +// FileSet facilitates recursive file listing for paths rooted at a given directory. // It optionally takes into account ignore rules through the [Ignorer] interface. type FileSet struct { // Root path of the fileset. root vfs.Path + // Paths to include in the fileset. + // Files are included as-is (if not ignored) and directories are traversed recursively. + // Defaults to []string{"."} if not specified. + paths []string + // Ignorer interface to check if a file or directory should be ignored. ignore Ignorer } // New returns a [FileSet] for the given root path. -func New(root vfs.Path) *FileSet { +// It optionally accepts a list of paths relative to the root to include in the fileset. +// If not specified, it defaults to including all files in the root path. +func New(root vfs.Path, args ...[]string) *FileSet { + // Default to including all files in the root path. + if len(args) == 0 { + args = [][]string{{"."}} + } + + // Collect list of normalized and cleaned paths. + var paths []string + for _, arg := range args { + for _, path := range arg { + path = filepath.ToSlash(path) + path = pathlib.Clean(path) + + // Skip path if it's already in the list. + if slices.Contains(paths, path) { + continue + } + + paths = append(paths, path) + } + } + return &FileSet{ root: root, + paths: paths, ignore: nopIgnorer{}, } } @@ -36,30 +67,38 @@ func (w *FileSet) SetIgnorer(ignore Ignorer) { w.ignore = ignore } -// Return all tracked files for Repo -func (w *FileSet) All() ([]File, error) { - return w.recursiveListFiles() +// Files returns performs recursive listing on all configured paths and returns +// the collection of files it finds (and are not ignored). +// The returned slice does not contain duplicates. +// The order of files in the slice is stable. +func (w *FileSet) Files() (out []File, err error) { + seen := make(map[string]struct{}) + for _, p := range w.paths { + files, err := w.recursiveListFiles(p, seen) + if err != nil { + return nil, err + } + out = append(out, files...) + } + return out, nil } // Recursively traverses dir in a depth first manner and returns a list of all files // that are being tracked in the FileSet (ie not being ignored for matching one of the // patterns in w.ignore) -func (w *FileSet) recursiveListFiles() (fileList []File, err error) { - err = fs.WalkDir(w.root, ".", func(name string, d fs.DirEntry, err error) error { +func (w *FileSet) recursiveListFiles(path string, seen map[string]struct{}) (out []File, err error) { + err = fs.WalkDir(w.root, path, func(name string, d fs.DirEntry, err error) error { if err != nil { return err } - // skip symlinks info, err := d.Info() if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { - return nil - } - if d.IsDir() { + switch { + case info.Mode().IsDir(): ign, err := w.ignore.IgnoreDirectory(name) if err != nil { return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) @@ -67,18 +106,28 @@ func (w *FileSet) recursiveListFiles() (fileList []File, err error) { if ign { return fs.SkipDir } - return nil + + case info.Mode().IsRegular(): + ign, err := w.ignore.IgnoreFile(name) + if err != nil { + return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) + } + if ign { + return nil + } + + // Skip duplicates + if _, ok := seen[name]; ok { + return nil + } + + seen[name] = struct{}{} + out = append(out, NewFile(w.root, d, name)) + + default: + // Skip non-regular files (e.g. symlinks). } - ign, err := w.ignore.IgnoreFile(name) - if err != nil { - return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) - } - if ign { - return nil - } - - fileList = append(fileList, NewFile(w.root, d, name)) return nil }) return diff --git a/libs/fileset/fileset_test.go b/libs/fileset/fileset_test.go new file mode 100644 index 000000000..be27b6b6f --- /dev/null +++ b/libs/fileset/fileset_test.go @@ -0,0 +1,144 @@ +package fileset + +import ( + "errors" + "testing" + + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" +) + +func TestFileSet_NoPaths(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 4) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) + assert.Equal(t, "dir2/a", files[2].Relative) + assert.Equal(t, "dir2/b", files[3].Relative) +} + +func TestFileSet_ParentPath(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{".."}) + _, err := fs.Files() + + // It is impossible to escape the root directory. + assert.Error(t, err) +} + +func TestFileSet_DuplicatePaths(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{"dir1", "dir1"}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) +} + +func TestFileSet_OverlappingPaths(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{"dir1", "dir1/a"}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) +} + +func TestFileSet_IgnoreDirError(t *testing.T) { + testError := errors.New("test error") + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{dirErr: testError}) + _, err := fs.Files() + assert.ErrorIs(t, err, testError) +} + +func TestFileSet_IgnoreDir(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{dir: []string{"dir1"}}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir2/a", files[0].Relative) + assert.Equal(t, "dir2/b", files[1].Relative) +} + +func TestFileSet_IgnoreFileError(t *testing.T) { + testError := errors.New("test error") + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{fileErr: testError}) + _, err := fs.Files() + assert.ErrorIs(t, err, testError) +} + +func TestFileSet_IgnoreFile(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{file: []string{"dir1/a"}}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 3) + assert.Equal(t, "dir1/b", files[0].Relative) + assert.Equal(t, "dir2/a", files[1].Relative) + assert.Equal(t, "dir2/b", files[2].Relative) +} + +type testIgnorer struct { + // dir is a list of directories to ignore. Strings are compared verbatim. + dir []string + + // dirErr is an error to return when IgnoreDirectory is called. + dirErr error + + // file is a list of files to ignore. Strings are compared verbatim. + file []string + + // fileErr is an error to return when IgnoreFile is called. + fileErr error +} + +// IgnoreDirectory returns true if the path is in the dir list. +// If dirErr is set, it returns dirErr. +func (t testIgnorer) IgnoreDirectory(path string) (bool, error) { + if t.dirErr != nil { + return false, t.dirErr + } + + for _, d := range t.dir { + if d == path { + return true, nil + } + } + + return false, nil +} + +// IgnoreFile returns true if the path is in the file list. +// If fileErr is set, it returns fileErr. +func (t testIgnorer) IgnoreFile(path string) (bool, error) { + if t.fileErr != nil { + return false, t.fileErr + } + + for _, f := range t.file { + if f == path { + return true, nil + } + } + + return false, nil +} diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 8418df73a..9eb786db9 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -24,15 +24,19 @@ func TestGlobFileset(t *testing.T) { entries, err := root.ReadDir(".") require.NoError(t, err) + // Remove testdata folder from entries + entries = slices.DeleteFunc(entries, func(de fs.DirEntry) bool { + return de.Name() == "testdata" + }) + g, err := NewGlobSet(root, []string{ "./*.go", }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) - // +1 as there's one folder in ../filer require.Equal(t, len(files), len(entries)) for _, f := range files { exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool { @@ -46,7 +50,7 @@ func TestGlobFileset(t *testing.T) { }) require.NoError(t, err) - files, err = g.All() + files, err = g.Files() require.NoError(t, err) require.Equal(t, len(files), 0) } @@ -61,7 +65,7 @@ func TestGlobFilesetWithRelativeRoot(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.Equal(t, len(files), len(entries)) } @@ -82,7 +86,7 @@ func TestGlobFilesetRecursively(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } @@ -103,7 +107,7 @@ func TestGlobFilesetDir(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } @@ -124,7 +128,7 @@ func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } diff --git a/libs/fileset/testdata/dir1/a b/libs/fileset/testdata/dir1/a new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir1/b b/libs/fileset/testdata/dir1/b new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir2/a b/libs/fileset/testdata/dir2/a new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir2/b b/libs/fileset/testdata/dir2/b new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir3/a b/libs/fileset/testdata/dir3/a new file mode 120000 index 000000000..5ac5651e9 --- /dev/null +++ b/libs/fileset/testdata/dir3/a @@ -0,0 +1 @@ +../dir1/a \ No newline at end of file diff --git a/libs/git/fileset.go b/libs/git/fileset.go index f1986aa20..bb1cd4692 100644 --- a/libs/git/fileset.go +++ b/libs/git/fileset.go @@ -7,15 +7,15 @@ import ( // FileSet is Git repository aware implementation of [fileset.FileSet]. // It forces checking if gitignore files have been modified every -// time a call to [FileSet.All] is made. +// time a call to [FileSet.Files] is made. type FileSet struct { fileset *fileset.FileSet view *View } // NewFileSet returns [FileSet] for the Git repository located at `root`. -func NewFileSet(root vfs.Path) (*FileSet, error) { - fs := fileset.New(root) +func NewFileSet(root vfs.Path, paths ...[]string) (*FileSet, error) { + fs := fileset.New(root, paths...) v, err := NewView(root) if err != nil { return nil, err @@ -35,9 +35,9 @@ func (f *FileSet) IgnoreDirectory(dir string) (bool, error) { return f.view.IgnoreDirectory(dir) } -func (f *FileSet) All() ([]fileset.File, error) { +func (f *FileSet) Files() ([]fileset.File, error) { f.view.repo.taintIgnoreRules() - return f.fileset.All() + return f.fileset.Files() } func (f *FileSet) EnsureValidGitIgnoreExists() error { diff --git a/libs/git/fileset_test.go b/libs/git/fileset_test.go index 4e6172bfd..37f3611d1 100644 --- a/libs/git/fileset_test.go +++ b/libs/git/fileset_test.go @@ -15,7 +15,7 @@ import ( func testFileSetAll(t *testing.T, root string) { fileSet, err := NewFileSet(vfs.MustNew(root)) require.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) require.Len(t, files, 3) assert.Equal(t, path.Join("a", "b", "world.txt"), files[0].Relative) @@ -37,7 +37,7 @@ func TestFileSetNonCleanRoot(t *testing.T) { // This should yield the same result as above test. fileSet, err := NewFileSet(vfs.MustNew("./testdata/../testdata")) require.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) assert.Len(t, files, 3) } diff --git a/libs/sync/snapshot_state_test.go b/libs/sync/snapshot_state_test.go index 92c14e8e0..248e5832c 100644 --- a/libs/sync/snapshot_state_test.go +++ b/libs/sync/snapshot_state_test.go @@ -13,7 +13,7 @@ import ( func TestSnapshotState(t *testing.T) { fileSet := fileset.New(vfs.MustNew("./testdata/sync-fileset")) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) // Assert initial contents of the fileset diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go index 050b5d965..b7830406d 100644 --- a/libs/sync/snapshot_test.go +++ b/libs/sync/snapshot_test.go @@ -47,7 +47,7 @@ func TestDiff(t *testing.T) { defer f2.Close(t) // New files are put - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -62,7 +62,7 @@ func TestDiff(t *testing.T) { // world.txt is editted f2.Overwrite(t, "bunnies are cute.") assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -77,7 +77,7 @@ func TestDiff(t *testing.T) { // hello.txt is deleted f1.Remove(t) assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -113,7 +113,7 @@ func TestSymlinkDiff(t *testing.T) { err = os.Symlink(filepath.Join(projectDir, "foo"), filepath.Join(projectDir, "bar")) assert.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -141,7 +141,7 @@ func TestFolderDiff(t *testing.T) { defer f1.Close(t) f1.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -153,7 +153,7 @@ func TestFolderDiff(t *testing.T) { assert.Contains(t, change.put, "foo/bar.py") f1.Remove(t) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -184,7 +184,7 @@ func TestPythonNotebookDiff(t *testing.T) { defer foo.Close(t) // Case 1: notebook foo.py is uploaded - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) foo.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") change, err := state.diff(ctx, files) @@ -199,7 +199,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 2: notebook foo.py is converted to python script by removing // magic keyword foo.Overwrite(t, "print(\"abc\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -213,7 +213,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 3: Python script foo.py is converted to a databricks notebook foo.Overwrite(t, "# Databricks notebook source\nprint(\"def\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -228,7 +228,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 4: Python notebook foo.py is deleted, and its remote name is used in change.delete foo.Remove(t) assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -260,7 +260,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { defer pythonFoo.Close(t) vanillaFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo")) defer vanillaFoo.Close(t) - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -271,7 +271,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { // errors out because they point to the same destination pythonFoo.Overwrite(t, "# Databricks notebook source\nprint(\"def\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.ErrorContains(t, err, "both foo and foo.py point to the same remote file location foo. Please remove one of them from your local project") @@ -296,7 +296,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { pythonFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.py")) defer pythonFoo.Close(t) pythonFoo.Overwrite(t, "# Databricks notebook source\n") - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -308,7 +308,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { sqlFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.sql")) defer sqlFoo.Close(t) sqlFoo.Overwrite(t, "-- Databricks notebook source\n") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 3d5bc61ec..ffcf3878e 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -195,14 +195,14 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) { all := set.NewSetF(func(f fileset.File) string { return f.Relative }) - gitFiles, err := s.fileSet.All() + gitFiles, err := s.fileSet.Files() if err != nil { log.Errorf(ctx, "cannot list files: %s", err) return nil, err } all.Add(gitFiles...) - include, err := s.includeFileSet.All() + include, err := s.includeFileSet.Files() if err != nil { log.Errorf(ctx, "cannot list include files: %s", err) return nil, err @@ -210,7 +210,7 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) { all.Add(include...) - exclude, err := s.excludeFileSet.All() + exclude, err := s.excludeFileSet.Files() if err != nil { log.Errorf(ctx, "cannot list exclude files: %s", err) return nil, err From 2b8cbc31cf03062287897b14af67aae55bd90f2a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 19 Aug 2024 17:41:02 +0200 Subject: [PATCH 73/88] Pass through paths argument to libs/sync (#1689) ## Changes Requires #1684. ## Tests Ran the sync integration tests. --- bundle/deploy/files/sync.go | 8 +++++--- cmd/sync/sync.go | 6 +++++- cmd/sync/sync_test.go | 4 ++-- libs/sync/sync.go | 14 ++++++++------ libs/sync/watchdog.go | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index a308668d3..dc45053f9 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -28,10 +28,12 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp } opts := &sync.SyncOptions{ - LocalPath: rb.BundleRoot(), + LocalRoot: rb.BundleRoot(), + Paths: []string{"."}, + Include: includes, + Exclude: rb.Config().Sync.Exclude, + RemotePath: rb.Config().Workspace.FilePath, - Include: includes, - Exclude: rb.Config().Sync.Exclude, Host: rb.WorkspaceClient().Config.Host, Full: false, diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index bab451593..23a4c018f 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -47,7 +47,11 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn } opts := sync.SyncOptions{ - LocalPath: vfs.MustNew(args[0]), + LocalRoot: vfs.MustNew(args[0]), + Paths: []string{"."}, + Include: nil, + Exclude: nil, + RemotePath: args[1], Full: f.full, PollInterval: f.interval, diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 564aeae56..0d0c57385 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -33,7 +33,7 @@ func TestSyncOptionsFromBundle(t *testing.T) { f := syncFlags{} opts, err := f.syncOptionsFromBundle(New(), []string{}, b) require.NoError(t, err) - assert.Equal(t, tempDir, opts.LocalPath.Native()) + assert.Equal(t, tempDir, opts.LocalRoot.Native()) assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath) assert.Equal(t, filepath.Join(tempDir, ".databricks", "bundle", "default"), opts.SnapshotBasePath) assert.NotNil(t, opts.WorkspaceClient) @@ -59,6 +59,6 @@ func TestSyncOptionsFromArgs(t *testing.T) { cmd.SetContext(root.SetWorkspaceClient(context.Background(), nil)) opts, err := f.syncOptionsFromArgs(cmd, []string{local, remote}) require.NoError(t, err) - assert.Equal(t, local, opts.LocalPath.Native()) + assert.Equal(t, local, opts.LocalRoot.Native()) assert.Equal(t, remote, opts.RemotePath) } diff --git a/libs/sync/sync.go b/libs/sync/sync.go index ffcf3878e..9eaebf2ad 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -16,10 +16,12 @@ import ( ) type SyncOptions struct { - LocalPath vfs.Path + LocalRoot vfs.Path + Paths []string + Include []string + Exclude []string + RemotePath string - Include []string - Exclude []string Full bool @@ -51,7 +53,7 @@ type Sync struct { // New initializes and returns a new [Sync] instance. func New(ctx context.Context, opts SyncOptions) (*Sync, error) { - fileSet, err := git.NewFileSet(opts.LocalPath) + fileSet, err := git.NewFileSet(opts.LocalRoot, opts.Paths) if err != nil { return nil, err } @@ -61,12 +63,12 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } - includeFileSet, err := fileset.NewGlobSet(opts.LocalPath, opts.Include) + includeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Include) if err != nil { return nil, err } - excludeFileSet, err := fileset.NewGlobSet(opts.LocalPath, opts.Exclude) + excludeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Exclude) if err != nil { return nil, err } diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index ca7ec46e9..cc2ca83c5 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -57,7 +57,7 @@ func (s *Sync) applyMkdir(ctx context.Context, localName string) error { func (s *Sync) applyPut(ctx context.Context, localName string) error { s.notifyProgress(ctx, EventActionPut, localName, 0.0) - localFile, err := s.LocalPath.Open(localName) + localFile, err := s.LocalRoot.Open(localName) if err != nil { return err } From 8238a6ad0aad46402e6a838d6272d34718c8ad92 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 19 Aug 2024 17:47:18 +0200 Subject: [PATCH 74/88] Remove reference to "dbt" in the default-sql template (#1696) ## Changes The `default-sql` template inadvertently mentioned dbt in one of the prompts. This PR removes that reference. --- .../templates/default-sql/databricks_template_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-sql/databricks_template_schema.json b/libs/template/templates/default-sql/databricks_template_schema.json index 329f91962..113cbef64 100644 --- a/libs/template/templates/default-sql/databricks_template_schema.json +++ b/libs/template/templates/default-sql/databricks_template_schema.json @@ -13,7 +13,7 @@ "type": "string", "pattern": "^/sql/.\\../warehouses/[a-z0-9]+$", "pattern_match_failure_message": "Path must be of the form /sql/1.0/warehouses/", - "description": "\nPlease provide the HTTP Path of the SQL warehouse you would like to use with dbt during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]", + "description": "\nPlease provide the HTTP Path of the SQL warehouse you would like to use during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]", "order": 2 }, "default_catalog": { From 07627023f5be85e90dd9fc27bc83a3144fcccc10 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 19 Aug 2024 18:27:57 +0200 Subject: [PATCH 75/88] Pause continuous pipelines when 'mode: development' is used (#1590) ## Changes This makes it so that the pipelines `continuous` property is set to false by default when using `mode: development`. --- bundle/config/mutator/process_target_mode.go | 1 + bundle/config/mutator/process_target_mode_test.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 9db97907d..fb71f751b 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -71,6 +71,7 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) diag.Diagno for i := range r.Pipelines { r.Pipelines[i].Name = prefix + r.Pipelines[i].Name r.Pipelines[i].Development = true + r.Pipelines[i].Continuous = false // (pipelines don't yet support tags) } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index f0c8aee9e..1a2e96fab 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -82,7 +82,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}}, + "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}}, @@ -145,6 +145,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) + assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous) assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) // Experiment 1 From 78d0ac5c6afcafe9d03c52acd043f2a0235c2afa Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 19 Aug 2024 20:18:50 +0200 Subject: [PATCH 76/88] Add configurable presets for name prefixes, tags, etc. (#1490) ## Changes This adds configurable transformations based on the transformations currently seen in `mode: development`. Example databricks.yml showcasing how some transformations: ``` bundle: name: my_bundle targets: dev: presets: prefix: "myprefix_" # prefix all resource names with myprefix_ pipelines_development: true # set development to true by default for pipelines trigger_pause_status: PAUSED # set pause_status to PAUSED by default for all triggers and schedules jobs_max_concurrent_runs: 10 # set max_concurrent runs to 10 by default for all jobs tags: dev: true ``` ## Tests * Existing process_target_mode tests that were adapted to use this new code * Unit tests specific for the new mutator * Unit tests for config loading and merging * Manual e2e testing --- bundle/config/mutator/apply_presets.go | 209 ++++++++++++++++++ bundle/config/mutator/apply_presets_test.go | 196 ++++++++++++++++ bundle/config/mutator/process_target_mode.go | 130 +++++------ .../mutator/process_target_mode_test.go | 150 ++++++++++++- bundle/config/presets.go | 32 +++ bundle/config/root.go | 5 + bundle/config/target.go | 4 + bundle/phases/initialize.go | 1 + bundle/tests/presets/databricks.yml | 22 ++ bundle/tests/presets_test.go | 28 +++ 10 files changed, 687 insertions(+), 90 deletions(-) create mode 100644 bundle/config/mutator/apply_presets.go create mode 100644 bundle/config/mutator/apply_presets_test.go create mode 100644 bundle/config/presets.go create mode 100644 bundle/tests/presets/databricks.yml create mode 100644 bundle/tests/presets_test.go diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go new file mode 100644 index 000000000..42e6ab95f --- /dev/null +++ b/bundle/config/mutator/apply_presets.go @@ -0,0 +1,209 @@ +package mutator + +import ( + "context" + "path" + "slices" + "sort" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type applyPresets struct{} + +// Apply all presets, e.g. the prefix presets that +// adds a prefix to all names of all resources. +func ApplyPresets() *applyPresets { + return &applyPresets{} +} + +type Tag struct { + Key string + Value string +} + +func (m *applyPresets) Name() string { + return "ApplyPresets" +} + +func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if d := validatePauseStatus(b); d != nil { + return d + } + + r := b.Config.Resources + t := b.Config.Presets + prefix := t.NamePrefix + tags := toTagArray(t.Tags) + + // Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus + for _, j := range r.Jobs { + j.Name = prefix + j.Name + if j.Tags == nil { + j.Tags = make(map[string]string) + } + for _, tag := range tags { + if j.Tags[tag.Key] == "" { + j.Tags[tag.Key] = tag.Value + } + } + if j.MaxConcurrentRuns == 0 { + j.MaxConcurrentRuns = t.JobsMaxConcurrentRuns + } + if t.TriggerPauseStatus != "" { + paused := jobs.PauseStatusPaused + if t.TriggerPauseStatus == config.Unpaused { + paused = jobs.PauseStatusUnpaused + } + + if j.Schedule != nil && j.Schedule.PauseStatus == "" { + j.Schedule.PauseStatus = paused + } + if j.Continuous != nil && j.Continuous.PauseStatus == "" { + j.Continuous.PauseStatus = paused + } + if j.Trigger != nil && j.Trigger.PauseStatus == "" { + j.Trigger.PauseStatus = paused + } + } + } + + // Pipelines presets: Prefix, PipelinesDevelopment + for i := range r.Pipelines { + r.Pipelines[i].Name = prefix + r.Pipelines[i].Name + if config.IsExplicitlyEnabled(t.PipelinesDevelopment) { + r.Pipelines[i].Development = true + } + if t.TriggerPauseStatus == config.Paused { + r.Pipelines[i].Continuous = false + } + + // As of 2024-06, pipelines don't yet support tags + } + + // Models presets: Prefix, Tags + for _, m := range r.Models { + m.Name = prefix + m.Name + for _, t := range tags { + exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool { + return modelTag.Key == t.Key + }) + if !exists { + // Only add this tag if the resource didn't include any tag that overrides its value. + m.Tags = append(m.Tags, ml.ModelTag{Key: t.Key, Value: t.Value}) + } + } + } + + // Experiments presets: Prefix, Tags + for _, e := range r.Experiments { + filepath := e.Name + dir := path.Dir(filepath) + base := path.Base(filepath) + if dir == "." { + e.Name = prefix + base + } else { + e.Name = dir + "/" + prefix + base + } + for _, t := range tags { + exists := false + for _, experimentTag := range e.Tags { + if experimentTag.Key == t.Key { + exists = true + break + } + } + if !exists { + e.Tags = append(e.Tags, ml.ExperimentTag{Key: t.Key, Value: t.Value}) + } + } + } + + // Model serving endpoint presets: Prefix + for i := range r.ModelServingEndpoints { + r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name + + // As of 2024-06, model serving endpoints don't yet support tags + } + + // Registered models presets: Prefix + for i := range r.RegisteredModels { + r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name + + // As of 2024-06, registered models don't yet support tags + } + + // Quality monitors presets: Prefix + if t.TriggerPauseStatus == config.Paused { + for i := range r.QualityMonitors { + // Remove all schedules from monitors, since they don't support pausing/unpausing. + // Quality monitors might support the "pause" property in the future, so at the + // CLI level we do respect that property if it is set to "unpaused." + if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + r.QualityMonitors[i].Schedule = nil + } + } + } + + // Schemas: Prefix + for i := range r.Schemas { + prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" + r.Schemas[i].Name = prefix + r.Schemas[i].Name + // HTTP API for schemas doesn't yet support tags. It's only supported in + // the Databricks UI and via the SQL API. + } + + return nil +} + +func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { + p := b.Config.Presets.TriggerPauseStatus + if p == "" || p == config.Paused || p == config.Unpaused { + return nil + } + return diag.Diagnostics{{ + Summary: "Invalid value for trigger_pause_status, should be PAUSED or UNPAUSED", + Severity: diag.Error, + Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + }} +} + +// toTagArray converts a map of tags to an array of tags. +// We sort tags so ensure stable ordering. +func toTagArray(tags map[string]string) []Tag { + var tagArray []Tag + if tags == nil { + return tagArray + } + for key, value := range tags { + tagArray = append(tagArray, Tag{Key: key, Value: value}) + } + sort.Slice(tagArray, func(i, j int) bool { + return tagArray[i].Key < tagArray[j].Key + }) + return tagArray +} + +// normalizePrefix prefixes strings like '[dev lennart] ' to 'dev_lennart_'. +// We leave unicode letters and numbers but remove all "special characters." +func normalizePrefix(prefix string) string { + prefix = strings.ReplaceAll(prefix, "[", "") + prefix = strings.Trim(prefix, " ") + + // If the prefix ends with a ']', we add an underscore to the end. + // This makes sure that we get names like "dev_user_endpoint" instead of "dev_userendpoint" + suffix := "" + if strings.HasSuffix(prefix, "]") { + suffix = "_" + } + + return textutil.NormalizeString(prefix) + suffix +} diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go new file mode 100644 index 000000000..35dac1f7d --- /dev/null +++ b/bundle/config/mutator/apply_presets_test.go @@ -0,0 +1,196 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestApplyPresetsPrefix(t *testing.T) { + tests := []struct { + name string + prefix string + job *resources.Job + want string + }{ + { + name: "add prefix to job", + prefix: "prefix-", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + }, + }, + want: "prefix-job1", + }, + { + name: "add empty prefix to job", + prefix: "", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + }, + }, + want: "job1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Jobs["job1"].Name) + }) + } +} + +func TestApplyPresetsTags(t *testing.T) { + tests := []struct { + name string + tags map[string]string + job *resources.Job + want map[string]string + }{ + { + name: "add tags to job", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: nil, + }, + }, + want: map[string]string{"env": "dev"}, + }, + { + name: "merge tags with existing job tags", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: map[string]string{"team": "data"}, + }, + }, + want: map[string]string{"env": "dev", "team": "data"}, + }, + { + name: "don't override existing job tags", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: map[string]string{"env": "prod"}, + }, + }, + want: map[string]string{"env": "prod"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + Tags: tt.tags, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + tags := b.Config.Resources.Jobs["job1"].Tags + require.Equal(t, tt.want, tags) + }) + } +} + +func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) { + tests := []struct { + name string + job *resources.Job + setting int + want int + }{ + { + name: "set max concurrent runs", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + MaxConcurrentRuns: 0, + }, + }, + setting: 5, + want: 5, + }, + { + name: "do not override existing max concurrent runs", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + MaxConcurrentRuns: 3, + }, + }, + setting: 5, + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + JobsMaxConcurrentRuns: tt.setting, + }, + }, + } + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) + }) + } +} diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index fb71f751b..92ed28689 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -2,17 +2,14 @@ package mutator import ( "context" - "path" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/databricks/databricks-sdk-go/service/ml" ) type processTargetMode struct{} @@ -30,103 +27,75 @@ func (m *processTargetMode) Name() string { // Mark all resources as being for 'development' purposes, i.e. // changing their their name, adding tags, and (in the future) // marking them as 'hidden' in the UI. -func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { if !b.Config.Bundle.Deployment.Lock.IsExplicitlyEnabled() { log.Infof(ctx, "Development mode: disabling deployment lock since bundle.deployment.lock.enabled is not set to true") disabled := false b.Config.Bundle.Deployment.Lock.Enabled = &disabled } - r := b.Config.Resources + t := &b.Config.Presets shortName := b.Config.Workspace.CurrentUser.ShortName - prefix := "[dev " + shortName + "] " - // Generate a normalized version of the short name that can be used as a tag value. - tagValue := b.Tagging.NormalizeValue(shortName) - - for i := range r.Jobs { - r.Jobs[i].Name = prefix + r.Jobs[i].Name - if r.Jobs[i].Tags == nil { - r.Jobs[i].Tags = make(map[string]string) - } - r.Jobs[i].Tags["dev"] = tagValue - if r.Jobs[i].MaxConcurrentRuns == 0 { - r.Jobs[i].MaxConcurrentRuns = developmentConcurrentRuns - } - - // Pause each job. As an exception, we don't pause jobs that are explicitly - // marked as "unpaused". This allows users to override the default behavior - // of the development mode. - if r.Jobs[i].Schedule != nil && r.Jobs[i].Schedule.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Schedule.PauseStatus = jobs.PauseStatusPaused - } - if r.Jobs[i].Continuous != nil && r.Jobs[i].Continuous.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Continuous.PauseStatus = jobs.PauseStatusPaused - } - if r.Jobs[i].Trigger != nil && r.Jobs[i].Trigger.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Trigger.PauseStatus = jobs.PauseStatusPaused - } + if t.NamePrefix == "" { + t.NamePrefix = "[dev " + shortName + "] " } - for i := range r.Pipelines { - r.Pipelines[i].Name = prefix + r.Pipelines[i].Name - r.Pipelines[i].Development = true - r.Pipelines[i].Continuous = false - // (pipelines don't yet support tags) + if t.Tags == nil { + t.Tags = map[string]string{} + } + _, exists := t.Tags["dev"] + if !exists { + t.Tags["dev"] = b.Tagging.NormalizeValue(shortName) } - for i := range r.Models { - r.Models[i].Name = prefix + r.Models[i].Name - r.Models[i].Tags = append(r.Models[i].Tags, ml.ModelTag{Key: "dev", Value: tagValue}) + if t.JobsMaxConcurrentRuns == 0 { + t.JobsMaxConcurrentRuns = developmentConcurrentRuns } - for i := range r.Experiments { - filepath := r.Experiments[i].Name - dir := path.Dir(filepath) - base := path.Base(filepath) - if dir == "." { - r.Experiments[i].Name = prefix + base - } else { - r.Experiments[i].Name = dir + "/" + prefix + base - } - r.Experiments[i].Tags = append(r.Experiments[i].Tags, ml.ExperimentTag{Key: "dev", Value: tagValue}) + if t.TriggerPauseStatus == "" { + t.TriggerPauseStatus = config.Paused } - for i := range r.ModelServingEndpoints { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.ModelServingEndpoints[i].Name = prefix + r.ModelServingEndpoints[i].Name - // (model serving doesn't yet support tags) + if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) { + enabled := true + t.PipelinesDevelopment = &enabled } - - for i := range r.RegisteredModels { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.RegisteredModels[i].Name = prefix + r.RegisteredModels[i].Name - // (registered models in Unity Catalog don't yet support tags) - } - - for i := range r.QualityMonitors { - // Remove all schedules from monitors, since they don't support pausing/unpausing. - // Quality monitors might support the "pause" property in the future, so at the - // CLI level we do respect that property if it is set to "unpaused". - if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { - r.QualityMonitors[i].Schedule = nil - } - } - - for i := range r.Schemas { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.Schemas[i].Name = prefix + r.Schemas[i].Name - // HTTP API for schemas doesn't yet support tags. It's only supported in - // the Databricks UI and via the SQL API. - } - - return nil } func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { + p := b.Config.Presets + u := b.Config.Workspace.CurrentUser + + // Make sure presets don't set the trigger status to UNPAUSED; + // this could be surprising since most users (and tools) expect triggers + // to be paused in development. + // (Note that there still is an exceptional case where users set the trigger + // status to UNPAUSED at the level of an individual object, whic hwas + // historically allowed.) + if p.TriggerPauseStatus == config.Unpaused { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default", + Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + }} + } + + // Make sure this development copy has unique names and paths to avoid conflicts if path := findNonUserPath(b); path != "" { return diag.Errorf("%s must start with '~/' or contain the current username when using 'mode: development'", path) } + if p.NamePrefix != "" && !strings.Contains(p.NamePrefix, u.ShortName) && !strings.Contains(p.NamePrefix, u.UserName) { + // Resources such as pipelines require a unique name, e.g. '[dev steve] my_pipeline'. + // For this reason we require the name prefix to contain the current username; + // it's a pitfall for users if they don't include it and later find out that + // only a single user can do development deployments. + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", + Locations: []dyn.Location{b.Config.GetLocation("presets.name_prefix")}, + }} + } return nil } @@ -183,10 +152,11 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di switch b.Config.Bundle.Mode { case config.Development: diags := validateDevelopmentMode(b) - if diags != nil { + if diags.HasError() { return diags } - return transformDevelopmentMode(ctx, b) + transformDevelopmentMode(ctx, b) + return diags case config.Production: isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName) return validateProductionMode(ctx, b, isPrincipal) diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 1a2e96fab..1c8671b4c 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -51,6 +52,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Schedule: &jobs.CronSchedule{ QuartzCronExpression: "* * * * *", }, + Tags: map[string]string{"existing": "tag"}, }, }, "job2": { @@ -129,12 +131,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { func TestProcessTargetModeDevelopment(t *testing.T) { b := mockBundle(config.Development) - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Job 1 assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name) + assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag") assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) @@ -183,7 +186,8 @@ func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place. @@ -197,7 +201,8 @@ func TestProcessTargetModeDevelopmentTagNormalizationForAzure(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place (Azure allows more characters than AWS). @@ -211,17 +216,53 @@ func TestProcessTargetModeDevelopmentTagNormalizationForGcp(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place. assert.Equal(t, "Hello_world", b.Config.Resources.Jobs["job1"].Tags["dev"]) } +func TestValidateDevelopmentMode(t *testing.T) { + // Test with a valid development mode bundle + b := mockBundle(config.Development) + diags := validateDevelopmentMode(b) + require.NoError(t, diags.Error()) + + // Test with a bundle that has a non-user path + b.Config.Workspace.RootPath = "/Shared/.bundle/x/y/state" + diags = validateDevelopmentMode(b) + require.ErrorContains(t, diags.Error(), "root_path") + + // Test with a bundle that has an unpaused trigger pause status + b = mockBundle(config.Development) + b.Config.Presets.TriggerPauseStatus = config.Unpaused + diags = validateDevelopmentMode(b) + require.ErrorContains(t, diags.Error(), "UNPAUSED") + + // Test with a bundle that has a prefix not containing the username or short name + b = mockBundle(config.Development) + b.Config.Presets.NamePrefix = "[prod]" + diags = validateDevelopmentMode(b) + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Contains(t, diags[0].Summary, "") + + // Test with a bundle that has valid user paths + b = mockBundle(config.Development) + b.Config.Workspace.RootPath = "/Users/lennart@company.com/.bundle/x/y/state" + b.Config.Workspace.StatePath = "/Users/lennart@company.com/.bundle/x/y/state" + b.Config.Workspace.FilePath = "/Users/lennart@company.com/.bundle/x/y/files" + b.Config.Workspace.ArtifactPath = "/Users/lennart@company.com/.bundle/x/y/artifacts" + diags = validateDevelopmentMode(b) + require.NoError(t, diags.Error()) +} + func TestProcessTargetModeDefault(t *testing.T) { b := mockBundle("") - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) @@ -307,7 +348,7 @@ func TestAllResourcesMocked(t *testing.T) { func TestAllResourcesRenamed(t *testing.T) { b := mockBundle(config.Development) - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) @@ -337,8 +378,7 @@ func TestDisableLocking(t *testing.T) { ctx := context.Background() b := mockBundle(config.Development) - err := bundle.Apply(ctx, b, ProcessTargetMode()) - require.Nil(t, err) + transformDevelopmentMode(ctx, b) assert.False(t, b.Config.Bundle.Deployment.Lock.IsEnabled()) } @@ -348,7 +388,97 @@ func TestDisableLockingDisabled(t *testing.T) { explicitlyEnabled := true b.Config.Bundle.Deployment.Lock.Enabled = &explicitlyEnabled - err := bundle.Apply(ctx, b, ProcessTargetMode()) - require.Nil(t, err) + transformDevelopmentMode(ctx, b) assert.True(t, b.Config.Bundle.Deployment.Lock.IsEnabled(), "Deployment lock should remain enabled in development mode when explicitly enabled") } + +func TestPrefixAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.NamePrefix = "custom_lennart_deploy_" + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "custom_lennart_deploy_job1", b.Config.Resources.Jobs["job1"].Name) +} + +func TestTagsAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = map[string]string{ + "custom": "tag", + "dev": "foo", + } + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["custom"]) + assert.Equal(t, "foo", b.Config.Resources.Jobs["job1"].Tags["dev"]) +} + +func TestTagsNil(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = nil + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) +} + +func TestTagsEmptySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = map[string]string{} + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) +} + +func TestJobsMaxConcurrentRunsAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.JobsMaxConcurrentRuns = 10 + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, 10, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) +} + +func TestJobsMaxConcurrentRunsDisabled(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.JobsMaxConcurrentRuns = 1 + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) +} + +func TestTriggerPauseStatusWhenUnpaused(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.TriggerPauseStatus = config.Unpaused + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.ErrorContains(t, diags.Error(), "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default") +} + +func TestPipelinesDevelopmentDisabled(t *testing.T) { + b := mockBundle(config.Development) + notEnabled := false + b.Config.Presets.PipelinesDevelopment = ¬Enabled + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) +} diff --git a/bundle/config/presets.go b/bundle/config/presets.go new file mode 100644 index 000000000..61009a252 --- /dev/null +++ b/bundle/config/presets.go @@ -0,0 +1,32 @@ +package config + +const Paused = "PAUSED" +const Unpaused = "UNPAUSED" + +type Presets struct { + // NamePrefix to prepend to all resource names. + NamePrefix string `json:"name_prefix,omitempty"` + + // PipelinesDevelopment is the default value for the development field of pipelines. + PipelinesDevelopment *bool `json:"pipelines_development,omitempty"` + + // TriggerPauseStatus is the default value for the pause status of all triggers and schedules. + // Either config.Paused, config.Unpaused, or empty. + TriggerPauseStatus string `json:"trigger_pause_status,omitempty"` + + // JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs. + JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"` + + // Tags to add to all resources. + Tags map[string]string `json:"tags,omitempty"` +} + +// IsExplicitlyEnabled tests whether this feature is explicitly enabled. +func IsExplicitlyEnabled(feature *bool) bool { + return feature != nil && *feature +} + +// IsExplicitlyDisabled tests whether this feature is explicitly disabled. +func IsExplicitlyDisabled(feature *bool) bool { + return feature != nil && !*feature +} diff --git a/bundle/config/root.go b/bundle/config/root.go index 2c6fe1a4a..86dc33921 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -60,6 +60,10 @@ type Root struct { // RunAs section allows to define an execution identity for jobs and pipelines runs RunAs *jobs.JobRunAs `json:"run_as,omitempty"` + // Presets applies preset transformations throughout the bundle, e.g. + // adding a name prefix to deployed resources. + Presets Presets `json:"presets,omitempty"` + Experimental *Experimental `json:"experimental,omitempty"` // Permissions section allows to define permissions which will be @@ -307,6 +311,7 @@ func (r *Root) MergeTargetOverrides(name string) error { "resources", "sync", "permissions", + "presets", } { if root, err = mergeField(root, target, f); err != nil { return err diff --git a/bundle/config/target.go b/bundle/config/target.go index acc493574..a2ef4d735 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -20,6 +20,10 @@ type Target struct { // development purposes. Mode Mode `json:"mode,omitempty"` + // Mutator configurations that e.g. change the + // name prefix of deployed resources. + Presets Presets `json:"presets,omitempty"` + // Overrides the compute used for jobs and other supported assets. ComputeID string `json:"compute_id,omitempty"` diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index fac3066dc..7a1081ded 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -47,6 +47,7 @@ func Initialize() bundle.Mutator { mutator.SetRunAs(), mutator.OverrideCompute(), mutator.ProcessTargetMode(), + mutator.ApplyPresets(), mutator.DefaultQueueing(), mutator.ExpandPipelineGlobPaths(), diff --git a/bundle/tests/presets/databricks.yml b/bundle/tests/presets/databricks.yml new file mode 100644 index 000000000..d83d31801 --- /dev/null +++ b/bundle/tests/presets/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: presets + +presets: + tags: + prod: true + team: finance + pipelines_development: true + +targets: + dev: + presets: + name_prefix: "myprefix" + pipelines_development: true + trigger_pause_status: PAUSED + jobs_max_concurrent_runs: 10 + tags: + dev: true + prod: false + prod: + presets: + pipelines_development: false diff --git a/bundle/tests/presets_test.go b/bundle/tests/presets_test.go new file mode 100644 index 000000000..5fcb5d95b --- /dev/null +++ b/bundle/tests/presets_test.go @@ -0,0 +1,28 @@ +package config_tests + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" +) + +func TestPresetsDev(t *testing.T) { + b := loadTarget(t, "./presets", "dev") + + assert.Equal(t, "myprefix", b.Config.Presets.NamePrefix) + assert.Equal(t, config.Paused, b.Config.Presets.TriggerPauseStatus) + assert.Equal(t, 10, b.Config.Presets.JobsMaxConcurrentRuns) + assert.Equal(t, true, *b.Config.Presets.PipelinesDevelopment) + assert.Equal(t, "true", b.Config.Presets.Tags["dev"]) + assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) + assert.Equal(t, "false", b.Config.Presets.Tags["prod"]) +} + +func TestPresetsProd(t *testing.T) { + b := loadTarget(t, "./presets", "prod") + + assert.Equal(t, false, *b.Config.Presets.PipelinesDevelopment) + assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) + assert.Equal(t, "true", b.Config.Presets.Tags["prod"]) +} From 242d4b51edd783341304aaa86184725102337268 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Tue, 20 Aug 2024 05:52:00 +0530 Subject: [PATCH 77/88] Report all empty resources present in error diagnostic (#1685) ## Changes This PR addressed post-merge feedback from https://github.com/databricks/cli/pull/1673. ## Tests Unit tests, and manually. ``` Error: experiment undefined-experiment is not defined at resources.experiments.undefined-experiment in databricks.yml:11:26 Error: job undefined-job is not defined at resources.jobs.undefined-job in databricks.yml:6:19 Error: pipeline undefined-pipeline is not defined at resources.pipelines.undefined-pipeline in databricks.yml:14:24 Name: undefined-job Target: default Found 3 errors ``` --- .../validate/all_resources_have_values.go | 42 ++++++++++------ .../environments_job_and_pipeline_test.go | 9 ---- bundle/tests/undefined_job/databricks.yml | 8 --- bundle/tests/undefined_job_test.go | 22 -------- .../tests/undefined_pipeline/databricks.yml | 8 --- .../tests/undefined_resources/databricks.yml | 14 ++++++ bundle/tests/undefined_resources_test.go | 50 +++++++++++++++++++ 7 files changed, 90 insertions(+), 63 deletions(-) delete mode 100644 bundle/tests/undefined_job/databricks.yml delete mode 100644 bundle/tests/undefined_job_test.go delete mode 100644 bundle/tests/undefined_pipeline/databricks.yml create mode 100644 bundle/tests/undefined_resources/databricks.yml create mode 100644 bundle/tests/undefined_resources_test.go diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 019fe48a2..7f96e529a 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -3,6 +3,7 @@ package validate import ( "context" "fmt" + "slices" "strings" "github.com/databricks/cli/bundle" @@ -21,27 +22,36 @@ func (m *allResourcesHaveValues) Name() string { } func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - rv := b.Config.Value().Get("resources") - - // Skip if there are no resources block defined, or the resources block is empty. - if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { - return nil - } + diags := diag.Diagnostics{} _, err := dyn.MapByPattern( - rv, - dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if v.Kind() == dyn.KindInvalid || v.Kind() == dyn.KindNil { - // Type of the resource, stripped of the trailing 's' to make it - // singular. - rType := strings.TrimSuffix(p[0].Key(), "s") - - rName := p[1].Key() - return v, fmt.Errorf("%s %s is not defined", rType, rName) + if v.Kind() != dyn.KindNil { + return v, nil } + + // Type of the resource, stripped of the trailing 's' to make it + // singular. + rType := strings.TrimSuffix(p[1].Key(), "s") + + // Name of the resource. Eg: "foo" in "jobs.foo". + rName := p[2].Key() + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s %s is not defined", rType, rName), + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + return v, nil }, ) - return diag.FromErr(err) + if err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + return diags } diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index 0abeb487c..218d2e470 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -15,8 +14,6 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -30,8 +27,6 @@ func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -44,16 +39,12 @@ func TestJobAndPipelineProductionWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - pl := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(pl.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - jl := b.Config.GetLocation("resources.jobs.pipeline_schedule") - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(jl.File)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/undefined_job/databricks.yml b/bundle/tests/undefined_job/databricks.yml deleted file mode 100644 index 12c19f946..000000000 --- a/bundle/tests/undefined_job/databricks.yml +++ /dev/null @@ -1,8 +0,0 @@ -bundle: - name: undefined-job - -resources: - jobs: - undefined: - test: - name: "Test Job" diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go deleted file mode 100644 index 4596f2069..000000000 --- a/bundle/tests/undefined_job_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package config_tests - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/validate" - "github.com/stretchr/testify/assert" -) - -func TestUndefinedJobLoadsWithError(t *testing.T) { - b := load(t, "./undefined_job") - diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) - assert.ErrorContains(t, diags.Error(), "job undefined is not defined") -} - -func TestUndefinedPipelineLoadsWithError(t *testing.T) { - b := load(t, "./undefined_pipeline") - diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) - assert.ErrorContains(t, diags.Error(), "pipeline undefined is not defined") -} diff --git a/bundle/tests/undefined_pipeline/databricks.yml b/bundle/tests/undefined_pipeline/databricks.yml deleted file mode 100644 index a52fda38c..000000000 --- a/bundle/tests/undefined_pipeline/databricks.yml +++ /dev/null @@ -1,8 +0,0 @@ -bundle: - name: undefined-pipeline - -resources: - pipelines: - undefined: - test: - name: "Test Pipeline" diff --git a/bundle/tests/undefined_resources/databricks.yml b/bundle/tests/undefined_resources/databricks.yml new file mode 100644 index 000000000..ffc0e46da --- /dev/null +++ b/bundle/tests/undefined_resources/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: undefined-job + +resources: + jobs: + undefined-job: + test: + name: "Test Job" + + experiments: + undefined-experiment: + + pipelines: + undefined-pipeline: diff --git a/bundle/tests/undefined_resources_test.go b/bundle/tests/undefined_resources_test.go new file mode 100644 index 000000000..3dbacbc25 --- /dev/null +++ b/bundle/tests/undefined_resources_test.go @@ -0,0 +1,50 @@ +package config_tests + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestUndefinedResourcesLoadWithError(t *testing.T) { + b := load(t, "./undefined_resources") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) + + assert.Len(t, diags, 3) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "job undefined-job is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 6, + Column: 19, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.undefined-job")}, + }) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "experiment undefined-experiment is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 11, + Column: 26, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.experiments.undefined-experiment")}, + }) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "pipeline undefined-pipeline is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 14, + Column: 24, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.undefined-pipeline")}, + }) +} From 6771ba09a699b3890316cf8f849b3a51733750e4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 20 Aug 2024 11:33:03 +0200 Subject: [PATCH 78/88] Correctly mark package names with versions as remote libraries (#1697) ## Changes Fixes https://github.com/databricks/setup-cli/issues/124 ## Tests Added regression test --- bundle/libraries/local_path.go | 5 +++++ bundle/libraries/local_path_test.go | 1 + 2 files changed, 6 insertions(+) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 5b5ec6c07..3e32adfde 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -66,6 +66,11 @@ func IsLibraryLocal(dep string) bool { } func isPackage(name string) bool { + // If the dependency has ==, it's a package with version + if strings.Contains(name, "==") { + return true + } + // If the dependency has no extension, it's a PyPi package name return path.Ext(name) == "" } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index be4028d52..7299cdc93 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -54,6 +54,7 @@ func TestIsLibraryLocal(t *testing.T) { {path: "-r /Workspace/my_project/requirements.txt", expected: false}, {path: "s3://mybucket/path/to/package", expected: false}, {path: "dbfs:/mnt/path/to/package", expected: false}, + {path: "beautifulsoup4==4.12.3", expected: false}, } for i, tc := range testCases { From af5048e73efab56dd2a13a02132e78d3ee84c5e7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 20 Aug 2024 14:54:56 +0200 Subject: [PATCH 79/88] Share test initializer in common helper function (#1695) ## Changes These tests inadvertently re-ran mutators, the first time through `loadTarget` and the second time by running `phases.Initialize()` themselves. Some of the mutators that are executed in `phases.Initialize()` are also run as part of `loadTarget`. This is overdue a refactor to make it unambiguous what runs when. Until then, this removes the duplicated execution. ## Tests Unit tests pass. --- bundle/tests/loader.go | 29 +++++++++++++++ bundle/tests/pipeline_glob_paths_test.go | 37 +------------------ .../tests/relative_path_translation_test.go | 29 +-------------- 3 files changed, 33 insertions(+), 62 deletions(-) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 069f09358..848132a13 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -8,6 +8,10 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -43,3 +47,28 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { )) return b, diags } + +func configureMock(t *testing.T, b *bundle.Bundle) { + // Configure mock workspace client + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &config.Config{ + Host: "https://mock.databricks.workspace.com", + } + m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ + UserName: "user@domain.com", + }, nil) + b.SetWorkpaceClient(m.WorkspaceClient) +} + +func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diagnostics) { + b := load(t, path) + configureMock(t, b) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + mutator.SelectTarget(env), + phases.Initialize(), + )) + + return b, diags +} diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go index bf5039b5f..c1c62cfb6 100644 --- a/bundle/tests/pipeline_glob_paths_test.go +++ b/bundle/tests/pipeline_glob_paths_test.go @@ -1,33 +1,13 @@ package config_tests import ( - "context" "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestExpandPipelineGlobPaths(t *testing.T) { - b := loadTarget(t, "./pipeline_glob_paths", "default") - - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - ctx := context.Background() - diags := bundle.Apply(ctx, b, phases.Initialize()) + b, diags := initializeTarget(t, "./pipeline_glob_paths", "default") require.NoError(t, diags.Error()) require.Equal( t, @@ -37,19 +17,6 @@ func TestExpandPipelineGlobPaths(t *testing.T) { } func TestExpandPipelineGlobPathsWithNonExistent(t *testing.T) { - b := loadTarget(t, "./pipeline_glob_paths", "error") - - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - ctx := context.Background() - diags := bundle.Apply(ctx, b, phases.Initialize()) + _, diags := initializeTarget(t, "./pipeline_glob_paths", "error") require.ErrorContains(t, diags.Error(), "notebook ./non-existent not found") } diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go index d5b80bea5..199871d23 100644 --- a/bundle/tests/relative_path_translation_test.go +++ b/bundle/tests/relative_path_translation_test.go @@ -1,36 +1,14 @@ package config_tests import ( - "context" "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func configureMock(t *testing.T, b *bundle.Bundle) { - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) -} - func TestRelativePathTranslationDefault(t *testing.T) { - b := loadTarget(t, "./relative_path_translation", "default") - configureMock(t, b) - - diags := bundle.Apply(context.Background(), b, phases.Initialize()) + b, diags := initializeTarget(t, "./relative_path_translation", "default") require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] @@ -40,10 +18,7 @@ func TestRelativePathTranslationDefault(t *testing.T) { } func TestRelativePathTranslationOverride(t *testing.T) { - b := loadTarget(t, "./relative_path_translation", "override") - configureMock(t, b) - - diags := bundle.Apply(context.Background(), b, phases.Initialize()) + b, diags := initializeTarget(t, "./relative_path_translation", "override") require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] From 44902fa3501033928a5ec46dbfcf4cb23f739788 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 20 Aug 2024 15:26:57 +0200 Subject: [PATCH 80/88] Make `pydabs/venv_path` optional (#1687) ## Changes Make `pydabs/venv_path` optional. When not specified, CLI detects the Python interpreter using `python.DetectExecutable`, the same way as for `artifacts`. `python.DetectExecutable` works correctly if a virtual environment is activated or `python3` is available on PATH through other means. Extract the venv detection code from PyDABs into `libs/python/detect`. This code will be used when we implement the `python/venv_path` section in `databricks.yml`. ## Tests Unit tests and manually --------- Co-authored-by: Pieter Noordhuis --- bundle/artifacts/whl/infer.go | 2 + bundle/config/experimental.go | 4 +- .../config/mutator/python/python_mutator.go | 33 ++++++------- .../mutator/python/python_mutator_test.go | 21 +++++++-- libs/python/detect.go | 46 +++++++++++++++++++ libs/python/detect_test.go | 46 +++++++++++++++++++ 6 files changed, 128 insertions(+), 24 deletions(-) create mode 100644 libs/python/detect_test.go diff --git a/bundle/artifacts/whl/infer.go b/bundle/artifacts/whl/infer.go index dd4ad2956..cb727de0e 100644 --- a/bundle/artifacts/whl/infer.go +++ b/bundle/artifacts/whl/infer.go @@ -15,6 +15,8 @@ type infer struct { func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { artifact := b.Config.Artifacts[m.name] + + // TODO use python.DetectVEnvExecutable once bundle has a way to specify venv path py, err := python.DetectExecutable(ctx) if err != nil { return diag.FromErr(err) diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 66e975820..061bbdae0 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -36,8 +36,8 @@ type PyDABs struct { // VEnvPath is path to the virtual environment. // - // Required if PyDABs is enabled. PyDABs will load the code in the specified - // environment. + // If enabled, PyDABs will execute code within this environment. If disabled, + // it defaults to using the Python interpreter available in the current shell. VEnvPath string `json:"venv_path,omitempty"` // Import contains a list Python packages with PyDABs code. diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index f9febe5b5..4f44df0a9 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -7,8 +7,8 @@ import ( "fmt" "os" "path/filepath" - "runtime" + "github.com/databricks/cli/libs/python" "github.com/databricks/databricks-sdk-go/logger" "github.com/databricks/cli/bundle/env" @@ -86,23 +86,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return nil } - if experimental.PyDABs.VEnvPath == "" { - return diag.Errorf("\"experimental.pydabs.enabled\" can only be used when \"experimental.pydabs.venv_path\" is set") - } - // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' var mutateDiags diag.Diagnostics var mutateDiagsHasError = errors.New("unexpected error") err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { - pythonPath := interpreterPath(experimental.PyDABs.VEnvPath) + pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath) - if _, err := os.Stat(pythonPath); err != nil { - if os.IsNotExist(err) { - return dyn.InvalidValue, fmt.Errorf("can't find %q, check if venv is created", pythonPath) - } else { - return dyn.InvalidValue, fmt.Errorf("can't find %q: %w", pythonPath, err) - } + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err) } cacheDir, err := createCacheDir(ctx) @@ -423,11 +415,16 @@ func isOmitemptyDelete(left dyn.Value) bool { } } -// interpreterPath returns platform-specific path to Python interpreter in the virtual environment. -func interpreterPath(venvPath string) string { - if runtime.GOOS == "windows" { - return filepath.Join(venvPath, "Scripts", "python3.exe") - } else { - return filepath.Join(venvPath, "bin", "python3") +// detectExecutable lookups Python interpreter in virtual environment, or if not set, in PATH. +func detectExecutable(ctx context.Context, venvPath string) (string, error) { + if venvPath == "" { + interpreter, err := python.DetectExecutable(ctx) + if err != nil { + return "", err + } + + return interpreter, nil } + + return python.DetectVEnvExecutable(venvPath) } diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index fbe835f92..ea02d1ced 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -282,7 +282,7 @@ func TestPythonMutator_venvRequired(t *testing.T) { } func TestPythonMutator_venvNotFound(t *testing.T) { - expectedError := fmt.Sprintf("can't find %q, check if venv is created", interpreterPath("bad_path")) + expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path")) b := loadYaml("databricks.yml", ` experimental: @@ -596,9 +596,7 @@ func loadYaml(name string, content string) *bundle.Bundle { } } -func withFakeVEnv(t *testing.T, path string) { - interpreterPath := interpreterPath(path) - +func withFakeVEnv(t *testing.T, venvPath string) { cwd, err := os.Getwd() if err != nil { panic(err) @@ -608,6 +606,8 @@ func withFakeVEnv(t *testing.T, path string) { panic(err) } + interpreterPath := interpreterPath(venvPath) + err = os.MkdirAll(filepath.Dir(interpreterPath), 0755) if err != nil { panic(err) @@ -618,9 +618,22 @@ func withFakeVEnv(t *testing.T, path string) { panic(err) } + err = os.WriteFile(filepath.Join(venvPath, "pyvenv.cfg"), []byte(""), 0755) + if err != nil { + panic(err) + } + t.Cleanup(func() { if err := os.Chdir(cwd); err != nil { panic(err) } }) } + +func interpreterPath(venvPath string) string { + if runtime.GOOS == "windows" { + return filepath.Join(venvPath, "Scripts", "python3.exe") + } else { + return filepath.Join(venvPath, "bin", "python3") + } +} diff --git a/libs/python/detect.go b/libs/python/detect.go index b0c1475c0..8fcc7cd9c 100644 --- a/libs/python/detect.go +++ b/libs/python/detect.go @@ -3,9 +3,23 @@ package python import ( "context" "errors" + "fmt" + "io/fs" + "os" "os/exec" + "path/filepath" + "runtime" ) +// DetectExecutable looks up the path to the python3 executable from the PATH +// environment variable. +// +// If virtualenv is activated, executable from the virtualenv is returned, +// because activating virtualenv adds python3 executable on a PATH. +// +// If python3 executable is not found on the PATH, the interpreter with the +// least version that satisfies minimal 3.8 version is returned, e.g. +// python3.10. func DetectExecutable(ctx context.Context) (string, error) { // TODO: add a shortcut if .python-version file is detected somewhere in // the parent directory tree. @@ -32,3 +46,35 @@ func DetectExecutable(ctx context.Context) (string, error) { } return interpreter.Path, nil } + +// DetectVEnvExecutable returns the path to the python3 executable inside venvPath, +// that is not necessarily activated. +// +// If virtualenv is not created, or executable doesn't exist, the error is returned. +func DetectVEnvExecutable(venvPath string) (string, error) { + interpreterPath := filepath.Join(venvPath, "bin", "python3") + if runtime.GOOS == "windows" { + interpreterPath = filepath.Join(venvPath, "Scripts", "python3.exe") + } + + if _, err := os.Stat(interpreterPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("can't find %q, check if virtualenv is created", interpreterPath) + } else { + return "", fmt.Errorf("can't find %q: %w", interpreterPath, err) + } + } + + // pyvenv.cfg must be always present in correctly configured virtualenv, + // read more in https://snarky.ca/how-virtual-environments-work/ + pyvenvPath := filepath.Join(venvPath, "pyvenv.cfg") + if _, err := os.Stat(pyvenvPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("expected %q to be virtualenv, but pyvenv.cfg is missing", venvPath) + } else { + return "", fmt.Errorf("can't find %q: %w", pyvenvPath, err) + } + } + + return interpreterPath, nil +} diff --git a/libs/python/detect_test.go b/libs/python/detect_test.go new file mode 100644 index 000000000..78c7067f7 --- /dev/null +++ b/libs/python/detect_test.go @@ -0,0 +1,46 @@ +package python + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetectVEnvExecutable(t *testing.T) { + dir := t.TempDir() + interpreterPath := interpreterPath(dir) + + err := os.Mkdir(filepath.Dir(interpreterPath), 0755) + require.NoError(t, err) + + err = os.WriteFile(interpreterPath, []byte(""), 0755) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(dir, "pyvenv.cfg"), []byte(""), 0755) + require.NoError(t, err) + + executable, err := DetectVEnvExecutable(dir) + + assert.NoError(t, err) + assert.Equal(t, interpreterPath, executable) +} + +func TestDetectVEnvExecutable_badLayout(t *testing.T) { + dir := t.TempDir() + + _, err := DetectVEnvExecutable(dir) + + assert.Errorf(t, err, "can't find %q, check if virtualenv is created", interpreterPath(dir)) +} + +func interpreterPath(venvPath string) string { + if runtime.GOOS == "windows" { + return filepath.Join(venvPath, "Scripts", "python3.exe") + } else { + return filepath.Join(venvPath, "bin", "python3") + } +} From a4c1ba3e2827abca29034115436d441310b7ee33 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:15:25 +0530 Subject: [PATCH 81/88] Use API mocks for duplicate path errors in workspace files extensions client (#1690) ## Changes `TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName` recently started failing in our nightlies because the upstream `import` API was changed to [prohibit conflicting file paths](https://docs.databricks.com/en/release-notes/product/2024/august.html#files-can-no-longer-have-identical-names-in-workspace-folders). Because existing conflicting file paths can still be grandfathered in, we need to retain coverage for the test. To do this, this PR: 1. Removes the failing `TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName` 2. Add an equivalent unit test with the `list` and `get-status` API calls mocked. --- internal/filer_test.go | 62 ------- libs/filer/workspace_files_client.go | 26 +-- .../workspace_files_extensions_client.go | 8 +- .../workspace_files_extensions_client_test.go | 151 ++++++++++++++++++ 4 files changed, 172 insertions(+), 75 deletions(-) create mode 100644 libs/filer/workspace_files_extensions_client_test.go diff --git a/internal/filer_test.go b/internal/filer_test.go index 275304256..bc4c94808 100644 --- a/internal/filer_test.go +++ b/internal/filer_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "io/fs" "path" @@ -722,67 +721,6 @@ func TestAccFilerWorkspaceFilesExtensionsStat(t *testing.T) { assert.ErrorIs(t, err, fs.ErrNotExist) } -func TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) { - t.Parallel() - - tcases := []struct { - files []struct{ name, content string } - name string - }{ - { - name: "python", - files: []struct{ name, content string }{ - {"foo.py", "print('foo')"}, - {"foo.py", "# Databricks notebook source\nprint('foo')"}, - }, - }, - { - name: "r", - files: []struct{ name, content string }{ - {"foo.r", "print('foo')"}, - {"foo.r", "# Databricks notebook source\nprint('foo')"}, - }, - }, - { - name: "sql", - files: []struct{ name, content string }{ - {"foo.sql", "SELECT 'foo'"}, - {"foo.sql", "-- Databricks notebook source\nSELECT 'foo'"}, - }, - }, - { - name: "scala", - files: []struct{ name, content string }{ - {"foo.scala", "println('foo')"}, - {"foo.scala", "// Databricks notebook source\nprintln('foo')"}, - }, - }, - // We don't need to test this for ipynb notebooks. The import API - // fails when the file extension is .ipynb but the content is not a - // valid juptyer notebook. - } - - for i := range tcases { - tc := tcases[i] - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - ctx := context.Background() - wf, tmpDir := setupWsfsExtensionsFiler(t) - - for _, f := range tc.files { - err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories) - require.NoError(t, err) - } - - _, err := wf.ReadDir(ctx, ".") - assert.ErrorAs(t, err, &filer.DuplicatePathError{}) - assert.ErrorContains(t, err, fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at %s and FILE at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", path.Join(tmpDir, "foo"), path.Join(tmpDir, tc.files[0].name), tc.files[0].name)) - }) - } - -} - func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) { t.Parallel() diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index e911f4409..d8ab5a6bb 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -102,13 +102,21 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) { return marshal.Marshal(info) } +// Interface for *client.DatabricksClient from the Databricks Go SDK. Abstracted +// as an interface to allow for mocking in tests. +type apiClient interface { + Do(ctx context.Context, method, path string, + headers map[string]string, request, response any, + visitors ...func(*http.Request) error) error +} + // WorkspaceFilesClient implements the files-in-workspace API. // NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled. // It can access any workspace path if files-in-workspace is enabled. -type WorkspaceFilesClient struct { +type workspaceFilesClient struct { workspaceClient *databricks.WorkspaceClient - apiClient *client.DatabricksClient + apiClient apiClient // File operations will be relative to this path. root WorkspaceRootPath @@ -120,7 +128,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, return nil, err } - return &WorkspaceFilesClient{ + return &workspaceFilesClient{ workspaceClient: w, apiClient: apiClient, @@ -128,7 +136,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, }, nil } -func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { +func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { absPath, err := w.root.Join(name) if err != nil { return err @@ -198,7 +206,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } -func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { +func (w *workspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err @@ -222,7 +230,7 @@ func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCl return w.workspaceClient.Workspace.Download(ctx, absPath) } -func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { +func (w *workspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { absPath, err := w.root.Join(name) if err != nil { return err @@ -266,7 +274,7 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ... return err } -func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { +func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err @@ -299,7 +307,7 @@ func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D return wsfsDirEntriesFromObjectInfos(objects), nil } -func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { +func (w *workspaceFilesClient) Mkdir(ctx context.Context, name string) error { dirPath, err := w.root.Join(name) if err != nil { return err @@ -309,7 +317,7 @@ func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { }) } -func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { +func (w *workspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go index 844e736b5..b24ecf7ee 100644 --- a/libs/filer/workspace_files_extensions_client.go +++ b/libs/filer/workspace_files_extensions_client.go @@ -133,14 +133,14 @@ func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx con }, nil } -type DuplicatePathError struct { +type duplicatePathError struct { oi1 workspace.ObjectInfo oi2 workspace.ObjectInfo commonName string } -func (e DuplicatePathError) Error() string { +func (e duplicatePathError) Error() string { return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName) } @@ -157,7 +157,7 @@ func (e ReadOnlyError) Error() string { // delete, and stat notebooks (and files in general) in the workspace, using their paths // with the extension included. // -// The ReadDir method returns a DuplicatePathError if this traditional file system view is +// The ReadDir method returns a duplicatePathError if this traditional file system view is // not possible. For example, a Python notebook called foo and a Python file called `foo.py` // would resolve to the same path `foo.py` in a tradition file system. // @@ -220,7 +220,7 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin // Error if we have seen this path before in the current directory. // If not seen before, add it to the seen paths. if _, ok := seenPaths[entries[i].Name()]; ok { - return nil, DuplicatePathError{ + return nil, duplicatePathError{ oi1: seenPaths[entries[i].Name()], oi2: sysInfo, commonName: path.Join(name, entries[i].Name()), diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go new file mode 100644 index 000000000..321c43712 --- /dev/null +++ b/libs/filer/workspace_files_extensions_client_test.go @@ -0,0 +1,151 @@ +package filer + +import ( + "context" + "net/http" + "testing" + + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// Mocks client.DatabricksClient from the databricks-sdk-go package. +type mockApiClient struct { + mock.Mock +} + +func (m *mockApiClient) Do(ctx context.Context, method, path string, + headers map[string]string, request any, response any, + visitors ...func(*http.Request) error) error { + args := m.Called(ctx, method, path, headers, request, response, visitors) + + // Set the http response from a value provided in the mock call. + p := response.(*wsfsFileInfo) + *p = args.Get(1).(wsfsFileInfo) + return args.Error(0) +} + +func TestFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) { + for _, tc := range []struct { + name string + language workspace.Language + notebookExportFormat workspace.ExportFormat + notebookPath string + filePath string + expectedError string + }{ + { + name: "python source notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.py", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.py resolve to the same name /foo.py. Changing the name of one of these objects will resolve this issue", + }, + { + name: "python jupyter notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatJupyter, + notebookPath: "/dir/foo", + filePath: "/dir/foo.py", + // Jupyter notebooks would correspond to foo.ipynb so an error is not expected. + expectedError: "", + }, + { + name: "scala source notebook and file", + language: workspace.LanguageScala, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.scala", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.scala resolve to the same name /foo.scala. Changing the name of one of these objects will resolve this issue", + }, + { + name: "r source notebook and file", + language: workspace.LanguageR, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.r", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.r resolve to the same name /foo.r. Changing the name of one of these objects will resolve this issue", + }, + { + name: "sql source notebook and file", + language: workspace.LanguageSql, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.sql", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.sql resolve to the same name /foo.sql. Changing the name of one of these objects will resolve this issue", + }, + { + name: "python jupyter notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatJupyter, + notebookPath: "/dir/foo", + filePath: "/dir/foo.ipynb", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.ipynb resolve to the same name /foo.ipynb. Changing the name of one of these objects will resolve this issue", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mockedWorkspaceClient := mocks.NewMockWorkspaceClient(t) + mockedApiClient := mockApiClient{} + + // Mock the workspace API's ListAll method. + workspaceApi := mockedWorkspaceClient.GetMockWorkspaceAPI() + workspaceApi.EXPECT().ListAll(mock.Anything, workspace.ListWorkspaceRequest{ + Path: "/dir", + }).Return([]workspace.ObjectInfo{ + { + Path: tc.filePath, + Language: tc.language, + ObjectType: workspace.ObjectTypeFile, + }, + { + Path: tc.notebookPath, + Language: tc.language, + ObjectType: workspace.ObjectTypeNotebook, + }, + }, nil) + + // Mock bespoke API calls to /api/2.0/workspace/get-status, that are + // used to figure out the right file extension for the notebook. + statNotebook := wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: tc.notebookPath, + Language: tc.language, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: tc.notebookExportFormat, + } + + mockedApiClient.On("Do", mock.Anything, http.MethodGet, "/api/2.0/workspace/get-status", map[string]string(nil), map[string]string{ + "path": tc.notebookPath, + "return_export_info": "true", + }, mock.AnythingOfType("*filer.wsfsFileInfo"), []func(*http.Request) error(nil)).Return(nil, statNotebook) + + workspaceFilesClient := workspaceFilesClient{ + workspaceClient: mockedWorkspaceClient.WorkspaceClient, + apiClient: &mockedApiClient, + root: NewWorkspaceRootPath("/dir"), + } + + workspaceFilesExtensionsClient := workspaceFilesExtensionsClient{ + workspaceClient: mockedWorkspaceClient.WorkspaceClient, + wsfs: &workspaceFilesClient, + } + + _, err := workspaceFilesExtensionsClient.ReadDir(context.Background(), "/") + + if tc.expectedError == "" { + assert.NoError(t, err) + } else { + assert.ErrorAs(t, err, &duplicatePathError{}) + assert.EqualError(t, err, tc.expectedError) + } + + // assert the mocked methods were actually called, as a sanity check. + workspaceApi.AssertNumberOfCalls(t, "ListAll", 1) + mockedApiClient.AssertNumberOfCalls(t, "Do", 1) + }) + } +} From c775d251eda6fa567de95e55e4558d5b99abce39 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 21 Aug 2024 10:22:35 +0200 Subject: [PATCH 82/88] Improves detection of PyPI package names in environment dependencies (#1699) ## Changes Improves detection of PyPi package names in environment dependencies ## Tests Added unit tests --- bundle/libraries/local_path.go | 22 ++++++++++++++++++---- bundle/libraries/local_path_test.go | 9 +++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 3e32adfde..417bce10e 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -3,6 +3,7 @@ package libraries import ( "net/url" "path" + "regexp" "strings" ) @@ -65,14 +66,27 @@ func IsLibraryLocal(dep string) bool { return IsLocalPath(dep) } +// ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_). +// \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security]. +// ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?)?: Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1). +// Spec for package name and version specifier: https://pip.pypa.io/en/stable/reference/requirement-specifiers/ +var packageRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+\s?(\[.*\])?\s?((==|!=|<=|>=|~=|==|>|<)\s?\d+(\.\d+){0,2}(\.\*)?)?$`) + func isPackage(name string) bool { - // If the dependency has ==, it's a package with version - if strings.Contains(name, "==") { + if packageRegex.MatchString(name) { return true } - // If the dependency has no extension, it's a PyPi package name - return path.Ext(name) == "" + return isUrlBasedLookup(name) +} + +func isUrlBasedLookup(name string) bool { + parts := strings.Split(name, " @ ") + if len(parts) != 2 { + return false + } + + return packageRegex.MatchString(parts[0]) && isRemoteStorageScheme(parts[1]) } func isRemoteStorageScheme(path string) bool { diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index 7299cdc93..7f84b3244 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -54,7 +54,16 @@ func TestIsLibraryLocal(t *testing.T) { {path: "-r /Workspace/my_project/requirements.txt", expected: false}, {path: "s3://mybucket/path/to/package", expected: false}, {path: "dbfs:/mnt/path/to/package", expected: false}, + {path: "beautifulsoup4", expected: false}, {path: "beautifulsoup4==4.12.3", expected: false}, + {path: "beautifulsoup4 >= 4.12.3", expected: false}, + {path: "beautifulsoup4 < 4.12.3", expected: false}, + {path: "beautifulsoup4 ~= 4.12.3", expected: false}, + {path: "beautifulsoup4[security, tests]", expected: false}, + {path: "beautifulsoup4[security, tests] ~= 4.12.3", expected: false}, + {path: "https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, + {path: "pip @ https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, + {path: "requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip", expected: false}, } for i, tc := range testCases { From 192f33bb13a156bebf9d7d2c2b06092d8ae9775d Mon Sep 17 00:00:00 2001 From: Witold Czaplewski Date: Wed, 21 Aug 2024 12:03:56 +0200 Subject: [PATCH 83/88] [DAB] Add support for requirements libraries in Job Tasks (#1543) ## Changes While experimenting with DAB I discovered that requirements libraries are being ignored. One thing worth mentioning is that `bundle validate` runs successfully, but `bundle deploy` fails. This PR only covers the second part. ## Tests Added a unit test --- bundle/config/mutator/translate_paths_jobs.go | 5 +++++ bundle/config/mutator/translate_paths_test.go | 9 +++++++++ bundle/libraries/helpers.go | 3 +++ bundle/libraries/helpers_test.go | 1 + 4 files changed, 18 insertions(+) diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 6febf4f8f..e34eeb2f0 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern t.translateNoOp, noSkipRewrite, }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + t.translateFilePath, + noSkipRewrite, + }, } } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 780a540df..fd64593be 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -110,6 +110,7 @@ func TestTranslatePaths(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py")) touchEmptyFile(t, filepath.Join(dir, "my_python_file.py")) touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) b := &bundle.Bundle{ RootPath: dir, @@ -140,6 +141,9 @@ func TestTranslatePaths(t *testing.T) { NotebookTask: &jobs.NotebookTask{ NotebookPath: "./my_job_notebook.py", }, + Libraries: []compute.Library{ + {Requirements: "./requirements.txt"}, + }, }, { PythonWheelTask: &jobs.PythonWheelTask{ @@ -232,6 +236,11 @@ func TestTranslatePaths(t *testing.T) { "/bundle/my_job_notebook", b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath, ) + assert.Equal( + t, + "/bundle/requirements.txt", + b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements, + ) assert.Equal( t, "/bundle/my_python_file.py", diff --git a/bundle/libraries/helpers.go b/bundle/libraries/helpers.go index 89679c91a..b7e707ccf 100644 --- a/bundle/libraries/helpers.go +++ b/bundle/libraries/helpers.go @@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string { if library.Egg != "" { return library.Egg } + if library.Requirements != "" { + return library.Requirements + } return "" } diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go index adc20a246..e4bd32770 100644 --- a/bundle/libraries/helpers_test.go +++ b/bundle/libraries/helpers_test.go @@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) { assert.Equal(t, path, libraryPath(&compute.Library{Whl: path})) assert.Equal(t, path, libraryPath(&compute.Library{Jar: path})) assert.Equal(t, path, libraryPath(&compute.Library{Egg: path})) + assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path})) assert.Equal(t, "", libraryPath(&compute.Library{})) } From f5df211320a5fad876c58737d959a0a034040c63 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 21 Aug 2024 18:23:54 +0530 Subject: [PATCH 84/88] Fix prefix preset used for UC schemas (#1704) ## Changes In https://github.com/databricks/cli/pull/1490 we regressed and started using the development mode prefix for UC schemas regardless of the mode of the bundle target. This PR fixes the regression and adds a regression test ## Tests Failing integration tests pass now. --- bundle/config/mutator/apply_presets.go | 3 +- bundle/config/mutator/apply_presets_test.go | 57 +++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 42e6ab95f..28d015c10 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -155,8 +155,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Schemas: Prefix for i := range r.Schemas { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.Schemas[i].Name = prefix + r.Schemas[i].Name + r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name // HTTP API for schemas doesn't yet support tags. It's only supported in // the Databricks UI and via the SQL API. } diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index 35dac1f7d..ab2478aee 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -68,6 +69,62 @@ func TestApplyPresetsPrefix(t *testing.T) { } } +func TestApplyPresetsPrefixForUcSchema(t *testing.T) { + tests := []struct { + name string + prefix string + schema *resources.Schema + want string + }{ + { + name: "add prefix to schema", + prefix: "[prefix]", + schema: &resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "schema1", + }, + }, + want: "prefix_schema1", + }, + { + name: "add empty prefix to schema", + prefix: "", + schema: &resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "schema1", + }, + }, + want: "schema1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "schema1": tt.schema, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Schemas["schema1"].Name) + }) + } +} + func TestApplyPresetsTags(t *testing.T) { tests := []struct { name string From 6f345293b1e5f4febcd702da8a362b15b606ebd9 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 21 Aug 2024 17:05:49 +0200 Subject: [PATCH 85/88] Added filtering flags for cluster list commands (#1703) ## Changes Fixes #1701 ## Tests ``` Usage: databricks clusters list [flags] Flags: --cluster-sources []string Filter clusters by source --cluster-states []string Filter clusters by states -h, --help help for list --is-pinned Filter clusters by pinned status --page-size int Use this field to specify the maximum number of results to be returned by the server. --page-token string Use next_page_token or prev_page_token returned from the previous request to list the next or previous page of clusters respectively. --policy-id string Filter clusters by policy id ``` --- cmd/workspace/clusters/overrides.go | 68 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/cmd/workspace/clusters/overrides.go b/cmd/workspace/clusters/overrides.go index 55976d406..6038978ae 100644 --- a/cmd/workspace/clusters/overrides.go +++ b/cmd/workspace/clusters/overrides.go @@ -1,17 +1,83 @@ package clusters import ( + "strings" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/spf13/cobra" ) -func listOverride(listCmd *cobra.Command, _ *compute.ListClustersRequest) { +// Below we add overrides for filter flags for cluster list command to allow for custom filtering +// Auto generating such flags is not yet supported by the CLI generator +func listOverride(listCmd *cobra.Command, listReq *compute.ListClustersRequest) { listCmd.Annotations["headerTemplate"] = cmdio.Heredoc(` {{header "ID"}} {{header "Name"}} {{header "State"}}`) listCmd.Annotations["template"] = cmdio.Heredoc(` {{range .}}{{.ClusterId | green}} {{.ClusterName | cyan}} {{if eq .State "RUNNING"}}{{green "%s" .State}}{{else if eq .State "TERMINATED"}}{{red "%s" .State}}{{else}}{{blue "%s" .State}}{{end}} {{end}}`) + + listReq.FilterBy = &compute.ListClustersFilterBy{} + listCmd.Flags().BoolVar(&listReq.FilterBy.IsPinned, "is-pinned", false, "Filter clusters by pinned status") + listCmd.Flags().StringVar(&listReq.FilterBy.PolicyId, "policy-id", "", "Filter clusters by policy id") + + sources := &clusterSources{source: &listReq.FilterBy.ClusterSources} + listCmd.Flags().Var(sources, "cluster-sources", "Filter clusters by source") + + states := &clusterStates{state: &listReq.FilterBy.ClusterStates} + listCmd.Flags().Var(states, "cluster-states", "Filter clusters by states") +} + +type clusterSources struct { + source *[]compute.ClusterSource +} + +func (c *clusterSources) String() string { + s := make([]string, len(*c.source)) + for i, source := range *c.source { + s[i] = string(source) + } + + return strings.Join(s, ",") +} + +func (c *clusterSources) Set(value string) error { + splits := strings.Split(value, ",") + for _, split := range splits { + *c.source = append(*c.source, compute.ClusterSource(split)) + } + + return nil +} + +func (c *clusterSources) Type() string { + return "[]string" +} + +type clusterStates struct { + state *[]compute.State +} + +func (c *clusterStates) String() string { + s := make([]string, len(*c.state)) + for i, source := range *c.state { + s[i] = string(source) + } + + return strings.Join(s, ",") +} + +func (c *clusterStates) Set(value string) error { + splits := strings.Split(value, ",") + for _, split := range splits { + *c.state = append(*c.state, compute.State(split)) + } + + return nil +} + +func (c *clusterStates) Type() string { + return "[]string" } func listNodeTypesOverride(listNodeTypesCmd *cobra.Command) { From 6e8cd835a3f699ffec0c04e9301e3a49fd61fc9c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 21 Aug 2024 17:33:25 +0200 Subject: [PATCH 86/88] Add paths field to bundle sync configuration (#1694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This field allows a user to configure paths to synchronize to the workspace. Allowed values are relative paths to files and directories anchored at the directory where the field is set. If one or more values traverse up the directory tree (to an ancestor of the bundle root directory), the CLI will dynamically determine the root path to use to ensure that the file tree structure remains intact. For example, given a `databricks.yml` in `my_bundle` that includes: ```yaml sync: paths: - ../common - . ``` Then upon synchronization, the workspace will look like: ``` . ├── common │ └── lib.py └── my_bundle ├── databricks.yml └── notebook.py ``` If not set behavior remains identical. ## Tests * Newly added unit tests for the mutators and under `bundle/tests`. * Manually confirmed a bundle without this configuration works the same. * Manually confirmed a bundle with this configuration works. --- bundle/bundle.go | 8 + bundle/bundle_read_only.go | 4 + bundle/config/mutator/configure_wsfs.go | 4 +- bundle/config/mutator/rewrite_sync_paths.go | 4 + .../config/mutator/rewrite_sync_paths_test.go | 16 ++ bundle/config/mutator/sync_default_path.go | 48 +++++ .../config/mutator/sync_default_path_test.go | 82 ++++++++ bundle/config/mutator/sync_infer_root.go | 120 +++++++++++ .../mutator/sync_infer_root_internal_test.go | 72 +++++++ bundle/config/mutator/sync_infer_root_test.go | 198 ++++++++++++++++++ bundle/config/mutator/trampoline.go | 2 +- bundle/config/mutator/trampoline_test.go | 8 +- bundle/config/mutator/translate_paths.go | 12 +- bundle/config/mutator/translate_paths_test.go | 60 +++--- bundle/config/sync.go | 4 + bundle/deploy/files/sync.go | 4 +- bundle/deploy/state_pull.go | 2 +- bundle/deploy/state_pull_test.go | 8 +- bundle/phases/initialize.go | 11 + bundle/python/conditional_transform_test.go | 22 +- bundle/tests/loader.go | 2 + bundle/tests/sync/paths/databricks.yml | 20 ++ .../tests/sync/paths_no_root/databricks.yml | 26 +++ .../sync/shared_code/bundle/databricks.yml | 10 + .../tests/sync/shared_code/common/library.txt | 1 + bundle/tests/sync_test.go | 65 ++++++ cmd/sync/sync_test.go | 6 +- 27 files changed, 760 insertions(+), 59 deletions(-) create mode 100644 bundle/config/mutator/sync_default_path.go create mode 100644 bundle/config/mutator/sync_default_path_test.go create mode 100644 bundle/config/mutator/sync_infer_root.go create mode 100644 bundle/config/mutator/sync_infer_root_internal_test.go create mode 100644 bundle/config/mutator/sync_infer_root_test.go create mode 100644 bundle/tests/sync/paths/databricks.yml create mode 100644 bundle/tests/sync/paths_no_root/databricks.yml create mode 100644 bundle/tests/sync/shared_code/bundle/databricks.yml create mode 100644 bundle/tests/sync/shared_code/common/library.txt diff --git a/bundle/bundle.go b/bundle/bundle.go index 032d98abc..8b5ff976d 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -39,6 +39,14 @@ type Bundle struct { // Exclusively use this field for filesystem operations. BundleRoot vfs.Path + // SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace. + // It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot]. + SyncRoot vfs.Path + + // SyncRootPath is the local path to the root directory of files that are synchronized to the workspace. + // It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access. + SyncRootPath string + Config config.Root // Metadata about the bundle deployment. This is the interface Databricks services diff --git a/bundle/bundle_read_only.go b/bundle/bundle_read_only.go index 59084f2ac..74b9d94de 100644 --- a/bundle/bundle_read_only.go +++ b/bundle/bundle_read_only.go @@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path { return r.b.BundleRoot } +func (r ReadOnlyBundle) SyncRoot() vfs.Path { + return r.b.SyncRoot +} + func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient { return r.b.WorkspaceClient() } diff --git a/bundle/config/mutator/configure_wsfs.go b/bundle/config/mutator/configure_wsfs.go index c7b764f00..1d1bec582 100644 --- a/bundle/config/mutator/configure_wsfs.go +++ b/bundle/config/mutator/configure_wsfs.go @@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string { } func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - root := b.BundleRoot.Native() + root := b.SyncRoot.Native() // The bundle root must be located in /Workspace/ if !strings.HasPrefix(root, "/Workspace/") { @@ -45,6 +45,6 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diag.FromErr(err) } - b.BundleRoot = p + b.SyncRoot = p return nil } diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index cfdc55f36..888714abe 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -45,6 +45,10 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { + v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + if err != nil { + return dyn.InvalidValue, err + } v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath))) if err != nil { return dyn.InvalidValue, err diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index 56ada19e6..fa7f124b7 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) { RootPath: ".", Config: config.Root{ Sync: config.Sync{ + Paths: []string{ + ".", + "../common", + }, Include: []string{ "foo", "bar", @@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) { }, } + bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml") + bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml") bundletest.SetLocation(b, "sync.include[0]", "./file.yml") bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml") bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml") @@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1]) assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0]) assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1]) assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0]) @@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { RootPath: "/tmp/dir", Config: config.Root{ Sync: config.Sync{ + Paths: []string{ + ".", + "../common", + }, Include: []string{ "foo", "bar", @@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { }, } + bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml") + bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml") bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml") bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml") bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml") @@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1]) assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0]) assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1]) assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0]) diff --git a/bundle/config/mutator/sync_default_path.go b/bundle/config/mutator/sync_default_path.go new file mode 100644 index 000000000..8e14ce202 --- /dev/null +++ b/bundle/config/mutator/sync_default_path.go @@ -0,0 +1,48 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type syncDefaultPath struct{} + +// SyncDefaultPath configures the default sync path to be equal to the bundle root. +func SyncDefaultPath() bundle.Mutator { + return &syncDefaultPath{} +} + +func (m *syncDefaultPath) Name() string { + return "SyncDefaultPath" +} + +func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + isset := false + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + pv, _ := dyn.Get(v, "sync.paths") + + // If the sync paths field is already set, do nothing. + // We know it is set if its value is either a nil or a sequence (empty or not). + switch pv.Kind() { + case dyn.KindNil, dyn.KindSequence: + isset = true + } + + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + // If the sync paths field is already set, do nothing. + if isset { + return nil + } + + // Set the sync paths to the default value. + b.Config.Sync.Paths = []string{"."} + return nil +} diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go new file mode 100644 index 000000000..a37e913d2 --- /dev/null +++ b/bundle/config/mutator/sync_default_path_test.go @@ -0,0 +1,82 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{}, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) +} + +func TestSyncDefaultPath_SkipIfSet(t *testing.T) { + tcases := []struct { + name string + paths dyn.Value + expect []string + }{ + { + name: "nil", + paths: dyn.V(nil), + expect: nil, + }, + { + name: "empty sequence", + paths: dyn.V([]dyn.Value{}), + expect: []string{}, + }, + { + name: "non-empty sequence", + paths: dyn.V([]dyn.Value{dyn.V("something")}), + expect: []string{"something"}, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{}, + } + + diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping())) + if err != nil { + return dyn.InvalidValue, err + } + v, err = dyn.Set(v, "sync.paths", tcase.paths) + if err != nil { + return dyn.InvalidValue, err + } + return v, nil + }) + return diag.FromErr(err) + }) + require.NoError(t, diags.Error()) + + ctx := context.Background() + diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath()) + require.NoError(t, diags.Error()) + + // If the sync paths field is already set, do nothing. + assert.Equal(t, tcase.expect, b.Config.Sync.Paths) + }) + } +} diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go new file mode 100644 index 000000000..012acf800 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root.go @@ -0,0 +1,120 @@ +package mutator + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" +) + +type syncInferRoot struct{} + +// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the +// paths in the sync configuration. The sync root may be different from the bundle root +// when the user intends to synchronize files outside the bundle root. +// +// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant. +// That is, the sync root must contain the bundle root. +// +// This mutator requires all sync-related paths and patterns to be relative to the bundle root path. +// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator. +func SyncInferRoot() bundle.Mutator { + return &syncInferRoot{} +} + +func (m *syncInferRoot) Name() string { + return "SyncInferRoot" +} + +// computeRoot finds the innermost path that contains the specified path. +// It traverses up the root path until it finds the innermost path. +// If the path does not exist, it returns an empty string. +// +// See "sync_infer_root_internal_test.go" for examples. +func (m *syncInferRoot) computeRoot(path string, root string) string { + for !filepath.IsLocal(path) { + // Break if we have reached the root of the filesystem. + dir := filepath.Dir(root) + if dir == root { + return "" + } + + // Update the sync path as we navigate up the directory tree. + path = filepath.Join(filepath.Base(root), path) + + // Move up the directory tree. + root = dir + } + + return filepath.Clean(root) +} + +func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Use the bundle root path as the starting point for inferring the sync root path. + bundleRootPath := filepath.Clean(b.RootPath) + + // Infer the sync root path by looking at each one of the sync paths. + // Every sync path must be a descendant of the final sync root path. + syncRootPath := bundleRootPath + for _, path := range b.Config.Sync.Paths { + computedPath := m.computeRoot(path, bundleRootPath) + if computedPath == "" { + continue + } + + // Update sync root path if the computed root path is an ancestor of the current sync root path. + if len(computedPath) < len(syncRootPath) { + syncRootPath = computedPath + } + } + + // The new sync root path can only be an ancestor of the previous root path. + // Compute the relative path from the sync root to the bundle root. + rel, err := filepath.Rel(syncRootPath, bundleRootPath) + if err != nil { + return diag.FromErr(err) + } + + // If during computation of the sync root path we hit the root of the filesystem, + // then one or more of the sync paths are outside the filesystem. + // Check if this happened by verifying that none of the paths escape the root + // when joined with the sync root path. + for i, path := range b.Config.Sync.Paths { + if filepath.IsLocal(filepath.Join(rel, path)) { + continue + } + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("invalid sync path %q", path), + Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), + Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))}, + }) + } + + if diags.HasError() { + return diags + } + + // Update all paths in the sync configuration to be relative to the sync root. + for i, p := range b.Config.Sync.Paths { + b.Config.Sync.Paths[i] = filepath.Join(rel, p) + } + for i, p := range b.Config.Sync.Include { + b.Config.Sync.Include[i] = filepath.Join(rel, p) + } + for i, p := range b.Config.Sync.Exclude { + b.Config.Sync.Exclude[i] = filepath.Join(rel, p) + } + + // Configure the sync root path. + b.SyncRoot = vfs.MustNew(syncRootPath) + b.SyncRootPath = syncRootPath + return nil +} diff --git a/bundle/config/mutator/sync_infer_root_internal_test.go b/bundle/config/mutator/sync_infer_root_internal_test.go new file mode 100644 index 000000000..9ab9c88f4 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root_internal_test.go @@ -0,0 +1,72 @@ +package mutator + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSyncInferRootInternal_ComputeRoot(t *testing.T) { + s := syncInferRoot{} + + tcases := []struct { + path string + root string + out string + }{ + { + // Test that "." doesn't change the root. + path: ".", + root: "/tmp/some/dir", + out: "/tmp/some/dir", + }, + { + // Test that a subdirectory doesn't change the root. + path: "sub", + root: "/tmp/some/dir", + out: "/tmp/some/dir", + }, + { + // Test that a parent directory changes the root. + path: "../common", + root: "/tmp/some/dir", + out: "/tmp/some", + }, + { + // Test that a deeply nested parent directory changes the root. + path: "../../../../../../common", + root: "/tmp/some/dir/that/is/very/deeply/nested", + out: "/tmp/some", + }, + { + // Test that a parent directory changes the root at the filesystem root boundary. + path: "../common", + root: "/tmp", + out: "/", + }, + { + // Test that an invalid parent directory doesn't change the root and returns an empty string. + path: "../common", + root: "/", + out: "", + }, + { + // Test that the returned path is cleaned even if the root doesn't change. + path: "sub", + root: "/tmp/some/../dir", + out: "/tmp/dir", + }, + { + // Test that a relative root path also works. + path: "../common", + root: "foo/bar", + out: "foo", + }, + } + + for _, tc := range tcases { + out := s.computeRoot(tc.path, tc.root) + assert.Equal(t, tc.out, filepath.ToSlash(out)) + } +} diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go new file mode 100644 index 000000000..383e56769 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -0,0 +1,198 @@ +package mutator_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncInferRoot_NominalAbsolute(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + ".", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some/dir"), b.SyncRootPath) + + // Check that the paths are unchanged. + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include) + assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_NominalRelative(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "./some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + ".", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("some/dir"), b.SyncRootPath) + + // Check that the paths are unchanged. + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include) + assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_ParentDirectory(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../common", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, []string{"common"}, b.Config.Sync.Paths) + assert.Equal(t, []string{filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/bar")}, b.Config.Sync.Include) + assert.Equal(t, []string{filepath.FromSlash("dir/baz"), filepath.FromSlash("dir/qux")}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir/that/is/very/deeply/nested", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../../../../../../common", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, []string{"common"}, b.Config.Sync.Paths) + assert.Equal(t, []string{ + filepath.FromSlash("dir/that/is/very/deeply/nested/foo"), + filepath.FromSlash("dir/that/is/very/deeply/nested/bar"), + }, b.Config.Sync.Include) + assert.Equal(t, []string{ + filepath.FromSlash("dir/that/is/very/deeply/nested/baz"), + filepath.FromSlash("dir/that/is/very/deeply/nested/qux"), + }, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_MultiplePaths(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/bundle/root", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "./foo", + "../common", + "./bar", + "../../baz", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, filepath.FromSlash("bundle/root/foo"), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.FromSlash("bundle/common"), b.Config.Sync.Paths[1]) + assert.Equal(t, filepath.FromSlash("bundle/root/bar"), b.Config.Sync.Paths[2]) + assert.Equal(t, filepath.FromSlash("baz"), b.Config.Sync.Paths[3]) +} + +func TestSyncInferRoot_Error(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../../../../error", + "../../../thisworks", + "../../../../../error", + }, + }, + }, + } + + bundletest.SetLocation(b, "sync.paths", "databricks.yml") + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + require.Len(t, diags, 2) + assert.Equal(t, `invalid sync path "../../../../error"`, diags[0].Summary) + assert.Equal(t, "databricks.yml:0:0", diags[0].Locations[0].String()) + assert.Equal(t, "sync.paths[0]", diags[0].Paths[0].String()) + assert.Equal(t, `invalid sync path "../../../../../error"`, diags[1].Summary) + assert.Equal(t, "databricks.yml:0:0", diags[1].Locations[0].String()) + assert.Equal(t, "sync.paths[2]", diags[1].Paths[0].String()) +} diff --git a/bundle/config/mutator/trampoline.go b/bundle/config/mutator/trampoline.go index dde9a299e..dcca50149 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/config/mutator/trampoline.go @@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund return err } - internalDirRel, err := filepath.Rel(b.RootPath, internalDir) + internalDirRel, err := filepath.Rel(b.SyncRootPath, internalDir) if err != nil { return err } diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index de395c165..08d3c8220 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -56,8 +56,12 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Bundle: config.Bundle{ Target: "development", }, @@ -89,6 +93,6 @@ func TestGenerateTrampoline(t *testing.T) { require.Equal(t, "Hello from Trampoline", string(bytes)) task := b.Config.Resources.Jobs["test"].Tasks[0] - require.Equal(t, task.NotebookTask.NotebookPath, ".databricks/bundle/development/.internal/notebook_test_to_trampoline") + require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_test_to_trampoline", task.NotebookTask.NotebookPath) require.Nil(t, task.PythonWheelTask) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 28f7d3d30..5f22570e7 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -93,14 +93,14 @@ func (t *translateContext) rewritePath( return nil } - // Local path must be contained in the bundle root. + // Local path must be contained in the sync root. // If it isn't, it won't be synchronized into the workspace. - localRelPath, err := filepath.Rel(t.b.RootPath, localPath) + localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath) if err != nil { return err } if strings.HasPrefix(localRelPath, "..") { - return fmt.Errorf("path %s is not contained in bundle root path", localPath) + return fmt.Errorf("path %s is not contained in sync root path", localPath) } // Prefix remote path with its remote root path. @@ -118,7 +118,7 @@ func (t *translateContext) rewritePath( } func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath)) + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath)) if errors.Is(err, fs.ErrNotExist) { return "", fmt.Errorf("notebook %s not found", literal) } @@ -134,7 +134,7 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe } func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath)) + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath)) if errors.Is(err, fs.ErrNotExist) { return "", fmt.Errorf("file %s not found", literal) } @@ -148,7 +148,7 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat } func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - info, err := t.b.BundleRoot.Stat(filepath.ToSlash(localRelPath)) + info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath)) if err != nil { return "", err } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index fd64593be..50fcd3b07 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -41,8 +41,8 @@ func touchEmptyFile(t *testing.T, path string) { func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -113,8 +113,8 @@ func TestTranslatePaths(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -289,8 +289,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -380,12 +380,12 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { ) } -func TestTranslatePathsOutsideBundleRoot(t *testing.T) { +func TestTranslatePathsOutsideSyncRoot(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -411,15 +411,15 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) { bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml")) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) - assert.ErrorContains(t, diags.Error(), "is not contained in bundle root") + assert.ErrorContains(t, diags.Error(), "is not contained in sync root path") } func TestJobNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -449,8 +449,8 @@ func TestJobFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -480,8 +480,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -511,8 +511,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -543,8 +543,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -578,8 +578,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -613,8 +613,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -648,8 +648,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -684,8 +684,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "env2.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -724,8 +724,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { func TestTranslatePathWithComplexVariables(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Variables: map[string]*variable.Variable{ "cluster_libraries": { diff --git a/bundle/config/sync.go b/bundle/config/sync.go index 0580e4c4f..377b1333e 100644 --- a/bundle/config/sync.go +++ b/bundle/config/sync.go @@ -1,6 +1,10 @@ package config type Sync struct { + // Paths contains a list of paths to synchronize relative to the bundle root path. + // If not configured, this defaults to synchronizing everything in the bundle root path (i.e. `.`). + Paths []string `json:"paths,omitempty"` + // Include contains a list of globs evaluated relative to the bundle root path // to explicitly include files that were excluded by the user's gitignore. Include []string `json:"include,omitempty"` diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index dc45053f9..347ed3079 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -28,8 +28,8 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp } opts := &sync.SyncOptions{ - LocalRoot: rb.BundleRoot(), - Paths: []string{"."}, + LocalRoot: rb.SyncRoot(), + Paths: rb.Config().Sync.Paths, Include: includes, Exclude: rb.Config().Sync.Exclude, diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 24ed9d360..5e301a6f3 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } log.Infof(ctx, "Creating new snapshot") - snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.BundleRoot), opts) + snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.SyncRoot), opts) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index 38f0b4021..f75193065 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -64,6 +64,10 @@ func testStatePull(t *testing.T, opts statePullOpts) { b := &bundle.Bundle{ RootPath: tmpDir, BundleRoot: vfs.MustNew(tmpDir), + + SyncRootPath: tmpDir, + SyncRoot: vfs.MustNew(tmpDir), + Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -81,11 +85,11 @@ func testStatePull(t *testing.T, opts statePullOpts) { ctx := context.Background() for _, file := range opts.localFiles { - testutil.Touch(t, b.RootPath, "bar", file) + testutil.Touch(t, b.SyncRootPath, "bar", file) } for _, file := range opts.localNotebooks { - testutil.TouchNotebook(t, b.RootPath, "bar", file) + testutil.TouchNotebook(t, b.SyncRootPath, "bar", file) } if opts.withExistingSnapshot { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 7a1081ded..8039a4f13 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -21,7 +21,18 @@ func Initialize() bundle.Mutator { "initialize", []bundle.Mutator{ validate.AllResourcesHaveValues(), + + // Update all path fields in the sync block to be relative to the bundle root path. mutator.RewriteSyncPaths(), + + // Configure the default sync path to equal the bundle root if not explicitly configured. + // By default, this means all files in the bundle root directory are synchronized. + mutator.SyncDefaultPath(), + + // Figure out if the sync root path is identical or an ancestor of the bundle root path. + // If it is an ancestor, this updates all paths to be relative to the sync root path. + mutator.SyncInferRoot(), + mutator.MergeJobClusters(), mutator.MergeJobParameters(), mutator.MergeJobTasks(), diff --git a/bundle/python/conditional_transform_test.go b/bundle/python/conditional_transform_test.go index 677970d70..1d397f7a7 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/python/conditional_transform_test.go @@ -2,7 +2,6 @@ package python import ( "context" - "path" "path/filepath" "testing" @@ -18,11 +17,15 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", }, + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { @@ -63,11 +66,15 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", }, + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { @@ -102,14 +109,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { task := b.Config.Resources.Jobs["job1"].Tasks[0] require.Nil(t, task.PythonWheelTask) require.NotNil(t, task.NotebookTask) - - dir, err := b.InternalDir(context.Background()) - require.NoError(t, err) - - internalDirRel, err := filepath.Rel(b.RootPath, dir) - require.NoError(t, err) - - require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath) + require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_job1_key1", task.NotebookTask.NotebookPath) require.Len(t, task.Libraries, 1) require.Equal(t, "/Workspace/Users/test@test.com/bundle/dist/test.jar", task.Libraries[0].Jar) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 848132a13..5c48d81cb 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -40,6 +40,8 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { diags := bundle.Apply(ctx, b, bundle.Seq( phases.LoadNamedTarget(env), mutator.RewriteSyncPaths(), + mutator.SyncDefaultPath(), + mutator.SyncInferRoot(), mutator.MergeJobClusters(), mutator.MergeJobParameters(), mutator.MergeJobTasks(), diff --git a/bundle/tests/sync/paths/databricks.yml b/bundle/tests/sync/paths/databricks.yml new file mode 100644 index 000000000..9ef6fa032 --- /dev/null +++ b/bundle/tests/sync/paths/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: sync_paths + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + paths: + - src + +targets: + development: + sync: + paths: + - development + + staging: + sync: + paths: + - staging diff --git a/bundle/tests/sync/paths_no_root/databricks.yml b/bundle/tests/sync/paths_no_root/databricks.yml new file mode 100644 index 000000000..df15b12b6 --- /dev/null +++ b/bundle/tests/sync/paths_no_root/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: sync_paths + +workspace: + host: https://acme.cloud.databricks.com/ + +targets: + development: + sync: + paths: + - development + + staging: + sync: + paths: + - staging + + undefined: ~ + + nil: + sync: + paths: ~ + + empty: + sync: + paths: [] diff --git a/bundle/tests/sync/shared_code/bundle/databricks.yml b/bundle/tests/sync/shared_code/bundle/databricks.yml new file mode 100644 index 000000000..738b6170c --- /dev/null +++ b/bundle/tests/sync/shared_code/bundle/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: shared_code + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + paths: + - "../common" + - "." diff --git a/bundle/tests/sync/shared_code/common/library.txt b/bundle/tests/sync/shared_code/common/library.txt new file mode 100644 index 000000000..83b323843 --- /dev/null +++ b/bundle/tests/sync/shared_code/common/library.txt @@ -0,0 +1 @@ +Placeholder for files to be deployed as part of multiple bundles. diff --git a/bundle/tests/sync_test.go b/bundle/tests/sync_test.go index d08e889c3..15644b67e 100644 --- a/bundle/tests/sync_test.go +++ b/bundle/tests/sync_test.go @@ -12,14 +12,20 @@ func TestSyncOverride(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/override", "development") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override", "staging") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override", "prod") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) } @@ -28,14 +34,20 @@ func TestSyncOverrideNoRootSync(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/override_no_root", "development") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override_no_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override_no_root", "prod") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) } @@ -44,10 +56,14 @@ func TestSyncNil(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/nil", "development") + assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.Nil(t, b.Config.Sync.Include) assert.Nil(t, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/nil", "staging") + assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) } @@ -56,10 +72,59 @@ func TestSyncNilRoot(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/nil_root", "development") + assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.Nil(t, b.Config.Sync.Include) assert.Nil(t, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/nil_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) } + +func TestSyncPaths(t *testing.T) { + var b *bundle.Bundle + + b = loadTarget(t, "./sync/paths", "development") + assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath) + assert.Equal(t, []string{"src", "development"}, b.Config.Sync.Paths) + + b = loadTarget(t, "./sync/paths", "staging") + assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath) + assert.Equal(t, []string{"src", "staging"}, b.Config.Sync.Paths) +} + +func TestSyncPathsNoRoot(t *testing.T) { + var b *bundle.Bundle + + b = loadTarget(t, "./sync/paths_no_root", "development") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"development"}, b.Config.Sync.Paths) + + b = loadTarget(t, "./sync/paths_no_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"staging"}, b.Config.Sync.Paths) + + // If not set at all, it defaults to "." + b = loadTarget(t, "./sync/paths_no_root", "undefined") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + + // If set to nil, it won't sync anything. + b = loadTarget(t, "./sync/paths_no_root", "nil") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Len(t, b.Config.Sync.Paths, 0) + + // If set to an empty sequence, it won't sync anything. + b = loadTarget(t, "./sync/paths_no_root", "empty") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Len(t, b.Config.Sync.Paths, 0) +} + +func TestSyncSharedCode(t *testing.T) { + b := loadTarget(t, "./sync/shared_code/bundle", "default") + assert.Equal(t, filepath.FromSlash("sync/shared_code"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"common", "bundle"}, b.Config.Sync.Paths) +} diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 0d0c57385..bd03eec91 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -17,8 +17,10 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tempDir, - BundleRoot: vfs.MustNew(tempDir), + RootPath: tempDir, + BundleRoot: vfs.MustNew(tempDir), + SyncRootPath: tempDir, + SyncRoot: vfs.MustNew(tempDir), Config: config.Root{ Bundle: config.Bundle{ Target: "default", From 35e48be81c634e50164edcfb086c362e948ca57e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 22 Aug 2024 10:44:22 +0200 Subject: [PATCH 87/88] [Release] Release v0.227.0 (#1705) CLI: * Added filtering flags for cluster list commands ([#1703](https://github.com/databricks/cli/pull/1703)). Bundles: * Remove reference to "dbt" in the default-sql template ([#1696](https://github.com/databricks/cli/pull/1696)). * Pause continuous pipelines when 'mode: development' is used ([#1590](https://github.com/databricks/cli/pull/1590)). * Add configurable presets for name prefixes, tags, etc. ([#1490](https://github.com/databricks/cli/pull/1490)). * Report all empty resources present in error diagnostic ([#1685](https://github.com/databricks/cli/pull/1685)). * Improves detection of PyPI package names in environment dependencies ([#1699](https://github.com/databricks/cli/pull/1699)). * [DAB] Add support for requirements libraries in Job Tasks ([#1543](https://github.com/databricks/cli/pull/1543)). * Add paths field to bundle sync configuration ([#1694](https://github.com/databricks/cli/pull/1694)). Internal: * Add `import` option for PyDABs ([#1693](https://github.com/databricks/cli/pull/1693)). * Make fileset take optional list of paths to list ([#1684](https://github.com/databricks/cli/pull/1684)). * Pass through paths argument to libs/sync ([#1689](https://github.com/databricks/cli/pull/1689)). * Correctly mark package names with versions as remote libraries ([#1697](https://github.com/databricks/cli/pull/1697)). * Share test initializer in common helper function ([#1695](https://github.com/databricks/cli/pull/1695)). * Make `pydabs/venv_path` optional ([#1687](https://github.com/databricks/cli/pull/1687)). * Use API mocks for duplicate path errors in workspace files extensions client ([#1690](https://github.com/databricks/cli/pull/1690)). * Fix prefix preset used for UC schemas ([#1704](https://github.com/databricks/cli/pull/1704)). --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39960e308..88a62d098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Version changelog +## [Release] Release v0.227.0 + +CLI: + * Added filtering flags for cluster list commands ([#1703](https://github.com/databricks/cli/pull/1703)). + +Bundles: + * Allow users to configure paths (including outside of the bundle root) to synchronize to the workspace. ([#1694](https://github.com/databricks/cli/pull/1694)). + * Add configurable presets for name prefixes, tags, etc. ([#1490](https://github.com/databricks/cli/pull/1490)). + * Add support for requirements libraries in Job Tasks ([#1543](https://github.com/databricks/cli/pull/1543)). + * Remove reference to "dbt" in the default-sql template ([#1696](https://github.com/databricks/cli/pull/1696)). + * Pause continuous pipelines when 'mode: development' is used ([#1590](https://github.com/databricks/cli/pull/1590)). + * Report all empty resources present in error diagnostic ([#1685](https://github.com/databricks/cli/pull/1685)). + * Improves detection of PyPI package names in environment dependencies ([#1699](https://github.com/databricks/cli/pull/1699)). + +Internal: + * Add `import` option for PyDABs ([#1693](https://github.com/databricks/cli/pull/1693)). + * Make fileset take optional list of paths to list ([#1684](https://github.com/databricks/cli/pull/1684)). + * Pass through paths argument to libs/sync ([#1689](https://github.com/databricks/cli/pull/1689)). + * Correctly mark package names with versions as remote libraries ([#1697](https://github.com/databricks/cli/pull/1697)). + * Share test initializer in common helper function ([#1695](https://github.com/databricks/cli/pull/1695)). + * Make `pydabs/venv_path` optional ([#1687](https://github.com/databricks/cli/pull/1687)). + * Use API mocks for duplicate path errors in workspace files extensions client ([#1690](https://github.com/databricks/cli/pull/1690)). + * Fix prefix preset used for UC schemas ([#1704](https://github.com/databricks/cli/pull/1704)). + + + ## [Release] Release v0.226.0 CLI: From 7fe08c2386edfa503985d93b1e6f633aa85e4f74 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 22 Aug 2024 20:34:26 +0530 Subject: [PATCH 88/88] Revert hc-install version to 0.7.0 (#1711) ## Changes With hc-install version `0.8.0` there was a regression where debug logs would be leaked into stderr. Reported upstream in https://github.com/hashicorp/hc-install/issues/239. Meanwhile we need to revert and pin to version`0.7.0`. This PR also includes a regression test. ## Tests Regression test. --- go.mod | 3 +-- go.sum | 8 ++------ internal/bundle/deploy_test.go | 31 +++++++++++++++++++++++++++++++ internal/bundle/helpers.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 1457a4d67..838a45f36 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.8.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause @@ -49,7 +49,6 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index b2985955c..f55f329f3 100644 --- a/go.sum +++ b/go.sum @@ -99,14 +99,10 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= -github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.8.0 h1:LdpZeXkZYMQhoKPCecJHlKvUkQFixN/nvyR1CdfOLjI= -github.com/hashicorp/hc-install v0.8.0/go.mod h1:+MwJYjDfCruSD/udvBmRB22Nlkwwkwf5sAB6uTIhSaU= +github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= +github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 3da885705..269b7c80a 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -123,3 +124,33 @@ func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } + +func TestAccDeployBasicBundleLogs(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + root, err := initTestTemplate(t, ctx, "basic", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + t.Cleanup(func() { + err = destroyBundle(t, ctx, root) + require.NoError(t, err) + }) + + currentUser, err := wt.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + stdout, stderr := blackBoxRun(t, root, "bundle", "deploy") + assert.Equal(t, strings.Join([]string{ + fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), + "Deploying resources...", + "Updating deployment state...", + "Deployment complete!\n", + }, "\n"), stderr) + assert.Equal(t, "", stdout) +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 03d9cff70..3547c1755 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -1,10 +1,12 @@ package bundle import ( + "bytes" "context" "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -15,6 +17,7 @@ import ( "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/template" + "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/require" ) @@ -114,3 +117,29 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId) return root } + +func blackBoxRun(t *testing.T, root string, args ...string) (stdout string, stderr string) { + cwd := vfs.MustNew(".") + gitRoot, err := vfs.FindLeafInTree(cwd, ".git") + require.NoError(t, err) + + t.Setenv("BUNDLE_ROOT", root) + + // Create the command + cmd := exec.Command("go", append([]string{"run", "main.go"}, args...)...) + cmd.Dir = gitRoot.Native() + + // Create buffers to capture output + var outBuffer, errBuffer bytes.Buffer + cmd.Stdout = &outBuffer + cmd.Stderr = &errBuffer + + // Run the command + err = cmd.Run() + require.NoError(t, err) + + // Get the output + stdout = outBuffer.String() + stderr = errBuffer.String() + return +}