diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index c4b47ca14..fef6f268b 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -7437dabb9dadee402c1fc060df4c1ce8cc5369f0 \ No newline at end of file +f98c07f9c71f579de65d2587bb0292f83d10e55d \ No newline at end of file diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 7e643a90c..431709f90 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -116,12 +116,12 @@ func allResolvers() *resolvers { {{range .Services -}} {{- if in $allowlist .KebabName -}} r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) if err != nil { return "", err } - return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .List.NamedIdMap.IdPath 0).PascalName) }}), nil + return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .NamedIdMap.IdPath 0).PascalName) }}), nil } {{end -}} {{- end}} diff --git a/.gitattributes b/.gitattributes index c11257e9e..bdb3f3982 100755 --- a/.gitattributes +++ b/.gitattributes @@ -24,10 +24,12 @@ cmd/account/service-principals/service-principals.go linguist-generated=true cmd/account/settings/settings.go linguist-generated=true cmd/account/storage-credentials/storage-credentials.go linguist-generated=true cmd/account/storage/storage.go linguist-generated=true +cmd/account/usage-dashboards/usage-dashboards.go linguist-generated=true cmd/account/users/users.go linguist-generated=true cmd/account/vpc-endpoints/vpc-endpoints.go linguist-generated=true cmd/account/workspace-assignment/workspace-assignment.go linguist-generated=true cmd/account/workspaces/workspaces.go linguist-generated=true +cmd/workspace/alerts-legacy/alerts-legacy.go linguist-generated=true cmd/workspace/alerts/alerts.go linguist-generated=true cmd/workspace/apps/apps.go linguist-generated=true cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true @@ -54,6 +56,7 @@ cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go lingu cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true cmd/workspace/functions/functions.go linguist-generated=true +cmd/workspace/genie/genie.go linguist-generated=true cmd/workspace/git-credentials/git-credentials.go linguist-generated=true cmd/workspace/global-init-scripts/global-init-scripts.go linguist-generated=true cmd/workspace/grants/grants.go linguist-generated=true @@ -67,6 +70,7 @@ cmd/workspace/libraries/libraries.go linguist-generated=true cmd/workspace/metastores/metastores.go linguist-generated=true cmd/workspace/model-registry/model-registry.go linguist-generated=true cmd/workspace/model-versions/model-versions.go linguist-generated=true +cmd/workspace/notification-destinations/notification-destinations.go linguist-generated=true cmd/workspace/online-tables/online-tables.go linguist-generated=true cmd/workspace/permission-migration/permission-migration.go linguist-generated=true cmd/workspace/permissions/permissions.go linguist-generated=true @@ -81,8 +85,10 @@ cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics cmd/workspace/provider-providers/provider-providers.go linguist-generated=true cmd/workspace/providers/providers.go linguist-generated=true cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true +cmd/workspace/queries-legacy/queries-legacy.go linguist-generated=true cmd/workspace/queries/queries.go linguist-generated=true cmd/workspace/query-history/query-history.go linguist-generated=true +cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go linguist-generated=true cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true cmd/workspace/recipient-activation/recipient-activation.go linguist-generated=true cmd/workspace/recipients/recipients.go linguist-generated=true diff --git a/.vscode/settings.json b/.vscode/settings.json index 869465286..9697e221d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "files.trimFinalNewlines": true, - "python.envFile": "${workspaceFolder}/.databricks/.databricks.env", + "python.envFile": "${workspaceRoot}/.env", "databricks.python.envFile": "${workspaceFolder}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d81f822..88a62d098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,192 @@ # Version changelog +## [Release] Release v0.227.0 + +CLI: + * Added filtering flags for cluster list commands ([#1703](https://github.com/databricks/cli/pull/1703)). + +Bundles: + * Allow users to configure paths (including outside of the bundle root) to synchronize to the workspace. ([#1694](https://github.com/databricks/cli/pull/1694)). + * Add configurable presets for name prefixes, tags, etc. ([#1490](https://github.com/databricks/cli/pull/1490)). + * Add support for requirements libraries in Job Tasks ([#1543](https://github.com/databricks/cli/pull/1543)). + * Remove reference to "dbt" in the default-sql template ([#1696](https://github.com/databricks/cli/pull/1696)). + * Pause continuous pipelines when 'mode: development' is used ([#1590](https://github.com/databricks/cli/pull/1590)). + * Report all empty resources present in error diagnostic ([#1685](https://github.com/databricks/cli/pull/1685)). + * Improves detection of PyPI package names in environment dependencies ([#1699](https://github.com/databricks/cli/pull/1699)). + +Internal: + * Add `import` option for PyDABs ([#1693](https://github.com/databricks/cli/pull/1693)). + * Make fileset take optional list of paths to list ([#1684](https://github.com/databricks/cli/pull/1684)). + * Pass through paths argument to libs/sync ([#1689](https://github.com/databricks/cli/pull/1689)). + * Correctly mark package names with versions as remote libraries ([#1697](https://github.com/databricks/cli/pull/1697)). + * Share test initializer in common helper function ([#1695](https://github.com/databricks/cli/pull/1695)). + * Make `pydabs/venv_path` optional ([#1687](https://github.com/databricks/cli/pull/1687)). + * Use API mocks for duplicate path errors in workspace files extensions client ([#1690](https://github.com/databricks/cli/pull/1690)). + * Fix prefix preset used for UC schemas ([#1704](https://github.com/databricks/cli/pull/1704)). + + + +## [Release] Release v0.226.0 + +CLI: + * Add command line autocomplete to the fs commands ([#1622](https://github.com/databricks/cli/pull/1622)). + * Add trailing slash to directory to produce completions for ([#1666](https://github.com/databricks/cli/pull/1666)). + * Fix ability to import the CLI repository as module ([#1671](https://github.com/databricks/cli/pull/1671)). + * Fix host resolution order in `auth login` ([#1370](https://github.com/databricks/cli/pull/1370)). + * Print text logs in `import-dir` and `export-dir` commands ([#1682](https://github.com/databricks/cli/pull/1682)). + +Bundles: + * Expand and upload local wheel libraries for all task types ([#1649](https://github.com/databricks/cli/pull/1649)). + * Clarify file format required for the `config-file` flag in `bundle init` ([#1651](https://github.com/databricks/cli/pull/1651)). + * Fixed incorrectly cleaning up python wheel dist folder ([#1656](https://github.com/databricks/cli/pull/1656)). + * Merge job parameters based on their name ([#1659](https://github.com/databricks/cli/pull/1659)). + * Fix glob expansion after running a generic build command ([#1662](https://github.com/databricks/cli/pull/1662)). + * Upload local libraries even if they don't have artifact defined ([#1664](https://github.com/databricks/cli/pull/1664)). + +Internal: + * Fix python wheel task integration tests ([#1648](https://github.com/databricks/cli/pull/1648)). + * Skip pushing Terraform state after destroy ([#1667](https://github.com/databricks/cli/pull/1667)). + * Enable Spark JAR task test ([#1658](https://github.com/databricks/cli/pull/1658)). + * Run Spark JAR task test on multiple DBR versions ([#1665](https://github.com/databricks/cli/pull/1665)). + * Stop tracking file path locations in bundle resources ([#1673](https://github.com/databricks/cli/pull/1673)). + * Update VS Code settings to match latest value from IDE plugin ([#1677](https://github.com/databricks/cli/pull/1677)). + * Use `service.NamedIdMap` to make lookup generation deterministic ([#1678](https://github.com/databricks/cli/pull/1678)). + * [Internal] Remove dependency to the `openapi` package of the Go SDK ([#1676](https://github.com/databricks/cli/pull/1676)). + * Upgrade TF provider to 1.50.0 ([#1681](https://github.com/databricks/cli/pull/1681)). + * Upgrade Go SDK to 0.44.0 ([#1679](https://github.com/databricks/cli/pull/1679)). + +API Changes: + * Changed `databricks account budgets create` command . New request type is . + * Changed `databricks account budgets create` command to return . + * Changed `databricks account budgets delete` command . New request type is . + * Changed `databricks account budgets delete` command to return . + * Changed `databricks account budgets get` command . New request type is . + * Changed `databricks account budgets get` command to return . + * Changed `databricks account budgets list` command to require request of . + * Changed `databricks account budgets list` command to return . + * Changed `databricks account budgets update` command . New request type is . + * Changed `databricks account budgets update` command to return . + * Added `databricks account usage-dashboards` command group. + * Changed `databricks model-versions get` command to return . + * Changed `databricks cluster-policies create` command with new required argument order. + * Changed `databricks cluster-policies edit` command with new required argument order. + * Added `databricks clusters update` command. + * Added `databricks genie` command group. + * Changed `databricks permission-migration migrate-permissions` command . New request type is . + * Changed `databricks permission-migration migrate-permissions` command to return . + * Changed `databricks account workspace-assignment delete` command to return . + * Changed `databricks account workspace-assignment update` command with new required argument order. + * Changed `databricks account custom-app-integration create` command with new required argument order. + * Changed `databricks account custom-app-integration list` command to require request of . + * Changed `databricks account published-app-integration list` command to require request of . + * Removed `databricks apps` command group. + * Added `databricks notification-destinations` command group. + * Changed `databricks shares list` command to require request of . + * Changed `databricks alerts create` command . New request type is . + * Changed `databricks alerts delete` command . New request type is . + * Changed `databricks alerts delete` command to return . + * Changed `databricks alerts get` command with new required argument order. + * Changed `databricks alerts list` command to require request of . + * Changed `databricks alerts list` command to return . + * Changed `databricks alerts update` command . New request type is . + * Changed `databricks alerts update` command to return . + * Changed `databricks queries create` command . New request type is . + * Changed `databricks queries delete` command . New request type is . + * Changed `databricks queries delete` command to return . + * Changed `databricks queries get` command with new required argument order. + * Changed `databricks queries list` command to return . + * Removed `databricks queries restore` command. + * Changed `databricks queries update` command . New request type is . + * Added `databricks queries list-visualizations` command. + * Changed `databricks query-visualizations create` command . New request type is . + * Changed `databricks query-visualizations delete` command . New request type is . + * Changed `databricks query-visualizations delete` command to return . + * Changed `databricks query-visualizations update` command . New request type is . + * Changed `databricks statement-execution execute-statement` command to return . + * Changed `databricks statement-execution get-statement` command to return . + * Added `databricks alerts-legacy` command group. + * Added `databricks queries-legacy` command group. + * Added `databricks query-visualizations-legacy` command group. + +OpenAPI commit f98c07f9c71f579de65d2587bb0292f83d10e55d (2024-08-12) +Dependency updates: + * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.8.0 ([#1652](https://github.com/databricks/cli/pull/1652)). + * Bump golang.org/x/sync from 0.7.0 to 0.8.0 ([#1655](https://github.com/databricks/cli/pull/1655)). + * Bump golang.org/x/mod from 0.19.0 to 0.20.0 ([#1654](https://github.com/databricks/cli/pull/1654)). + * Bump golang.org/x/oauth2 from 0.21.0 to 0.22.0 ([#1653](https://github.com/databricks/cli/pull/1653)). + * Bump golang.org/x/text from 0.16.0 to 0.17.0 ([#1670](https://github.com/databricks/cli/pull/1670)). + * Bump golang.org/x/term from 0.22.0 to 0.23.0 ([#1669](https://github.com/databricks/cli/pull/1669)). + +## 0.225.0 + +Bundles: + * Add resource for UC schemas to DABs ([#1413](https://github.com/databricks/cli/pull/1413)). + +Internal: + * Use dynamic walking to validate unique resource keys ([#1614](https://github.com/databricks/cli/pull/1614)). + * Regenerate TF schema ([#1635](https://github.com/databricks/cli/pull/1635)). + * Add upgrade and upgrade eager flags to pip install call ([#1636](https://github.com/databricks/cli/pull/1636)). + * Added test for negation pattern in sync include exclude section ([#1637](https://github.com/databricks/cli/pull/1637)). + * Use precomputed terraform plan for `bundle deploy` ([#1640](https://github.com/databricks/cli/pull/1640)). + +## 0.224.1 + +Bundles: + * Add UUID function to bundle template functions ([#1612](https://github.com/databricks/cli/pull/1612)). + * Upgrade TF provider to 1.49.0 ([#1617](https://github.com/databricks/cli/pull/1617)). + * Upgrade TF provider to 1.49.1 ([#1626](https://github.com/databricks/cli/pull/1626)). + * Support multiple locations for diagnostics ([#1610](https://github.com/databricks/cli/pull/1610)). + * Split artifact cleanup into prepare step before build ([#1618](https://github.com/databricks/cli/pull/1618)). + * Move to a single prompt during bundle destroy ([#1583](https://github.com/databricks/cli/pull/1583)). + +Internal: + * Add tests for the Workspace API readahead cache ([#1605](https://github.com/databricks/cli/pull/1605)). + * Update Python dependencies before install when upgrading a labs project ([#1624](https://github.com/databricks/cli/pull/1624)). + + + +## 0.224.0 + +CLI: + * Do not buffer files in memory when downloading ([#1599](https://github.com/databricks/cli/pull/1599)). + +Bundles: + * Allow artifacts (JARs, wheels) to be uploaded to UC Volumes ([#1591](https://github.com/databricks/cli/pull/1591)). + * Upgrade TF provider to 1.48.3 ([#1600](https://github.com/databricks/cli/pull/1600)). + * Fixed job name normalisation for bundle generate ([#1601](https://github.com/databricks/cli/pull/1601)). + +Internal: + * Add UUID to uniquely identify a deployment state ([#1595](https://github.com/databricks/cli/pull/1595)). + * Track multiple locations associated with a `dyn.Value` ([#1510](https://github.com/databricks/cli/pull/1510)). + * Attribute Terraform API requests the CLI ([#1598](https://github.com/databricks/cli/pull/1598)). + * Implement readahead cache for Workspace API calls ([#1582](https://github.com/databricks/cli/pull/1582)). + * Add read-only mode for extension aware workspace filer ([#1609](https://github.com/databricks/cli/pull/1609)). + +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.43.0 to 0.43.2 ([#1594](https://github.com/databricks/cli/pull/1594)). + +## 0.223.2 + +Bundles: + * Override complex variables with target overrides instead of merging ([#1567](https://github.com/databricks/cli/pull/1567)). + * Rewrite local path for libraries in foreach tasks ([#1569](https://github.com/databricks/cli/pull/1569)). + * Change SetVariables mutator to mutate dynamic configuration instead ([#1573](https://github.com/databricks/cli/pull/1573)). + * Return early in bundle destroy if no deployment exists ([#1581](https://github.com/databricks/cli/pull/1581)). + * Let notebook detection code use underlying metadata if available ([#1574](https://github.com/databricks/cli/pull/1574)). + * Remove schema override for variable default value ([#1536](https://github.com/databricks/cli/pull/1536)). + * Print diagnostics in 'bundle deploy' ([#1579](https://github.com/databricks/cli/pull/1579)). + +Internal: + * Update actions/upload-artifact to v4 ([#1559](https://github.com/databricks/cli/pull/1559)). + * Use Go 1.22 to build and test ([#1562](https://github.com/databricks/cli/pull/1562)). + * Move bespoke status call to main workspace files filer ([#1570](https://github.com/databricks/cli/pull/1570)). + * Add new template ([#1578](https://github.com/databricks/cli/pull/1578)). + * Add regression tests for CLI error output ([#1566](https://github.com/databricks/cli/pull/1566)). + +Dependency updates: + * Bump golang.org/x/mod from 0.18.0 to 0.19.0 ([#1576](https://github.com/databricks/cli/pull/1576)). + * Bump golang.org/x/term from 0.21.0 to 0.22.0 ([#1577](https://github.com/databricks/cli/pull/1577)). + ## 0.223.1 This bugfix release fixes missing error messages in v0.223.0. diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index a5f41ae4b..e5e55a14d 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -1,21 +1,15 @@ package artifacts import ( - "bytes" "context" - "errors" "fmt" - "os" - "path" - "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts/whl" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) @@ -25,7 +19,9 @@ var buildMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactTy config.ArtifactPythonWheel: whl.Build, } -var uploadMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{} +var prepareMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{ + config.ArtifactPythonWheel: whl.Prepare, +} func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { mutatorFactory, ok := buildMutators[t] @@ -36,10 +32,12 @@ func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { return mutatorFactory(name) } -func getUploadMutator(t config.ArtifactType, name string) bundle.Mutator { - mutatorFactory, ok := uploadMutators[t] +func getPrepareMutator(t config.ArtifactType, name string) bundle.Mutator { + mutatorFactory, ok := prepareMutators[t] if !ok { - mutatorFactory = BasicUpload + mutatorFactory = func(_ string) bundle.Mutator { + return mutator.NoOp() + } } return mutatorFactory(name) @@ -74,162 +72,3 @@ func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return nil } - -// Basic Upload defines a general upload mutator which uploads artifact as a library to workspace -type basicUpload struct { - name string -} - -func BasicUpload(name string) bundle.Mutator { - return &basicUpload{name: name} -} - -func (m *basicUpload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - uploadPath, err := getUploadBasePath(b) - if err != nil { - return diag.FromErr(err) - } - - client, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(), uploadPath) - if err != nil { - return diag.FromErr(err) - } - - err = uploadArtifact(ctx, b, artifact, uploadPath, client) - if err != nil { - return diag.Errorf("upload for %s failed, error: %v", m.name, err) - } - - return nil -} - -func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, uploadPath string, client filer.Filer) error { - for i := range a.Files { - f := &a.Files[i] - - filename := filepath.Base(f.Source) - cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) - - err := uploadArtifactFile(ctx, f.Source, client) - if err != nil { - return err - } - - log.Infof(ctx, "Upload succeeded") - f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source)) - - // TODO: confirm if we still need to update the remote path to start with /Workspace - wsfsBase := "/Workspace" - remotePath := path.Join(wsfsBase, f.RemotePath) - - for _, job := range b.Config.Resources.Jobs { - rewriteArtifactPath(b, f, job, remotePath) - - } - } - - return nil -} - -func rewriteArtifactPath(b *bundle.Bundle, f *config.ArtifactFile, job *resources.Job, remotePath string) { - // Rewrite artifact path in job task libraries - for i := range job.Tasks { - task := &job.Tasks[i] - for j := range task.Libraries { - lib := &task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - - // Rewrite artifact path in job task libraries for ForEachTask - if task.ForEachTask != nil { - forEachTask := task.ForEachTask - for j := range forEachTask.Task.Libraries { - lib := &forEachTask.Task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - } - } - - // Rewrite artifact path in job environments - for i := range job.Environments { - env := &job.Environments[i] - if env.Spec == nil { - continue - } - - for j := range env.Spec.Dependencies { - lib := env.Spec.Dependencies[j] - if isArtifactMatchLibrary(f, lib, b) { - env.Spec.Dependencies[j] = remotePath - } - } - } -} - -func isArtifactMatchLibrary(f *config.ArtifactFile, libPath string, b *bundle.Bundle) bool { - if !filepath.IsAbs(libPath) { - libPath = filepath.Join(b.RootPath, libPath) - } - - // libPath can be a glob pattern, so do the match first - matches, err := filepath.Glob(libPath) - if err != nil { - return false - } - - for _, m := range matches { - if m == f.Source { - return true - } - } - - return false -} - -// Function to upload artifact file to Workspace -func uploadArtifactFile(ctx context.Context, file string, client filer.Filer) error { - raw, err := os.ReadFile(file) - if err != nil { - return fmt.Errorf("unable to read %s: %w", file, errors.Unwrap(err)) - } - - filename := filepath.Base(file) - err = client.Write(ctx, filename, bytes.NewReader(raw), filer.OverwriteIfExists, filer.CreateParentDirectories) - if err != nil { - return fmt.Errorf("unable to import %s: %w", filename, err) - } - - return nil -} - -func getUploadBasePath(b *bundle.Bundle) (string, error) { - artifactPath := b.Config.Workspace.ArtifactPath - if artifactPath == "" { - return "", fmt.Errorf("remote artifact path not configured") - } - - return path.Join(artifactPath, ".internal"), nil -} diff --git a/bundle/artifacts/artifacts_test.go b/bundle/artifacts/artifacts_test.go deleted file mode 100644 index 53c2798ed..000000000 --- a/bundle/artifacts/artifacts_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package artifacts - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestArtifactUpload(t *testing.T) { - tmpDir := t.TempDir() - whlFolder := filepath.Join(tmpDir, "whl") - testutil.Touch(t, whlFolder, "source.whl") - whlLocalPath := filepath.Join(whlFolder, "source.whl") - - b := &bundle.Bundle{ - RootPath: tmpDir, - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/foo/bar/artifacts", - }, - Artifacts: config.Artifacts{ - "whl": { - Type: config.ArtifactPythonWheel, - Files: []config.ArtifactFile{ - {Source: whlLocalPath}, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - { - ForEachTask: &jobs.ForEachTask{ - Task: jobs.Task{ - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - Environments: []jobs.JobEnvironment{ - { - Spec: &compute.Environment{ - Dependencies: []string{ - filepath.Join("whl", "source.whl"), - "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - artifact := b.Config.Artifacts["whl"] - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("source.whl"), - mock.AnythingOfType("*bytes.Reader"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - err := uploadArtifact(context.Background(), b, artifact, "/foo/bar/artifacts", mockFiler) - require.NoError(t, err) - - // Test that libraries path is updated - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) -} diff --git a/bundle/artifacts/autodetect.go b/bundle/artifacts/autodetect.go index 0e94edd82..569a480f0 100644 --- a/bundle/artifacts/autodetect.go +++ b/bundle/artifacts/autodetect.go @@ -29,6 +29,5 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return bundle.Apply(ctx, b, bundle.Seq( whl.DetectPackage(), - whl.DefineArtifactsFromLibraries(), )) } diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 722891ada..0446135b6 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -3,10 +3,8 @@ package artifacts import ( "context" "fmt" - "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" ) @@ -35,35 +33,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("artifact doesn't exist: %s", m.name) } - // Check if source paths are absolute, if not, make them absolute - for k := range artifact.Files { - f := &artifact.Files[k] - if !filepath.IsAbs(f.Source) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - f.Source = filepath.Join(dirPath, f.Source) - } - } - - // Expand any glob reference in files source path - files := make([]config.ArtifactFile, 0, len(artifact.Files)) - for _, f := range artifact.Files { - matches, err := filepath.Glob(f.Source) - if err != nil { - return diag.Errorf("unable to find files for %s: %v", f.Source, err) - } - - if len(matches) == 0 { - return diag.Errorf("no files found for %s", f.Source) - } - - for _, match := range matches { - files = append(files, config.ArtifactFile{ - Source: match, - }) - } - } - - artifact.Files = files + var mutators []bundle.Mutator // Skip building if build command is not specified or infered if artifact.BuildCommand == "" { @@ -72,18 +42,16 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if len(artifact.Files) == 0 { return diag.Errorf("misconfigured artifact: please specify 'build' or 'files' property") } - return nil + + // We can skip calling build mutator if there is no build command + // But we still need to expand glob references in files source path. + } else { + mutators = append(mutators, getBuildMutator(artifact.Type, m.name)) } - // If artifact path is not provided, use bundle root dir - if artifact.Path == "" { - artifact.Path = b.RootPath - } - - if !filepath.IsAbs(artifact.Path) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - artifact.Path = filepath.Join(dirPath, artifact.Path) - } - - return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) + // We need to expand glob reference after build mutator is applied because + // if we do it before, any files that are generated by build command will + // not be included into artifact.Files and thus will not be uploaded. + mutators = append(mutators, &expandGlobs{name: m.name}) + return bundle.Apply(ctx, b, bundle.Seq(mutators...)) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go new file mode 100644 index 000000000..617444054 --- /dev/null +++ b/bundle/artifacts/expand_globs.go @@ -0,0 +1,110 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expandGlobs struct { + name string +} + +func (m *expandGlobs) Name() string { + return fmt.Sprintf("artifacts.ExpandGlobs(%s)", m.name) +} + +func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { + // The pattern contained in v is an absolute path. + // Make it relative to the value's location to make it more readable. + source := v.MustString() + if l := v.Location(); l.File != "" { + rel, err := filepath.Rel(filepath.Dir(l.File), source) + if err == nil { + source = rel + } + } + + return diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s: %s", source, message), + Locations: []dyn.Location{v.Location()}, + + Paths: []dyn.Path{ + // Hack to clone the path. This path copy is mutable. + // To be addressed in a later PR. + p.Append(), + }, + } +} + +func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Base path for this mutator. + // This path is set with the list of expanded globs when done. + base := dyn.NewPath( + dyn.Key("artifacts"), + dyn.Key(m.name), + dyn.Key("files"), + ) + + // Pattern to match the source key in the files sequence. + pattern := dyn.NewPatternFromPath(base).Append( + dyn.AnyIndex(), + dyn.Key("source"), + ) + + var diags diag.Diagnostics + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var output []dyn.Value + _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() != dyn.KindString { + return v, nil + } + + source := v.MustString() + + // Expand any glob reference in files source path + matches, err := filepath.Glob(source) + if err != nil { + diags = diags.Append(createGlobError(v, p, err.Error())) + + // Continue processing and leave this value unchanged. + return v, nil + } + + if len(matches) == 0 { + diags = diags.Append(createGlobError(v, p, "no matching files")) + + // Continue processing and leave this value unchanged. + return v, nil + } + + for _, match := range matches { + output = append(output, dyn.V( + map[string]dyn.Value{ + "source": dyn.NewValue(match, v.Locations()), + }, + )) + } + + return v, nil + }) + + if err != nil || diags.HasError() { + return v, err + } + + // Set the expanded globs back into the configuration. + return dyn.SetByPath(v, base, dyn.V(output)) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go new file mode 100644 index 000000000..c9c478448 --- /dev/null +++ b/bundle/artifacts/expand_globs_test.go @@ -0,0 +1,156 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpandGlobs_Nominal(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "aa1.txt") + testutil.Touch(t, tmpDir, "aa2.txt") + testutil.Touch(t, tmpDir, "bb.txt") + testutil.Touch(t, tmpDir, "bc.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "./aa*.txt"}, + {Source: "./b[bc].txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + require.NoError(t, diags.Error()) + + // Assert that the expanded paths are correct. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + assert.Len(t, a.Files, 4) + assert.Equal(t, filepath.Join(tmpDir, "aa1.txt"), a.Files[0].Source) + assert.Equal(t, filepath.Join(tmpDir, "aa2.txt"), a.Files[1].Source) + assert.Equal(t, filepath.Join(tmpDir, "bb.txt"), a.Files[2].Source) + assert.Equal(t, filepath.Join(tmpDir, "bc.txt"), a.Files[3].Source) +} + +func TestExpandGlobs_InvalidPattern(t *testing.T) { + tmpDir := t.TempDir() + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a[.txt"}, + {Source: "./a[.txt"}, + {Source: "../a[.txt"}, + {Source: "subdir/a[.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 4) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String()) +} + +func TestExpandGlobs_NoMatches(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "a1.txt") + testutil.Touch(t, tmpDir, "a2.txt") + testutil.Touch(t, tmpDir, "b1.txt") + testutil.Touch(t, tmpDir, "b2.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a*.txt"}, + {Source: "b*.txt"}, + {Source: "c*.txt"}, + {Source: "d*.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 2) + assert.Equal(t, "c*.txt: no matching files", diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[0].Paths[0].String()) + assert.Equal(t, "d*.txt: no matching files", diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[1].Paths[0].String()) + + // Assert that the original paths are unchanged. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + + assert.Len(t, a.Files, 4) + assert.Equal(t, "a*.txt", filepath.Base(a.Files[0].Source)) + assert.Equal(t, "b*.txt", filepath.Base(a.Files[1].Source)) + assert.Equal(t, "c*.txt", filepath.Base(a.Files[2].Source)) + assert.Equal(t, "d*.txt", filepath.Base(a.Files[3].Source)) +} diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go new file mode 100644 index 000000000..fb61ed9e2 --- /dev/null +++ b/bundle/artifacts/prepare.go @@ -0,0 +1,58 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +func PrepareAll() bundle.Mutator { + return &all{ + name: "Prepare", + fn: prepareArtifactByName, + } +} + +type prepare struct { + name string +} + +func prepareArtifactByName(name string) (bundle.Mutator, error) { + return &prepare{name}, nil +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + l := b.Config.GetLocation("artifacts." + m.name) + dirPath := filepath.Dir(l.File) + + // Check if source paths are absolute, if not, make them absolute + for k := range artifact.Files { + f := &artifact.Files[k] + if !filepath.IsAbs(f.Source) { + f.Source = filepath.Join(dirPath, f.Source) + } + } + + // If artifact path is not provided, use bundle root dir + if artifact.Path == "" { + artifact.Path = b.RootPath + } + + if !filepath.IsAbs(artifact.Path) { + artifact.Path = filepath.Join(dirPath, artifact.Path) + } + + return bundle.Apply(ctx, b, getPrepareMutator(artifact.Type, m.name)) +} diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 5c12c9444..58c006dc1 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -2,49 +2,18 @@ package artifacts import ( "context" - "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" ) -func UploadAll() bundle.Mutator { - return &all{ - name: "Upload", - fn: uploadArtifactByName, - } -} - func CleanUp() bundle.Mutator { return &cleanUp{} } -type upload struct { - name string -} - -func uploadArtifactByName(name string) (bundle.Mutator, error) { - return &upload{name}, nil -} - -func (m *upload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name)) -} - type cleanUp struct{} func (m *cleanUp) Name() string { @@ -52,17 +21,23 @@ func (m *cleanUp) Name() string { } func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - uploadPath, err := getUploadBasePath(b) + uploadPath, err := libraries.GetUploadBasePath(b) if err != nil { return diag.FromErr(err) } - b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{ - Path: uploadPath, - Recursive: true, - }) + client, err := libraries.GetFilerForLibraries(b.WorkspaceClient(), uploadPath) + if err != nil { + return diag.FromErr(err) + } - err = b.WorkspaceClient().Workspace.MkdirsByPath(ctx, uploadPath) + // We intentionally ignore the error because it is not critical to the deployment + err = client.Delete(ctx, ".", filer.DeleteRecursively) + if err != nil { + log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err) + } + + err = client.Mkdir(ctx, ".") if err != nil { return diag.Errorf("unable to create directory for %s: %v", uploadPath, err) } diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go deleted file mode 100644 index cf08843a7..000000000 --- a/bundle/artifacts/upload_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package artifacts - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/testfile" - "github.com/stretchr/testify/require" -) - -type noop struct{} - -func (n *noop) Apply(context.Context, *bundle.Bundle) diag.Diagnostics { - return nil -} - -func (n *noop) Name() string { - return "noop" -} - -func TestExpandGlobFilesSource(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar")) - t1.Close(t) - - t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar")) - t2.Close(t) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "*.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.NoError(t, diags.Error()) - - require.Equal(t, 2, len(b.Config.Artifacts["test"].Files)) - require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source) - require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source) -} - -func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "myjar.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.ErrorContains(t, diags.Error(), "no files found for") -} diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index ee77fff01..1601767f6 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -27,9 +27,9 @@ func (m *detectPkg) Name() string { } func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - wheelTasks := libraries.FindAllWheelTasksWithLocalLibraries(b) - if len(wheelTasks) == 0 { - log.Infof(ctx, "No local wheel tasks in databricks.yml config, skipping auto detect") + tasks := libraries.FindTasksWithLocalLibraries(b) + if len(tasks) == 0 { + log.Infof(ctx, "No local tasks in databricks.yml config, skipping auto detect") return nil } log.Infof(ctx, "Detecting Python wheel project...") diff --git a/bundle/artifacts/whl/build.go b/bundle/artifacts/whl/build.go index 992ade297..18d4b8ede 100644 --- a/bundle/artifacts/whl/build.go +++ b/bundle/artifacts/whl/build.go @@ -3,7 +3,6 @@ package whl import ( "context" "fmt" - "os" "path/filepath" "github.com/databricks/cli/bundle" @@ -36,18 +35,14 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Building %s...", m.name)) - dir := artifact.Path - - distPath := filepath.Join(dir, "dist") - os.RemoveAll(distPath) - python.CleanupWheelFolder(dir) - out, err := artifact.Build(ctx) if err != nil { return diag.Errorf("build failed %s, error: %v, output: %s", m.name, err, out) } log.Infof(ctx, "Build succeeded") + dir := artifact.Path + distPath := filepath.Join(artifact.Path, "dist") wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") if len(wheels) == 0 { return diag.Errorf("cannot find built wheel in %s for package %s", dir, m.name) diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go deleted file mode 100644 index ad321557c..000000000 --- a/bundle/artifacts/whl/from_libraries.go +++ /dev/null @@ -1,74 +0,0 @@ -package whl - -import ( - "context" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" -) - -type fromLibraries struct{} - -func DefineArtifactsFromLibraries() bundle.Mutator { - return &fromLibraries{} -} - -func (m *fromLibraries) Name() string { - return "artifacts.whl.DefineArtifactsFromLibraries" -} - -func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if len(b.Config.Artifacts) != 0 { - log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") - return nil - } - - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) - for _, task := range tasks { - for _, lib := range task.Libraries { - matchAndAdd(ctx, lib.Whl, b) - } - } - - envs := libraries.FindAllEnvironments(b) - for _, jobEnvs := range envs { - for _, env := range jobEnvs { - if env.Spec != nil { - for _, dep := range env.Spec.Dependencies { - if libraries.IsEnvironmentDependencyLocal(dep) { - matchAndAdd(ctx, dep, b) - } - } - } - } - } - - return nil -} - -func matchAndAdd(ctx context.Context, lib string, b *bundle.Bundle) { - matches, err := filepath.Glob(filepath.Join(b.RootPath, lib)) - // File referenced from libraries section does not exists, skipping - if err != nil { - return - } - - for _, match := range matches { - name := filepath.Base(match) - if b.Config.Artifacts == nil { - b.Config.Artifacts = make(map[string]*config.Artifact) - } - - log.Debugf(ctx, "Adding an artifact block for %s", match) - b.Config.Artifacts[name] = &config.Artifact{ - Files: []config.ArtifactFile{ - {Source: match}, - }, - Type: config.ArtifactPythonWheel, - } - } -} diff --git a/bundle/artifacts/whl/infer.go b/bundle/artifacts/whl/infer.go index dd4ad2956..cb727de0e 100644 --- a/bundle/artifacts/whl/infer.go +++ b/bundle/artifacts/whl/infer.go @@ -15,6 +15,8 @@ type infer struct { func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { artifact := b.Config.Artifacts[m.name] + + // TODO use python.DetectVEnvExecutable once bundle has a way to specify venv path py, err := python.DetectExecutable(ctx) if err != nil { return diag.FromErr(err) diff --git a/bundle/artifacts/whl/prepare.go b/bundle/artifacts/whl/prepare.go new file mode 100644 index 000000000..0fbb2080a --- /dev/null +++ b/bundle/artifacts/whl/prepare.go @@ -0,0 +1,53 @@ +package whl + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/python" +) + +type prepare struct { + name string +} + +func Prepare(name string) bundle.Mutator { + return &prepare{ + name: name, + } +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.whl.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + // If there is no build command for the artifact, we don't need to cleanup the dist folder before + if artifact.BuildCommand == "" { + return nil + } + + dir := artifact.Path + + distPath := filepath.Join(dir, "dist") + + // If we have multiple artifacts con figured, prepare will be called multiple times + // The first time we will remove the folders, other times will be no-op. + err := os.RemoveAll(distPath) + if err != nil { + log.Infof(ctx, "Failed to remove dist folder: %v", err) + } + python.CleanupWheelFolder(dir) + + return nil +} diff --git a/bundle/bundle.go b/bundle/bundle.go index 032d98abc..8b5ff976d 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -39,6 +39,14 @@ type Bundle struct { // Exclusively use this field for filesystem operations. BundleRoot vfs.Path + // SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace. + // It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot]. + SyncRoot vfs.Path + + // SyncRootPath is the local path to the root directory of files that are synchronized to the workspace. + // It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access. + SyncRootPath string + Config config.Root // Metadata about the bundle deployment. This is the interface Databricks services diff --git a/bundle/bundle_read_only.go b/bundle/bundle_read_only.go index 59084f2ac..74b9d94de 100644 --- a/bundle/bundle_read_only.go +++ b/bundle/bundle_read_only.go @@ -28,6 +28,10 @@ func (r ReadOnlyBundle) BundleRoot() vfs.Path { return r.b.BundleRoot } +func (r ReadOnlyBundle) SyncRoot() vfs.Path { + return r.b.SyncRoot +} + func (r ReadOnlyBundle) WorkspaceClient() *databricks.WorkspaceClient { return r.b.WorkspaceClient() } diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index 219def571..9a5690f57 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -4,18 +4,11 @@ import ( "context" "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/exec" ) type Artifacts map[string]*Artifact -func (artifacts Artifacts) ConfigureConfigFilePath() { - for _, artifact := range artifacts { - artifact.ConfigureConfigFilePath() - } -} - type ArtifactType string const ArtifactPythonWheel ArtifactType = `whl` @@ -40,8 +33,6 @@ type Artifact struct { BuildCommand string `json:"build,omitempty"` Executable exec.ExecutableType `json:"executable,omitempty"` - - paths.Paths } func (a *Artifact) Build(ctx context.Context) ([]byte, error) { diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 12048a322..061bbdae0 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -36,9 +36,15 @@ type PyDABs struct { // VEnvPath is path to the virtual environment. // - // Required if PyDABs is enabled. PyDABs will load the code in the specified - // environment. + // If enabled, PyDABs will execute code within this environment. If disabled, + // it defaults to using the Python interpreter available in the current shell. VEnvPath string `json:"venv_path,omitempty"` + + // Import contains a list Python packages with PyDABs code. + // + // These packages are imported to discover resources, resource generators, and mutators. + // This list can include namespace packages, which causes the import of nested packages. + Import []string `json:"import,omitempty"` } type Command string diff --git a/bundle/config/generate/job.go b/bundle/config/generate/job.go index 3ab5e0122..28bc86412 100644 --- a/bundle/config/generate/job.go +++ b/bundle/config/generate/job.go @@ -22,7 +22,7 @@ func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) { tasks = append(tasks, v) } // We're using location lines to define the order of keys in exported YAML. - value["tasks"] = dyn.NewValue(tasks, dyn.Location{Line: jobOrder.Get("tasks")}) + value["tasks"] = dyn.NewValue(tasks, []dyn.Location{{Line: jobOrder.Get("tasks")}}) } return yamlsaver.ConvertToMapValue(job.Settings, jobOrder, []string{"format", "new_cluster", "existing_cluster_id"}, value) diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go new file mode 100644 index 000000000..28d015c10 --- /dev/null +++ b/bundle/config/mutator/apply_presets.go @@ -0,0 +1,208 @@ +package mutator + +import ( + "context" + "path" + "slices" + "sort" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type applyPresets struct{} + +// Apply all presets, e.g. the prefix presets that +// adds a prefix to all names of all resources. +func ApplyPresets() *applyPresets { + return &applyPresets{} +} + +type Tag struct { + Key string + Value string +} + +func (m *applyPresets) Name() string { + return "ApplyPresets" +} + +func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if d := validatePauseStatus(b); d != nil { + return d + } + + r := b.Config.Resources + t := b.Config.Presets + prefix := t.NamePrefix + tags := toTagArray(t.Tags) + + // Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus + for _, j := range r.Jobs { + j.Name = prefix + j.Name + if j.Tags == nil { + j.Tags = make(map[string]string) + } + for _, tag := range tags { + if j.Tags[tag.Key] == "" { + j.Tags[tag.Key] = tag.Value + } + } + if j.MaxConcurrentRuns == 0 { + j.MaxConcurrentRuns = t.JobsMaxConcurrentRuns + } + if t.TriggerPauseStatus != "" { + paused := jobs.PauseStatusPaused + if t.TriggerPauseStatus == config.Unpaused { + paused = jobs.PauseStatusUnpaused + } + + if j.Schedule != nil && j.Schedule.PauseStatus == "" { + j.Schedule.PauseStatus = paused + } + if j.Continuous != nil && j.Continuous.PauseStatus == "" { + j.Continuous.PauseStatus = paused + } + if j.Trigger != nil && j.Trigger.PauseStatus == "" { + j.Trigger.PauseStatus = paused + } + } + } + + // Pipelines presets: Prefix, PipelinesDevelopment + for i := range r.Pipelines { + r.Pipelines[i].Name = prefix + r.Pipelines[i].Name + if config.IsExplicitlyEnabled(t.PipelinesDevelopment) { + r.Pipelines[i].Development = true + } + if t.TriggerPauseStatus == config.Paused { + r.Pipelines[i].Continuous = false + } + + // As of 2024-06, pipelines don't yet support tags + } + + // Models presets: Prefix, Tags + for _, m := range r.Models { + m.Name = prefix + m.Name + for _, t := range tags { + exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool { + return modelTag.Key == t.Key + }) + if !exists { + // Only add this tag if the resource didn't include any tag that overrides its value. + m.Tags = append(m.Tags, ml.ModelTag{Key: t.Key, Value: t.Value}) + } + } + } + + // Experiments presets: Prefix, Tags + for _, e := range r.Experiments { + filepath := e.Name + dir := path.Dir(filepath) + base := path.Base(filepath) + if dir == "." { + e.Name = prefix + base + } else { + e.Name = dir + "/" + prefix + base + } + for _, t := range tags { + exists := false + for _, experimentTag := range e.Tags { + if experimentTag.Key == t.Key { + exists = true + break + } + } + if !exists { + e.Tags = append(e.Tags, ml.ExperimentTag{Key: t.Key, Value: t.Value}) + } + } + } + + // Model serving endpoint presets: Prefix + for i := range r.ModelServingEndpoints { + r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name + + // As of 2024-06, model serving endpoints don't yet support tags + } + + // Registered models presets: Prefix + for i := range r.RegisteredModels { + r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name + + // As of 2024-06, registered models don't yet support tags + } + + // Quality monitors presets: Prefix + if t.TriggerPauseStatus == config.Paused { + for i := range r.QualityMonitors { + // Remove all schedules from monitors, since they don't support pausing/unpausing. + // Quality monitors might support the "pause" property in the future, so at the + // CLI level we do respect that property if it is set to "unpaused." + if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + r.QualityMonitors[i].Schedule = nil + } + } + } + + // Schemas: Prefix + for i := range r.Schemas { + r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name + // HTTP API for schemas doesn't yet support tags. It's only supported in + // the Databricks UI and via the SQL API. + } + + return nil +} + +func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { + p := b.Config.Presets.TriggerPauseStatus + if p == "" || p == config.Paused || p == config.Unpaused { + return nil + } + return diag.Diagnostics{{ + Summary: "Invalid value for trigger_pause_status, should be PAUSED or UNPAUSED", + Severity: diag.Error, + Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + }} +} + +// toTagArray converts a map of tags to an array of tags. +// We sort tags so ensure stable ordering. +func toTagArray(tags map[string]string) []Tag { + var tagArray []Tag + if tags == nil { + return tagArray + } + for key, value := range tags { + tagArray = append(tagArray, Tag{Key: key, Value: value}) + } + sort.Slice(tagArray, func(i, j int) bool { + return tagArray[i].Key < tagArray[j].Key + }) + return tagArray +} + +// normalizePrefix prefixes strings like '[dev lennart] ' to 'dev_lennart_'. +// We leave unicode letters and numbers but remove all "special characters." +func normalizePrefix(prefix string) string { + prefix = strings.ReplaceAll(prefix, "[", "") + prefix = strings.Trim(prefix, " ") + + // If the prefix ends with a ']', we add an underscore to the end. + // This makes sure that we get names like "dev_user_endpoint" instead of "dev_userendpoint" + suffix := "" + if strings.HasSuffix(prefix, "]") { + suffix = "_" + } + + return textutil.NormalizeString(prefix) + suffix +} diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go new file mode 100644 index 000000000..ab2478aee --- /dev/null +++ b/bundle/config/mutator/apply_presets_test.go @@ -0,0 +1,253 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestApplyPresetsPrefix(t *testing.T) { + tests := []struct { + name string + prefix string + job *resources.Job + want string + }{ + { + name: "add prefix to job", + prefix: "prefix-", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + }, + }, + want: "prefix-job1", + }, + { + name: "add empty prefix to job", + prefix: "", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + }, + }, + want: "job1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Jobs["job1"].Name) + }) + } +} + +func TestApplyPresetsPrefixForUcSchema(t *testing.T) { + tests := []struct { + name string + prefix string + schema *resources.Schema + want string + }{ + { + name: "add prefix to schema", + prefix: "[prefix]", + schema: &resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "schema1", + }, + }, + want: "prefix_schema1", + }, + { + name: "add empty prefix to schema", + prefix: "", + schema: &resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "schema1", + }, + }, + want: "schema1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "schema1": tt.schema, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Schemas["schema1"].Name) + }) + } +} + +func TestApplyPresetsTags(t *testing.T) { + tests := []struct { + name string + tags map[string]string + job *resources.Job + want map[string]string + }{ + { + name: "add tags to job", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: nil, + }, + }, + want: map[string]string{"env": "dev"}, + }, + { + name: "merge tags with existing job tags", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: map[string]string{"team": "data"}, + }, + }, + want: map[string]string{"env": "dev", "team": "data"}, + }, + { + name: "don't override existing job tags", + tags: map[string]string{"env": "dev"}, + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + Tags: map[string]string{"env": "prod"}, + }, + }, + want: map[string]string{"env": "prod"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + Tags: tt.tags, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + tags := b.Config.Resources.Jobs["job1"].Tags + require.Equal(t, tt.want, tags) + }) + } +} + +func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) { + tests := []struct { + name string + job *resources.Job + setting int + want int + }{ + { + name: "set max concurrent runs", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + MaxConcurrentRuns: 0, + }, + }, + setting: 5, + want: 5, + }, + { + name: "do not override existing max concurrent runs", + job: &resources.Job{ + JobSettings: &jobs.JobSettings{ + Name: "job1", + MaxConcurrentRuns: 3, + }, + }, + setting: 5, + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": tt.job, + }, + }, + Presets: config.Presets{ + JobsMaxConcurrentRuns: tt.setting, + }, + }, + } + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) + }) + } +} diff --git a/bundle/config/mutator/configure_wsfs.go b/bundle/config/mutator/configure_wsfs.go index 17af4828f..1d1bec582 100644 --- a/bundle/config/mutator/configure_wsfs.go +++ b/bundle/config/mutator/configure_wsfs.go @@ -24,7 +24,7 @@ func (m *configureWSFS) Name() string { } func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - root := b.BundleRoot.Native() + root := b.SyncRoot.Native() // The bundle root must be located in /Workspace/ if !strings.HasPrefix(root, "/Workspace/") { @@ -39,12 +39,12 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno // If so, swap out vfs.Path instance of the sync root with one that // makes all Workspace File System interactions extension aware. p, err := vfs.NewFilerPath(ctx, root, func(path string) (filer.Filer, error) { - return filer.NewWorkspaceFilesExtensionsClient(b.WorkspaceClient(), path) + return filer.NewReadOnlyWorkspaceFilesExtensionsClient(b.WorkspaceClient(), path) }) if err != nil { return diag.FromErr(err) } - b.BundleRoot = p + b.SyncRoot = p return nil } diff --git a/bundle/config/mutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/expand_pipeline_glob_paths.go index 268d8fa48..5703332fa 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths.go @@ -59,7 +59,7 @@ func (m *expandPipelineGlobPaths) expandLibrary(v dyn.Value) ([]dyn.Value, error if err != nil { return nil, err } - nv, err := dyn.SetByPath(v, p, dyn.NewValue(m, pv.Location())) + nv, err := dyn.SetByPath(v, p, dyn.NewValue(m, pv.Locations())) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (m *expandPipelineGlobPaths) expandSequence(p dyn.Path, v dyn.Value) (dyn.V vs = append(vs, v...) } - return dyn.NewValue(vs, v.Location()), nil + return dyn.NewValue(vs, v.Locations()), nil } func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/if.go b/bundle/config/mutator/if.go deleted file mode 100644 index 1b7856b3c..000000000 --- a/bundle/config/mutator/if.go +++ /dev/null @@ -1,36 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" -) - -type ifMutator struct { - condition func(*bundle.Bundle) bool - onTrueMutator bundle.Mutator - onFalseMutator bundle.Mutator -} - -func If( - condition func(*bundle.Bundle) bool, - onTrueMutator bundle.Mutator, - onFalseMutator bundle.Mutator, -) bundle.Mutator { - return &ifMutator{ - condition, onTrueMutator, onFalseMutator, - } -} - -func (m *ifMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.condition(b) { - return bundle.Apply(ctx, b, m.onTrueMutator) - } else { - return bundle.Apply(ctx, b, m.onFalseMutator) - } -} - -func (m *ifMutator) Name() string { - return "If" -} diff --git a/bundle/config/mutator/merge_job_parameters.go b/bundle/config/mutator/merge_job_parameters.go new file mode 100644 index 000000000..51a919d98 --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters.go @@ -0,0 +1,45 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" +) + +type mergeJobParameters struct{} + +func MergeJobParameters() bundle.Mutator { + return &mergeJobParameters{} +} + +func (m *mergeJobParameters) Name() string { + return "MergeJobParameters" +} + +func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { + switch v.Kind() { + case dyn.KindInvalid, dyn.KindNil: + return "" + case dyn.KindString: + return v.MustString() + default: + panic("task key must be a string") + } +} + +func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + if v.Kind() == dyn.KindNil { + return v, nil + } + + return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { + return dyn.Map(job, "parameters", merge.ElementsByKey("name", m.parameterNameString)) + })) + }) + + return diag.FromErr(err) +} diff --git a/bundle/config/mutator/merge_job_parameters_test.go b/bundle/config/mutator/merge_job_parameters_test.go new file mode 100644 index 000000000..f03dea734 --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters_test.go @@ -0,0 +1,80 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestMergeJobParameters(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Name: "foo", + Default: "v1", + }, + { + Name: "bar", + Default: "v1", + }, + { + Name: "foo", + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + + j := b.Config.Resources.Jobs["foo"] + + assert.Len(t, j.Parameters, 2) + assert.Equal(t, "foo", j.Parameters[0].Name) + assert.Equal(t, "v2", j.Parameters[0].Default) + assert.Equal(t, "bar", j.Parameters[1].Name) + assert.Equal(t, "v1", j.Parameters[1].Default) +} + +func TestMergeJobParametersWithNilKey(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Default: "v1", + }, + { + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + assert.Len(t, b.Config.Resources.Jobs["foo"].Parameters, 1) +} diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 52f85eeb8..0458beff4 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/scripts" ) @@ -26,5 +27,9 @@ func DefaultMutators() []bundle.Mutator { DefineDefaultTarget(), LoadGitDetails(), pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad), + + // Note: This mutator must run before the target overrides are merged. + // See the mutator for more details. + validate.UniqueResourceKeys(), } } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index b50716fd6..92ed28689 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -2,17 +2,14 @@ package mutator import ( "context" - "path" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/databricks/databricks-sdk-go/service/ml" ) type processTargetMode struct{} @@ -30,95 +27,75 @@ func (m *processTargetMode) Name() string { // Mark all resources as being for 'development' purposes, i.e. // changing their their name, adding tags, and (in the future) // marking them as 'hidden' in the UI. -func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { if !b.Config.Bundle.Deployment.Lock.IsExplicitlyEnabled() { log.Infof(ctx, "Development mode: disabling deployment lock since bundle.deployment.lock.enabled is not set to true") disabled := false b.Config.Bundle.Deployment.Lock.Enabled = &disabled } - r := b.Config.Resources + t := &b.Config.Presets shortName := b.Config.Workspace.CurrentUser.ShortName - prefix := "[dev " + shortName + "] " - // Generate a normalized version of the short name that can be used as a tag value. - tagValue := b.Tagging.NormalizeValue(shortName) - - for i := range r.Jobs { - r.Jobs[i].Name = prefix + r.Jobs[i].Name - if r.Jobs[i].Tags == nil { - r.Jobs[i].Tags = make(map[string]string) - } - r.Jobs[i].Tags["dev"] = tagValue - if r.Jobs[i].MaxConcurrentRuns == 0 { - r.Jobs[i].MaxConcurrentRuns = developmentConcurrentRuns - } - - // Pause each job. As an exception, we don't pause jobs that are explicitly - // marked as "unpaused". This allows users to override the default behavior - // of the development mode. - if r.Jobs[i].Schedule != nil && r.Jobs[i].Schedule.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Schedule.PauseStatus = jobs.PauseStatusPaused - } - if r.Jobs[i].Continuous != nil && r.Jobs[i].Continuous.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Continuous.PauseStatus = jobs.PauseStatusPaused - } - if r.Jobs[i].Trigger != nil && r.Jobs[i].Trigger.PauseStatus != jobs.PauseStatusUnpaused { - r.Jobs[i].Trigger.PauseStatus = jobs.PauseStatusPaused - } + if t.NamePrefix == "" { + t.NamePrefix = "[dev " + shortName + "] " } - for i := range r.Pipelines { - r.Pipelines[i].Name = prefix + r.Pipelines[i].Name - r.Pipelines[i].Development = true - // (pipelines don't yet support tags) + if t.Tags == nil { + t.Tags = map[string]string{} + } + _, exists := t.Tags["dev"] + if !exists { + t.Tags["dev"] = b.Tagging.NormalizeValue(shortName) } - for i := range r.Models { - r.Models[i].Name = prefix + r.Models[i].Name - r.Models[i].Tags = append(r.Models[i].Tags, ml.ModelTag{Key: "dev", Value: tagValue}) + if t.JobsMaxConcurrentRuns == 0 { + t.JobsMaxConcurrentRuns = developmentConcurrentRuns } - for i := range r.Experiments { - filepath := r.Experiments[i].Name - dir := path.Dir(filepath) - base := path.Base(filepath) - if dir == "." { - r.Experiments[i].Name = prefix + base - } else { - r.Experiments[i].Name = dir + "/" + prefix + base - } - r.Experiments[i].Tags = append(r.Experiments[i].Tags, ml.ExperimentTag{Key: "dev", Value: tagValue}) + if t.TriggerPauseStatus == "" { + t.TriggerPauseStatus = config.Paused } - for i := range r.ModelServingEndpoints { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.ModelServingEndpoints[i].Name = prefix + r.ModelServingEndpoints[i].Name - // (model serving doesn't yet support tags) + if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) { + enabled := true + t.PipelinesDevelopment = &enabled } - - for i := range r.RegisteredModels { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.RegisteredModels[i].Name = prefix + r.RegisteredModels[i].Name - // (registered models in Unity Catalog don't yet support tags) - } - - for i := range r.QualityMonitors { - // Remove all schedules from monitors, since they don't support pausing/unpausing. - // Quality monitors might support the "pause" property in the future, so at the - // CLI level we do respect that property if it is set to "unpaused". - if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { - r.QualityMonitors[i].Schedule = nil - } - } - - return nil } func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { + p := b.Config.Presets + u := b.Config.Workspace.CurrentUser + + // Make sure presets don't set the trigger status to UNPAUSED; + // this could be surprising since most users (and tools) expect triggers + // to be paused in development. + // (Note that there still is an exceptional case where users set the trigger + // status to UNPAUSED at the level of an individual object, whic hwas + // historically allowed.) + if p.TriggerPauseStatus == config.Unpaused { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default", + Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, + }} + } + + // Make sure this development copy has unique names and paths to avoid conflicts if path := findNonUserPath(b); path != "" { return diag.Errorf("%s must start with '~/' or contain the current username when using 'mode: development'", path) } + if p.NamePrefix != "" && !strings.Contains(p.NamePrefix, u.ShortName) && !strings.Contains(p.NamePrefix, u.UserName) { + // Resources such as pipelines require a unique name, e.g. '[dev steve] my_pipeline'. + // For this reason we require the name prefix to contain the current username; + // it's a pitfall for users if they don't include it and later find out that + // only a single user can do development deployments. + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", + Locations: []dyn.Location{b.Config.GetLocation("presets.name_prefix")}, + }} + } return nil } @@ -175,10 +152,11 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di switch b.Config.Bundle.Mode { case config.Development: diags := validateDevelopmentMode(b) - if diags != nil { + if diags.HasError() { return diags } - return transformDevelopmentMode(ctx, b) + transformDevelopmentMode(ctx, b) + return diags case config.Production: isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName) return validateProductionMode(ctx, b, isPrincipal) diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 03da64e77..1c8671b4c 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -51,6 +52,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Schedule: &jobs.CronSchedule{ QuartzCronExpression: "* * * * *", }, + Tags: map[string]string{"existing": "tag"}, }, }, "job2": { @@ -82,7 +84,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, Pipelines: map[string]*resources.Pipeline{ - "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}}, + "pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}}, @@ -114,6 +116,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Schemas: map[string]*resources.Schema{ + "schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}}, + }, }, }, // Use AWS implementation for testing. @@ -126,12 +131,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { func TestProcessTargetModeDevelopment(t *testing.T) { b := mockBundle(config.Development) - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Job 1 assert.Equal(t, "[dev lennart] job1", b.Config.Resources.Jobs["job1"].Name) + assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["existing"], "tag") assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) @@ -142,6 +148,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Pipeline 1 assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) + assert.Equal(t, false, b.Config.Resources.Pipelines["pipeline1"].Continuous) assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) // Experiment 1 @@ -167,6 +174,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) assert.Nil(t, b.Config.Resources.QualityMonitors["qualityMonitor2"].Schedule) assert.Equal(t, catalog.MonitorCronSchedulePauseStatusUnpaused, b.Config.Resources.QualityMonitors["qualityMonitor3"].Schedule.PauseStatus) + + // Schema 1 + assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { @@ -176,7 +186,8 @@ func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place. @@ -190,7 +201,8 @@ func TestProcessTargetModeDevelopmentTagNormalizationForAzure(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place (Azure allows more characters than AWS). @@ -204,17 +216,53 @@ func TestProcessTargetModeDevelopmentTagNormalizationForGcp(t *testing.T) { }) b.Config.Workspace.CurrentUser.ShortName = "Héllö wörld?!" - diags := bundle.Apply(context.Background(), b, ProcessTargetMode()) + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) // Assert that tag normalization took place. assert.Equal(t, "Hello_world", b.Config.Resources.Jobs["job1"].Tags["dev"]) } +func TestValidateDevelopmentMode(t *testing.T) { + // Test with a valid development mode bundle + b := mockBundle(config.Development) + diags := validateDevelopmentMode(b) + require.NoError(t, diags.Error()) + + // Test with a bundle that has a non-user path + b.Config.Workspace.RootPath = "/Shared/.bundle/x/y/state" + diags = validateDevelopmentMode(b) + require.ErrorContains(t, diags.Error(), "root_path") + + // Test with a bundle that has an unpaused trigger pause status + b = mockBundle(config.Development) + b.Config.Presets.TriggerPauseStatus = config.Unpaused + diags = validateDevelopmentMode(b) + require.ErrorContains(t, diags.Error(), "UNPAUSED") + + // Test with a bundle that has a prefix not containing the username or short name + b = mockBundle(config.Development) + b.Config.Presets.NamePrefix = "[prod]" + diags = validateDevelopmentMode(b) + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Contains(t, diags[0].Summary, "") + + // Test with a bundle that has valid user paths + b = mockBundle(config.Development) + b.Config.Workspace.RootPath = "/Users/lennart@company.com/.bundle/x/y/state" + b.Config.Workspace.StatePath = "/Users/lennart@company.com/.bundle/x/y/state" + b.Config.Workspace.FilePath = "/Users/lennart@company.com/.bundle/x/y/files" + b.Config.Workspace.ArtifactPath = "/Users/lennart@company.com/.bundle/x/y/artifacts" + diags = validateDevelopmentMode(b) + require.NoError(t, diags.Error()) +} + func TestProcessTargetModeDefault(t *testing.T) { b := mockBundle("") - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name) @@ -300,7 +348,7 @@ func TestAllResourcesMocked(t *testing.T) { func TestAllResourcesRenamed(t *testing.T) { b := mockBundle(config.Development) - m := ProcessTargetMode() + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) diags := bundle.Apply(context.Background(), b, m) require.NoError(t, diags.Error()) @@ -330,8 +378,7 @@ func TestDisableLocking(t *testing.T) { ctx := context.Background() b := mockBundle(config.Development) - err := bundle.Apply(ctx, b, ProcessTargetMode()) - require.Nil(t, err) + transformDevelopmentMode(ctx, b) assert.False(t, b.Config.Bundle.Deployment.Lock.IsEnabled()) } @@ -341,7 +388,97 @@ func TestDisableLockingDisabled(t *testing.T) { explicitlyEnabled := true b.Config.Bundle.Deployment.Lock.Enabled = &explicitlyEnabled - err := bundle.Apply(ctx, b, ProcessTargetMode()) - require.Nil(t, err) + transformDevelopmentMode(ctx, b) assert.True(t, b.Config.Bundle.Deployment.Lock.IsEnabled(), "Deployment lock should remain enabled in development mode when explicitly enabled") } + +func TestPrefixAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.NamePrefix = "custom_lennart_deploy_" + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "custom_lennart_deploy_job1", b.Config.Resources.Jobs["job1"].Name) +} + +func TestTagsAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = map[string]string{ + "custom": "tag", + "dev": "foo", + } + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "tag", b.Config.Resources.Jobs["job1"].Tags["custom"]) + assert.Equal(t, "foo", b.Config.Resources.Jobs["job1"].Tags["dev"]) +} + +func TestTagsNil(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = nil + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) +} + +func TestTagsEmptySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.Tags = map[string]string{} + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, "lennart", b.Config.Resources.Jobs["job2"].Tags["dev"]) +} + +func TestJobsMaxConcurrentRunsAlreadySet(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.JobsMaxConcurrentRuns = 10 + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, 10, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) +} + +func TestJobsMaxConcurrentRunsDisabled(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.JobsMaxConcurrentRuns = 1 + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.Equal(t, 1, b.Config.Resources.Jobs["job1"].MaxConcurrentRuns) +} + +func TestTriggerPauseStatusWhenUnpaused(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Presets.TriggerPauseStatus = config.Unpaused + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.ErrorContains(t, diags.Error(), "target with 'mode: development' cannot set trigger pause status to UNPAUSED by default") +} + +func TestPipelinesDevelopmentDisabled(t *testing.T) { + b := mockBundle(config.Development) + notEnabled := false + b.Config.Presets.PipelinesDevelopment = ¬Enabled + + m := bundle.Seq(ProcessTargetMode(), ApplyPresets()) + diags := bundle.Apply(context.Background(), b, m) + require.NoError(t, diags.Error()) + + assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) +} diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index b8efc9ef7..12822065b 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -54,13 +54,23 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { if err != nil { return nil, fmt.Errorf("failed to parse path: %s", err) } + var paths []dyn.Path + if path != nil { + paths = []dyn.Path{path} + } + + var locations []dyn.Location + location := convertPythonLocation(parsedLine.Location) + if location != (dyn.Location{}) { + locations = append(locations, location) + } diag := diag.Diagnostic{ - Severity: severity, - Summary: parsedLine.Summary, - Detail: parsedLine.Detail, - Location: convertPythonLocation(parsedLine.Location), - Path: path, + Severity: severity, + Summary: parsedLine.Summary, + Detail: parsedLine.Detail, + Locations: locations, + Paths: paths, } diags = diags.Append(diag) diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 7b66e2537..b73b0f73c 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -39,10 +39,12 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Location: dyn.Location{ - File: "src/examples/file.py", - Line: 1, - Column: 2, + Locations: []dyn.Location{ + { + File: "src/examples/file.py", + Line: 1, + Column: 2, + }, }, }, }, @@ -54,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Path: dyn.MustPathFromString("resources.jobs.job0.name"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.job0.name")}, }, }, }, diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index f9febe5b5..4f44df0a9 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -7,8 +7,8 @@ import ( "fmt" "os" "path/filepath" - "runtime" + "github.com/databricks/cli/libs/python" "github.com/databricks/databricks-sdk-go/logger" "github.com/databricks/cli/bundle/env" @@ -86,23 +86,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return nil } - if experimental.PyDABs.VEnvPath == "" { - return diag.Errorf("\"experimental.pydabs.enabled\" can only be used when \"experimental.pydabs.venv_path\" is set") - } - // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' var mutateDiags diag.Diagnostics var mutateDiagsHasError = errors.New("unexpected error") err := b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { - pythonPath := interpreterPath(experimental.PyDABs.VEnvPath) + pythonPath, err := detectExecutable(ctx, experimental.PyDABs.VEnvPath) - if _, err := os.Stat(pythonPath); err != nil { - if os.IsNotExist(err) { - return dyn.InvalidValue, fmt.Errorf("can't find %q, check if venv is created", pythonPath) - } else { - return dyn.InvalidValue, fmt.Errorf("can't find %q: %w", pythonPath, err) - } + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to get Python interpreter path: %w", err) } cacheDir, err := createCacheDir(ctx) @@ -423,11 +415,16 @@ func isOmitemptyDelete(left dyn.Value) bool { } } -// interpreterPath returns platform-specific path to Python interpreter in the virtual environment. -func interpreterPath(venvPath string) string { - if runtime.GOOS == "windows" { - return filepath.Join(venvPath, "Scripts", "python3.exe") - } else { - return filepath.Join(venvPath, "bin", "python3") +// detectExecutable lookups Python interpreter in virtual environment, or if not set, in PATH. +func detectExecutable(ctx context.Context, venvPath string) (string, error) { + if venvPath == "" { + interpreter, err := python.DetectExecutable(ctx) + if err != nil { + return "", err + } + + return interpreter, nil } + + return python.DetectVEnvExecutable(venvPath) } diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 9a0ed8c3a..ea02d1ced 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -97,11 +97,14 @@ func TestPythonMutator_load(t *testing.T) { assert.Equal(t, 1, len(diags)) assert.Equal(t, "job doesn't have any tasks", diags[0].Summary) - assert.Equal(t, dyn.Location{ - File: "src/examples/file.py", - Line: 10, - Column: 5, - }, diags[0].Location) + assert.Equal(t, []dyn.Location{ + { + File: "src/examples/file.py", + Line: 10, + Column: 5, + }, + }, diags[0].Locations) + } func TestPythonMutator_load_disallowed(t *testing.T) { @@ -279,7 +282,7 @@ func TestPythonMutator_venvRequired(t *testing.T) { } func TestPythonMutator_venvNotFound(t *testing.T) { - expectedError := fmt.Sprintf("can't find %q, check if venv is created", interpreterPath("bad_path")) + expectedError := fmt.Sprintf("failed to get Python interpreter path: can't find %q, check if virtualenv is created", interpreterPath("bad_path")) b := loadYaml("databricks.yml", ` experimental: @@ -305,8 +308,8 @@ type createOverrideVisitorTestCase struct { } func TestCreateOverrideVisitor(t *testing.T) { - left := dyn.NewValue(42, dyn.Location{}) - right := dyn.NewValue(1337, dyn.Location{}) + left := dyn.V(42) + right := dyn.V(1337) testCases := []createOverrideVisitorTestCase{ { @@ -470,21 +473,21 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { // this is not happening, but adding for completeness name: "undo delete of empty variables", path: dyn.MustPathFromString("variables"), - left: dyn.NewValue([]dyn.Value{}, location), + left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "undo delete of empty job clusters", path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{}, location), + left: dyn.NewValue([]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "allow delete of non-empty job clusters", path: dyn.MustPathFromString("resources.jobs.job0.job_clusters"), - left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", location)}, location), + left: dyn.NewValue([]dyn.Value{dyn.NewValue("abc", []dyn.Location{location})}, []dyn.Location{location}), expectedErr: nil, // deletions aren't allowed in 'load' phase phases: []phase{PythonMutatorPhaseInit}, @@ -492,17 +495,15 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { { name: "undo delete of empty tags", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue(map[string]dyn.Value{}, location), + left: dyn.NewValue(map[string]dyn.Value{}, []dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, { name: "allow delete of non-empty tags", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NewValue( - map[string]dyn.Value{"dev": dyn.NewValue("true", location)}, - location, - ), + left: dyn.NewValue(map[string]dyn.Value{"dev": dyn.NewValue("true", []dyn.Location{location})}, []dyn.Location{location}), + expectedErr: nil, // deletions aren't allowed in 'load' phase phases: []phase{PythonMutatorPhaseInit}, @@ -510,7 +511,7 @@ func TestCreateOverrideVisitor_omitempty(t *testing.T) { { name: "undo delete of nil", path: dyn.MustPathFromString("resources.jobs.job0.tags"), - left: dyn.NilValue.WithLocation(location), + left: dyn.NilValue.WithLocations([]dyn.Location{location}), expectedErr: merge.ErrOverrideUndoDelete, phases: allPhases, }, @@ -595,9 +596,7 @@ func loadYaml(name string, content string) *bundle.Bundle { } } -func withFakeVEnv(t *testing.T, path string) { - interpreterPath := interpreterPath(path) - +func withFakeVEnv(t *testing.T, venvPath string) { cwd, err := os.Getwd() if err != nil { panic(err) @@ -607,6 +606,8 @@ func withFakeVEnv(t *testing.T, path string) { panic(err) } + interpreterPath := interpreterPath(venvPath) + err = os.MkdirAll(filepath.Dir(interpreterPath), 0755) if err != nil { panic(err) @@ -617,9 +618,22 @@ func withFakeVEnv(t *testing.T, path string) { panic(err) } + err = os.WriteFile(filepath.Join(venvPath, "pyvenv.cfg"), []byte(""), 0755) + if err != nil { + panic(err) + } + t.Cleanup(func() { if err := os.Chdir(cwd); err != nil { panic(err) } }) } + +func interpreterPath(venvPath string) string { + if runtime.GOOS == "windows" { + return filepath.Join(venvPath, "Scripts", "python3.exe") + } else { + return filepath.Join(venvPath, "bin", "python3") + } +} diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 85db79797..888714abe 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -38,13 +38,17 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { return dyn.InvalidValue, err } - return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Location()), nil + return dyn.NewValue(filepath.Join(rel, v.MustString()), v.Locations()), nil } } func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { + v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + if err != nil { + return dyn.InvalidValue, err + } v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath))) if err != nil { return dyn.InvalidValue, err diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index 56ada19e6..fa7f124b7 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -17,6 +17,10 @@ func TestRewriteSyncPathsRelative(t *testing.T) { RootPath: ".", Config: config.Root{ Sync: config.Sync{ + Paths: []string{ + ".", + "../common", + }, Include: []string{ "foo", "bar", @@ -29,6 +33,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) { }, } + bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml") + bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml") bundletest.SetLocation(b, "sync.include[0]", "./file.yml") bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml") bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml") @@ -37,6 +43,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1]) assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0]) assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1]) assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0]) @@ -48,6 +56,10 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { RootPath: "/tmp/dir", Config: config.Root{ Sync: config.Sync{ + Paths: []string{ + ".", + "../common", + }, Include: []string{ "foo", "bar", @@ -60,6 +72,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { }, } + bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml") + bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml") bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml") bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml") bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml") @@ -68,6 +82,8 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.Clean("."), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.Clean("../common"), b.Config.Sync.Paths[1]) assert.Equal(t, filepath.Clean("foo"), b.Config.Sync.Include[0]) assert.Equal(t, filepath.Clean("a/bar"), b.Config.Sync.Include[1]) assert.Equal(t, filepath.Clean("a/b/baz"), b.Config.Sync.Exclude[0]) diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index e4bdbaa1a..6b3069d44 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -35,8 +35,8 @@ func reportRunAsNotSupported(resourceType string, location dyn.Location, current Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+ "Current identity: %s. Run as identity: %s.\n"+ "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser), - Location: location, - Severity: diag.Error, + Locations: []dyn.Location{location}, + Severity: diag.Error, }} } @@ -44,9 +44,9 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} neitherSpecifiedErr := diag.Diagnostics{{ - Summary: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", - Location: b.Config.GetLocation("run_as"), - Severity: diag.Error, + Summary: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, }} // Fail fast if neither service_principal_name nor user_name are specified, but the @@ -64,9 +64,9 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { if runAs.UserName != "" && runAs.ServicePrincipalName != "" { diags = diags.Extend(diag.Diagnostics{{ - Summary: "run_as section cannot specify both user_name and service_principal_name", - Location: b.Config.GetLocation("run_as"), - Severity: diag.Error, + Summary: "run_as section cannot specify both user_name and service_principal_name", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, }}) } @@ -172,10 +172,10 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { setRunAsForJobs(b) return diag.Diagnostics{ { - Severity: diag.Warning, - Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Path: dyn.MustPathFromString("experimental.use_legacy_run_as"), - Location: b.Config.GetLocation("experimental.use_legacy_run_as"), + Severity: diag.Warning, + Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", + Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, + Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } } diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index 5ad4b6933..5b674609b 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -39,6 +39,7 @@ func allResourceTypes(t *testing.T) []string { "pipelines", "quality_monitors", "registered_models", + "schemas", }, resourceTypes, ) @@ -136,6 +137,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { "models", "registered_models", "experiments", + "schemas", } base := config.Root{ diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index b3a9cf400..47ce2ad03 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -2,10 +2,12 @@ package mutator import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" ) @@ -21,52 +23,63 @@ func (m *setVariables) Name() string { return "SetVariables" } -func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Diagnostics { +func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) { // case: variable already has value initialized, so skip - if v.HasValue() { - return nil + if variable.HasValue() { + return v, nil } // case: read and set variable value from process environment envVarName := bundleVarPrefix + name if val, ok := env.Lookup(ctx, envVarName); ok { - if v.IsComplex() { - return diag.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) + if variable.IsComplex() { + return dyn.InvalidValue, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name) } - err := v.Set(val) + v, err := dyn.Set(v, "value", dyn.V(val)) if err != nil { - return diag.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) + return dyn.InvalidValue, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err) } - return nil + return v, nil } // case: Defined a variable for named lookup for a resource // It will be resolved later in ResolveResourceReferences mutator - if v.Lookup != nil { - return nil + if variable.Lookup != nil { + return v, nil } // case: Set the variable to its default value - if v.HasDefault() { - err := v.Set(v.Default) + if variable.HasDefault() { + vDefault, err := dyn.Get(v, "default") if err != nil { - return diag.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, v.Default, name, err) + return dyn.InvalidValue, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err) } - return nil + + v, err := dyn.Set(v, "value", vDefault) + if err != nil { + return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err) + } + return v, nil } // We should have had a value to set for the variable at this point. - return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - for name, variable := range b.Config.Variables { - diags = diags.Extend(setVariable(ctx, variable, name)) - if diags.HasError() { - return diags - } - } - return diags + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { + name := p[1].Key() + v, ok := b.Config.Variables[name] + if !ok { + return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) + } + + return setVariable(ctx, variable, v, name) + })) + }) + + return diag.FromErr(err) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index 65dedee97..d9719793f 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -7,6 +7,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,9 +22,14 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "process-env") } @@ -33,8 +40,14 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { Default: defaultVal, } - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "default") } @@ -49,8 +62,14 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the default value - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "assigned-value") } @@ -68,8 +87,14 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { // since a value is already assigned to the variable, it would not be overridden // by the value from environment - diags := setVariable(context.Background(), &variable, "foo") - require.NoError(t, diags.Error()) + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + v, err = setVariable(context.Background(), v, &variable, "foo") + require.NoError(t, err) + + err = convert.ToTyped(&variable, v) + require.NoError(t, err) assert.Equal(t, variable.Value, "assigned-value") } @@ -79,8 +104,11 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { } // fails because we could not resolve a value for the variable - diags := setVariable(context.Background(), &variable, "foo") - assert.ErrorContains(t, diags.Error(), "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + _, err = setVariable(context.Background(), v, &variable, "foo") + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") } func TestSetVariablesMutator(t *testing.T) { @@ -126,6 +154,9 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { // set value for variable as an environment variable t.Setenv("BUNDLE_VAR_foo", "process-env") - diags := setVariable(context.Background(), &variable, "foo") - assert.ErrorContains(t, diags.Error(), "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") + v, err := convert.FromTyped(variable, dyn.NilValue) + require.NoError(t, err) + + _, err = setVariable(context.Background(), v, &variable, "foo") + assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/mutator/sync_default_path.go b/bundle/config/mutator/sync_default_path.go new file mode 100644 index 000000000..8e14ce202 --- /dev/null +++ b/bundle/config/mutator/sync_default_path.go @@ -0,0 +1,48 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type syncDefaultPath struct{} + +// SyncDefaultPath configures the default sync path to be equal to the bundle root. +func SyncDefaultPath() bundle.Mutator { + return &syncDefaultPath{} +} + +func (m *syncDefaultPath) Name() string { + return "SyncDefaultPath" +} + +func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + isset := false + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + pv, _ := dyn.Get(v, "sync.paths") + + // If the sync paths field is already set, do nothing. + // We know it is set if its value is either a nil or a sequence (empty or not). + switch pv.Kind() { + case dyn.KindNil, dyn.KindSequence: + isset = true + } + + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + // If the sync paths field is already set, do nothing. + if isset { + return nil + } + + // Set the sync paths to the default value. + b.Config.Sync.Paths = []string{"."} + return nil +} diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go new file mode 100644 index 000000000..a37e913d2 --- /dev/null +++ b/bundle/config/mutator/sync_default_path_test.go @@ -0,0 +1,82 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{}, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) +} + +func TestSyncDefaultPath_SkipIfSet(t *testing.T) { + tcases := []struct { + name string + paths dyn.Value + expect []string + }{ + { + name: "nil", + paths: dyn.V(nil), + expect: nil, + }, + { + name: "empty sequence", + paths: dyn.V([]dyn.Value{}), + expect: []string{}, + }, + { + name: "non-empty sequence", + paths: dyn.V([]dyn.Value{dyn.V("something")}), + expect: []string{"something"}, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.name, func(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{}, + } + + diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping())) + if err != nil { + return dyn.InvalidValue, err + } + v, err = dyn.Set(v, "sync.paths", tcase.paths) + if err != nil { + return dyn.InvalidValue, err + } + return v, nil + }) + return diag.FromErr(err) + }) + require.NoError(t, diags.Error()) + + ctx := context.Background() + diags = bundle.Apply(ctx, b, mutator.SyncDefaultPath()) + require.NoError(t, diags.Error()) + + // If the sync paths field is already set, do nothing. + assert.Equal(t, tcase.expect, b.Config.Sync.Paths) + }) + } +} diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go new file mode 100644 index 000000000..012acf800 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root.go @@ -0,0 +1,120 @@ +package mutator + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" +) + +type syncInferRoot struct{} + +// SyncInferRoot is a mutator that infers the root path of all files to synchronize by looking at the +// paths in the sync configuration. The sync root may be different from the bundle root +// when the user intends to synchronize files outside the bundle root. +// +// The sync root can be equivalent to or an ancestor of the bundle root, but not a descendant. +// That is, the sync root must contain the bundle root. +// +// This mutator requires all sync-related paths and patterns to be relative to the bundle root path. +// This is done by the [RewriteSyncPaths] mutator, which must run before this mutator. +func SyncInferRoot() bundle.Mutator { + return &syncInferRoot{} +} + +func (m *syncInferRoot) Name() string { + return "SyncInferRoot" +} + +// computeRoot finds the innermost path that contains the specified path. +// It traverses up the root path until it finds the innermost path. +// If the path does not exist, it returns an empty string. +// +// See "sync_infer_root_internal_test.go" for examples. +func (m *syncInferRoot) computeRoot(path string, root string) string { + for !filepath.IsLocal(path) { + // Break if we have reached the root of the filesystem. + dir := filepath.Dir(root) + if dir == root { + return "" + } + + // Update the sync path as we navigate up the directory tree. + path = filepath.Join(filepath.Base(root), path) + + // Move up the directory tree. + root = dir + } + + return filepath.Clean(root) +} + +func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Use the bundle root path as the starting point for inferring the sync root path. + bundleRootPath := filepath.Clean(b.RootPath) + + // Infer the sync root path by looking at each one of the sync paths. + // Every sync path must be a descendant of the final sync root path. + syncRootPath := bundleRootPath + for _, path := range b.Config.Sync.Paths { + computedPath := m.computeRoot(path, bundleRootPath) + if computedPath == "" { + continue + } + + // Update sync root path if the computed root path is an ancestor of the current sync root path. + if len(computedPath) < len(syncRootPath) { + syncRootPath = computedPath + } + } + + // The new sync root path can only be an ancestor of the previous root path. + // Compute the relative path from the sync root to the bundle root. + rel, err := filepath.Rel(syncRootPath, bundleRootPath) + if err != nil { + return diag.FromErr(err) + } + + // If during computation of the sync root path we hit the root of the filesystem, + // then one or more of the sync paths are outside the filesystem. + // Check if this happened by verifying that none of the paths escape the root + // when joined with the sync root path. + for i, path := range b.Config.Sync.Paths { + if filepath.IsLocal(filepath.Join(rel, path)) { + continue + } + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("invalid sync path %q", path), + Locations: b.Config.GetLocations(fmt.Sprintf("sync.paths[%d]", i)), + Paths: []dyn.Path{dyn.NewPath(dyn.Key("sync"), dyn.Key("paths"), dyn.Index(i))}, + }) + } + + if diags.HasError() { + return diags + } + + // Update all paths in the sync configuration to be relative to the sync root. + for i, p := range b.Config.Sync.Paths { + b.Config.Sync.Paths[i] = filepath.Join(rel, p) + } + for i, p := range b.Config.Sync.Include { + b.Config.Sync.Include[i] = filepath.Join(rel, p) + } + for i, p := range b.Config.Sync.Exclude { + b.Config.Sync.Exclude[i] = filepath.Join(rel, p) + } + + // Configure the sync root path. + b.SyncRoot = vfs.MustNew(syncRootPath) + b.SyncRootPath = syncRootPath + return nil +} diff --git a/bundle/config/mutator/sync_infer_root_internal_test.go b/bundle/config/mutator/sync_infer_root_internal_test.go new file mode 100644 index 000000000..9ab9c88f4 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root_internal_test.go @@ -0,0 +1,72 @@ +package mutator + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSyncInferRootInternal_ComputeRoot(t *testing.T) { + s := syncInferRoot{} + + tcases := []struct { + path string + root string + out string + }{ + { + // Test that "." doesn't change the root. + path: ".", + root: "/tmp/some/dir", + out: "/tmp/some/dir", + }, + { + // Test that a subdirectory doesn't change the root. + path: "sub", + root: "/tmp/some/dir", + out: "/tmp/some/dir", + }, + { + // Test that a parent directory changes the root. + path: "../common", + root: "/tmp/some/dir", + out: "/tmp/some", + }, + { + // Test that a deeply nested parent directory changes the root. + path: "../../../../../../common", + root: "/tmp/some/dir/that/is/very/deeply/nested", + out: "/tmp/some", + }, + { + // Test that a parent directory changes the root at the filesystem root boundary. + path: "../common", + root: "/tmp", + out: "/", + }, + { + // Test that an invalid parent directory doesn't change the root and returns an empty string. + path: "../common", + root: "/", + out: "", + }, + { + // Test that the returned path is cleaned even if the root doesn't change. + path: "sub", + root: "/tmp/some/../dir", + out: "/tmp/dir", + }, + { + // Test that a relative root path also works. + path: "../common", + root: "foo/bar", + out: "foo", + }, + } + + for _, tc := range tcases { + out := s.computeRoot(tc.path, tc.root) + assert.Equal(t, tc.out, filepath.ToSlash(out)) + } +} diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go new file mode 100644 index 000000000..383e56769 --- /dev/null +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -0,0 +1,198 @@ +package mutator_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSyncInferRoot_NominalAbsolute(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + ".", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some/dir"), b.SyncRootPath) + + // Check that the paths are unchanged. + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include) + assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_NominalRelative(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "./some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + ".", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("some/dir"), b.SyncRootPath) + + // Check that the paths are unchanged. + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + assert.Equal(t, []string{"foo", "bar"}, b.Config.Sync.Include) + assert.Equal(t, []string{"baz", "qux"}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_ParentDirectory(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../common", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, []string{"common"}, b.Config.Sync.Paths) + assert.Equal(t, []string{filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/bar")}, b.Config.Sync.Include) + assert.Equal(t, []string{filepath.FromSlash("dir/baz"), filepath.FromSlash("dir/qux")}, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir/that/is/very/deeply/nested", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../../../../../../common", + }, + Include: []string{ + "foo", + "bar", + }, + Exclude: []string{ + "baz", + "qux", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, []string{"common"}, b.Config.Sync.Paths) + assert.Equal(t, []string{ + filepath.FromSlash("dir/that/is/very/deeply/nested/foo"), + filepath.FromSlash("dir/that/is/very/deeply/nested/bar"), + }, b.Config.Sync.Include) + assert.Equal(t, []string{ + filepath.FromSlash("dir/that/is/very/deeply/nested/baz"), + filepath.FromSlash("dir/that/is/very/deeply/nested/qux"), + }, b.Config.Sync.Exclude) +} + +func TestSyncInferRoot_MultiplePaths(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/bundle/root", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "./foo", + "../common", + "./bar", + "../../baz", + }, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + assert.NoError(t, diags.Error()) + assert.Equal(t, filepath.FromSlash("/tmp/some"), b.SyncRootPath) + + // Check that the paths are updated. + assert.Equal(t, filepath.FromSlash("bundle/root/foo"), b.Config.Sync.Paths[0]) + assert.Equal(t, filepath.FromSlash("bundle/common"), b.Config.Sync.Paths[1]) + assert.Equal(t, filepath.FromSlash("bundle/root/bar"), b.Config.Sync.Paths[2]) + assert.Equal(t, filepath.FromSlash("baz"), b.Config.Sync.Paths[3]) +} + +func TestSyncInferRoot_Error(t *testing.T) { + b := &bundle.Bundle{ + RootPath: "/tmp/some/dir", + Config: config.Root{ + Sync: config.Sync{ + Paths: []string{ + "../../../../error", + "../../../thisworks", + "../../../../../error", + }, + }, + }, + } + + bundletest.SetLocation(b, "sync.paths", "databricks.yml") + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) + require.Len(t, diags, 2) + assert.Equal(t, `invalid sync path "../../../../error"`, diags[0].Summary) + assert.Equal(t, "databricks.yml:0:0", diags[0].Locations[0].String()) + assert.Equal(t, "sync.paths[0]", diags[0].Paths[0].String()) + assert.Equal(t, `invalid sync path "../../../../../error"`, diags[1].Summary) + assert.Equal(t, "databricks.yml:0:0", diags[1].Locations[0].String()) + assert.Equal(t, "sync.paths[2]", diags[1].Paths[0].String()) +} diff --git a/bundle/config/mutator/trampoline.go b/bundle/config/mutator/trampoline.go index dde9a299e..dcca50149 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/config/mutator/trampoline.go @@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund return err } - internalDirRel, err := filepath.Rel(b.RootPath, internalDir) + internalDirRel, err := filepath.Rel(b.SyncRootPath, internalDir) if err != nil { return err } diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index e39076647..08d3c8220 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -57,17 +56,18 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Bundle: config.Bundle{ Target: "development", }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: tasks, }, @@ -93,6 +93,6 @@ func TestGenerateTrampoline(t *testing.T) { require.Equal(t, "Hello from Trampoline", string(bytes)) task := b.Config.Resources.Jobs["test"].Tasks[0] - require.Equal(t, task.NotebookTask.NotebookPath, ".databricks/bundle/development/.internal/notebook_test_to_trampoline") + require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_test_to_trampoline", task.NotebookTask.NotebookPath) require.Nil(t, task.PythonWheelTask) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a01d3d6a7..5f22570e7 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -93,14 +93,14 @@ func (t *translateContext) rewritePath( return nil } - // Local path must be contained in the bundle root. + // Local path must be contained in the sync root. // If it isn't, it won't be synchronized into the workspace. - localRelPath, err := filepath.Rel(t.b.RootPath, localPath) + localRelPath, err := filepath.Rel(t.b.SyncRootPath, localPath) if err != nil { return err } if strings.HasPrefix(localRelPath, "..") { - return fmt.Errorf("path %s is not contained in bundle root path", localPath) + return fmt.Errorf("path %s is not contained in sync root path", localPath) } // Prefix remote path with its remote root path. @@ -118,7 +118,7 @@ func (t *translateContext) rewritePath( } func (t *translateContext) translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath)) + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath)) if errors.Is(err, fs.ErrNotExist) { return "", fmt.Errorf("notebook %s not found", literal) } @@ -134,7 +134,7 @@ func (t *translateContext) translateNotebookPath(literal, localFullPath, localRe } func (t *translateContext) translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - nb, _, err := notebook.DetectWithFS(t.b.BundleRoot, filepath.ToSlash(localRelPath)) + nb, _, err := notebook.DetectWithFS(t.b.SyncRoot, filepath.ToSlash(localRelPath)) if errors.Is(err, fs.ErrNotExist) { return "", fmt.Errorf("file %s not found", literal) } @@ -148,7 +148,7 @@ func (t *translateContext) translateFilePath(literal, localFullPath, localRelPat } func (t *translateContext) translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) { - info, err := t.b.BundleRoot.Stat(filepath.ToSlash(localRelPath)) + info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath)) if err != nil { return "", err } @@ -182,7 +182,7 @@ func (t *translateContext) rewriteValue(p dyn.Path, v dyn.Value, fn rewriteFunc, return dyn.InvalidValue, err } - return dyn.NewValue(out, v.Location()), nil + return dyn.NewValue(out, v.Locations()), nil } func (t *translateContext) rewriteRelativeTo(p dyn.Path, v dyn.Value, fn rewriteFunc, dir, fallback string) (dyn.Value, error) { diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 60cc8bb9a..e34eeb2f0 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern t.translateNoOp, noSkipRewrite, }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + t.translateFilePath, + noSkipRewrite, + }, } } @@ -78,7 +83,7 @@ func (t *translateContext) jobRewritePatterns() []jobRewritePattern { ), t.translateNoOpWithPrefix, func(s string) bool { - return !libraries.IsEnvironmentDependencyLocal(s) + return !libraries.IsLibraryLocal(s) }, }, } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 8476ee38a..50fcd3b07 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -11,7 +11,10 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -38,8 +41,8 @@ func touchEmptyFile(t *testing.T, path string) { func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -107,10 +110,11 @@ func TestTranslatePaths(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py")) touchEmptyFile(t, filepath.Join(dir, "my_python_file.py")) touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar")) + touchEmptyFile(t, filepath.Join(dir, "requirements.txt")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -137,6 +141,9 @@ func TestTranslatePaths(t *testing.T) { NotebookTask: &jobs.NotebookTask{ NotebookPath: "./my_job_notebook.py", }, + Libraries: []compute.Library{ + {Requirements: "./requirements.txt"}, + }, }, { PythonWheelTask: &jobs.PythonWheelTask{ @@ -229,6 +236,11 @@ func TestTranslatePaths(t *testing.T) { "/bundle/my_job_notebook", b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath, ) + assert.Equal( + t, + "/bundle/requirements.txt", + b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements, + ) assert.Equal( t, "/bundle/my_python_file.py", @@ -277,8 +289,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -368,12 +380,12 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { ) } -func TestTranslatePathsOutsideBundleRoot(t *testing.T) { +func TestTranslatePathsOutsideSyncRoot(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -399,15 +411,15 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) { bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml")) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) - assert.ErrorContains(t, diags.Error(), "is not contained in bundle root") + assert.ErrorContains(t, diags.Error(), "is not contained in sync root path") } func TestJobNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -437,8 +449,8 @@ func TestJobFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -468,8 +480,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -499,8 +511,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -531,8 +543,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -566,8 +578,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -601,8 +613,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -636,8 +648,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -672,8 +684,8 @@ func TestTranslatePathJobEnvironments(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "env2.py")) b := &bundle.Bundle{ - RootPath: dir, - BundleRoot: vfs.MustNew(dir), + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -708,3 +720,64 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) } + +func TestTranslatePathWithComplexVariables(t *testing.T) { + dir := t.TempDir() + b := &bundle.Bundle{ + SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "cluster_libraries": { + Type: variable.VariableTypeComplex, + Default: [](map[string]string){ + { + "whl": "./local/whl.whl", + }, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "test", + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml")) + bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + + ctx := context.Background() + // Assign the variables to the dynamic configuration. + diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + p := dyn.MustPathFromString("resources.jobs.job.tasks[0]") + return dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}")) + }) + return diag.FromErr(err) + }) + require.NoError(t, diags.Error()) + + diags = bundle.Apply(ctx, b, + bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferences("variables"), + mutator.TranslatePaths(), + )) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.Join("variables", "local", "whl.whl"), + b.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl, + ) +} diff --git a/bundle/config/paths/paths.go b/bundle/config/paths/paths.go deleted file mode 100644 index 95977ee37..000000000 --- a/bundle/config/paths/paths.go +++ /dev/null @@ -1,22 +0,0 @@ -package paths - -import ( - "github.com/databricks/cli/libs/dyn" -) - -type Paths struct { - // Absolute path on the local file system to the configuration file that holds - // the definition of this resource. - ConfigFilePath string `json:"-" bundle:"readonly"` - - // DynamicValue stores the [dyn.Value] of the containing struct. - // This assumes that this struct is always embedded. - DynamicValue dyn.Value `json:"-"` -} - -func (p *Paths) ConfigureConfigFilePath() { - if !p.DynamicValue.IsValid() { - panic("DynamicValue not set") - } - p.ConfigFilePath = p.DynamicValue.Location().File -} diff --git a/bundle/config/presets.go b/bundle/config/presets.go new file mode 100644 index 000000000..61009a252 --- /dev/null +++ b/bundle/config/presets.go @@ -0,0 +1,32 @@ +package config + +const Paused = "PAUSED" +const Unpaused = "UNPAUSED" + +type Presets struct { + // NamePrefix to prepend to all resource names. + NamePrefix string `json:"name_prefix,omitempty"` + + // PipelinesDevelopment is the default value for the development field of pipelines. + PipelinesDevelopment *bool `json:"pipelines_development,omitempty"` + + // TriggerPauseStatus is the default value for the pause status of all triggers and schedules. + // Either config.Paused, config.Unpaused, or empty. + TriggerPauseStatus string `json:"trigger_pause_status,omitempty"` + + // JobsMaxConcurrentRuns is the default value for the max concurrent runs of jobs. + JobsMaxConcurrentRuns int `json:"jobs_max_concurrent_runs,omitempty"` + + // Tags to add to all resources. + Tags map[string]string `json:"tags,omitempty"` +} + +// IsExplicitlyEnabled tests whether this feature is explicitly enabled. +func IsExplicitlyEnabled(feature *bool) bool { + return feature != nil && *feature +} + +// IsExplicitlyDisabled tests whether this feature is explicitly disabled. +func IsExplicitlyDisabled(feature *bool) bool { + return feature != nil && !*feature +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 70030c664..22d69ffb5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -2,7 +2,6 @@ package config import ( "context" - "encoding/json" "fmt" "github.com/databricks/cli/bundle/config/resources" @@ -19,206 +18,17 @@ type Resources struct { ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` -} - -type UniqueResourceIdTracker struct { - Type map[string]string - ConfigPath map[string]string + Schemas map[string]*resources.Schema `json:"schemas,omitempty"` } type ConfigResource interface { + // Function to assert if the resource exists in the workspace configured in + // the input workspace client. Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) + + // Terraform equivalent name of the resource. For example "databricks_job" + // for jobs and "databricks_pipeline" for pipelines. TerraformResourceName() string - Validate() error - - json.Marshaler - json.Unmarshaler -} - -// verifies merging is safe by checking no duplicate identifiers exist -func (r *Resources) VerifySafeMerge(other *Resources) error { - rootTracker, err := r.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - otherTracker, err := other.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - for k := range otherTracker.Type { - if _, ok := rootTracker.Type[k]; ok { - return fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - rootTracker.Type[k], - rootTracker.ConfigPath[k], - otherTracker.Type[k], - otherTracker.ConfigPath[k], - ) - } - } - return nil -} - -// This function verifies there are no duplicate names used for the resource definations -func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker, error) { - tracker := &UniqueResourceIdTracker{ - Type: make(map[string]string), - ConfigPath: make(map[string]string), - } - for k := range r.Jobs { - tracker.Type[k] = "job" - tracker.ConfigPath[k] = r.Jobs[k].ConfigFilePath - } - for k := range r.Pipelines { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "pipeline", - r.Pipelines[k].ConfigFilePath, - ) - } - tracker.Type[k] = "pipeline" - tracker.ConfigPath[k] = r.Pipelines[k].ConfigFilePath - } - for k := range r.Models { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_model", - r.Models[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_model" - tracker.ConfigPath[k] = r.Models[k].ConfigFilePath - } - for k := range r.Experiments { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_experiment", - r.Experiments[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_experiment" - tracker.ConfigPath[k] = r.Experiments[k].ConfigFilePath - } - for k := range r.ModelServingEndpoints { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "model_serving_endpoint", - r.ModelServingEndpoints[k].ConfigFilePath, - ) - } - tracker.Type[k] = "model_serving_endpoint" - tracker.ConfigPath[k] = r.ModelServingEndpoints[k].ConfigFilePath - } - for k := range r.RegisteredModels { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "registered_model", - r.RegisteredModels[k].ConfigFilePath, - ) - } - tracker.Type[k] = "registered_model" - tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath - } - for k := range r.QualityMonitors { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "quality_monitor", - r.QualityMonitors[k].ConfigFilePath, - ) - } - tracker.Type[k] = "quality_monitor" - tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath - } - return tracker, nil -} - -type resource struct { - resource ConfigResource - resource_type string - key string -} - -func (r *Resources) allResources() []resource { - all := make([]resource, 0) - for k, e := range r.Jobs { - all = append(all, resource{resource_type: "job", resource: e, key: k}) - } - for k, e := range r.Pipelines { - all = append(all, resource{resource_type: "pipeline", resource: e, key: k}) - } - for k, e := range r.Models { - all = append(all, resource{resource_type: "model", resource: e, key: k}) - } - for k, e := range r.Experiments { - all = append(all, resource{resource_type: "experiment", resource: e, key: k}) - } - for k, e := range r.ModelServingEndpoints { - all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k}) - } - for k, e := range r.RegisteredModels { - all = append(all, resource{resource_type: "registered model", resource: e, key: k}) - } - for k, e := range r.QualityMonitors { - all = append(all, resource{resource_type: "quality monitor", resource: e, key: k}) - } - return all -} - -func (r *Resources) VerifyAllResourcesDefined() error { - all := r.allResources() - for _, e := range all { - err := e.resource.Validate() - if err != nil { - return fmt.Errorf("%s %s is not defined", e.resource_type, e.key) - } - } - - return nil -} - -// ConfigureConfigFilePath sets the specified path for all resources contained in this instance. -// This property is used to correctly resolve paths relative to the path -// of the configuration file they were defined in. -func (r *Resources) ConfigureConfigFilePath() { - for _, e := range r.Jobs { - e.ConfigureConfigFilePath() - } - for _, e := range r.Pipelines { - e.ConfigureConfigFilePath() - } - for _, e := range r.Models { - e.ConfigureConfigFilePath() - } - for _, e := range r.Experiments { - e.ConfigureConfigFilePath() - } - for _, e := range r.ModelServingEndpoints { - e.ConfigureConfigFilePath() - } - for _, e := range r.RegisteredModels { - e.ConfigureConfigFilePath() - } - for _, e := range r.QualityMonitors { - e.ConfigureConfigFilePath() - } } func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) { diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index dde5d5663..d8f97a2db 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,10 +2,8 @@ package resources import ( "context" - "fmt" "strconv" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -17,8 +15,6 @@ type Job struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *jobs.JobSettings } @@ -48,11 +44,3 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri func (j *Job) TerraformResourceName() string { return "databricks_job" } - -func (j *Job) Validate() error { - if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil { - return fmt.Errorf("job is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index 7854ee7e8..0ab486436 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowExperiment struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Experiment } @@ -43,11 +39,3 @@ func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceCl func (s *MlflowExperiment) TerraformResourceName() string { return "databricks_mlflow_experiment" } - -func (s *MlflowExperiment) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("experiment is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 40da9f87d..300474e35 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowModel struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Model } @@ -43,11 +39,3 @@ func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, func (s *MlflowModel) TerraformResourceName() string { return "databricks_mlflow_model" } - -func (s *MlflowModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("model is not defined") - } - - return nil -} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 503cfbbb7..5efb7ea26 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -20,10 +18,6 @@ type ModelServingEndpoint struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This is a resource agnostic implementation of permissions for ACLs. // Implementation could be different based on the resource type. Permissions []Permission `json:"permissions,omitempty"` @@ -53,11 +47,3 @@ func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.Workspa func (s *ModelServingEndpoint) TerraformResourceName() string { return "databricks_model_serving" } - -func (s *ModelServingEndpoint) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("serving endpoint is not defined") - } - - return nil -} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 7e914b909..55270be65 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type Pipeline struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *pipelines.PipelineSpec } @@ -43,11 +39,3 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (p *Pipeline) TerraformResourceName() string { return "databricks_pipeline" } - -func (p *Pipeline) Validate() error { - if p == nil || !p.DynamicValue.IsValid() { - return fmt.Errorf("pipeline is not defined") - } - - return nil -} diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go index 0d13e58fa..9160782cd 100644 --- a/bundle/config/resources/quality_monitor.go +++ b/bundle/config/resources/quality_monitor.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type QualityMonitor struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` } @@ -50,11 +44,3 @@ func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClie func (s *QualityMonitor) TerraformResourceName() string { return "databricks_quality_monitor" } - -func (s *QualityMonitor) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("quality monitor is not defined") - } - - return nil -} diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index fba643c69..6033ffdf2 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type RegisteredModel struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This represents the input args for terraform, and will get converted // to a HCL representation for CRUD *catalog.CreateRegisteredModelRequest @@ -54,11 +48,3 @@ func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceCli func (s *RegisteredModel) TerraformResourceName() string { return "databricks_registered_model" } - -func (s *RegisteredModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("registered model is not defined") - } - - return nil -} diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go new file mode 100644 index 000000000..7ab00495a --- /dev/null +++ b/bundle/config/resources/schema.go @@ -0,0 +1,27 @@ +package resources + +import ( + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Schema struct { + // List of grants to apply on this schema. + Grants []Grant `json:"grants,omitempty"` + + // Full name of the schema (catalog_name.schema_name). This value is read from + // the terraform state after deployment succeeds. + ID string `json:"id,omitempty" bundle:"readonly"` + + *catalog.CreateSchema + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` +} + +func (s *Schema) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Schema) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 7415029b1..6860d73da 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -5,129 +5,9 @@ import ( "reflect" "testing" - "github.com/databricks/cli/bundle/config/paths" - "github.com/databricks/cli/bundle/config/resources" "github.com/stretchr/testify/assert" ) -func TestVerifyUniqueResourceIdentifiers(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - Experiments: map[string]*resources.MlflowExperiment{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - _, err := r.VerifyUniqueResourceIdentifiers() - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, mlflow_experiment at foo2.yml)") -} - -func TestVerifySafeMerge(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Pipelines: map[string]*resources.Pipeline{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, pipeline at foo2.yml)") -} - -func TestVerifySafeMergeForSameResourceType(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, job at foo2.yml)") -} - -func TestVerifySafeMergeForRegisteredModels(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)") -} - // This test ensures that all resources have a custom marshaller and unmarshaller. // This is required because DABs resources map to Databricks APIs, and they do so // by embedding the corresponding Go SDK structs. diff --git a/bundle/config/root.go b/bundle/config/root.go index 2bbb78696..86dc33921 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -60,6 +60,10 @@ type Root struct { // RunAs section allows to define an execution identity for jobs and pipelines runs RunAs *jobs.JobRunAs `json:"run_as,omitempty"` + // Presets applies preset transformations throughout the bundle, e.g. + // adding a name prefix to deployed resources. + Presets Presets `json:"presets,omitempty"` + Experimental *Experimental `json:"experimental,omitempty"` // Permissions section allows to define permissions which will be @@ -100,11 +104,6 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { if err != nil { return nil, diag.Errorf("failed to load %s: %v", path, err) } - - _, err = r.Resources.VerifyUniqueResourceIdentifiers() - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } return &r, diags } @@ -141,17 +140,6 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error { // Assign the normalized configuration tree. r.value = nv - - // At the moment the check has to be done as part of updateWithDynamicValue - // because otherwise ConfigureConfigFilePath will fail with a panic. - // In the future, we should move this check to a separate mutator in initialise phase. - err = r.Resources.VerifyAllResourcesDefined() - if err != nil { - return err - } - - // Assign config file paths after converting to typed configuration. - r.ConfigureConfigFilePath() return nil } @@ -243,15 +231,6 @@ func (r *Root) MarkMutatorExit(ctx context.Context) error { return nil } -// SetConfigFilePath configures the path that its configuration -// was loaded from in configuration leafs that require it. -func (r *Root) ConfigureConfigFilePath() { - r.Resources.ConfigureConfigFilePath() - if r.Artifacts != nil { - r.Artifacts.ConfigureConfigFilePath() - } -} - // Initializes variables using values passed from the command line flag // Input has to be a string of the form `foo=bar`. In this case the variable with // name `foo` is assigned the value `bar` @@ -281,12 +260,6 @@ func (r *Root) InitializeVariables(vars []string) error { } func (r *Root) Merge(other *Root) error { - // Check for safe merge, protecting against duplicate resource identifiers - err := r.Resources.VerifySafeMerge(&other.Resources) - if err != nil { - return err - } - // Merge dynamic configuration values. return r.Mutate(func(root dyn.Value) (dyn.Value, error) { return merge.Merge(root, other.value) @@ -338,6 +311,7 @@ func (r *Root) MergeTargetOverrides(name string) error { "resources", "sync", "permissions", + "presets", } { if root, err = mergeField(root, target, f); err != nil { return err @@ -378,7 +352,7 @@ func (r *Root) MergeTargetOverrides(name string) error { // Below, we're setting fields on the bundle key, so make sure it exists. if root.Get("bundle").Kind() == dyn.KindInvalid { - root, err = dyn.Set(root, "bundle", dyn.NewValue(map[string]dyn.Value{}, dyn.Location{})) + root, err = dyn.Set(root, "bundle", dyn.V(map[string]dyn.Value{})) if err != nil { return err } @@ -404,7 +378,7 @@ func (r *Root) MergeTargetOverrides(name string) error { if v := target.Get("git"); v.Kind() != dyn.KindInvalid { ref, err := dyn.GetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("git"))) if err != nil { - ref = dyn.NewValue(map[string]dyn.Value{}, dyn.Location{}) + ref = dyn.V(map[string]dyn.Value{}) } // Merge the override into the reference. @@ -415,7 +389,7 @@ func (r *Root) MergeTargetOverrides(name string) error { // If the branch was overridden, we need to clear the inferred flag. if branch := v.Get("branch"); branch.Kind() != dyn.KindInvalid { - out, err = dyn.SetByPath(out, dyn.NewPath(dyn.Key("inferred")), dyn.NewValue(false, dyn.Location{})) + out, err = dyn.SetByPath(out, dyn.NewPath(dyn.Key("inferred")), dyn.V(false)) if err != nil { return err } @@ -456,7 +430,7 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { // configuration will convert this to a string if necessary. return dyn.NewValue(map[string]dyn.Value{ "default": variable, - }, variable.Location()), nil + }, variable.Locations()), nil case dyn.KindMap, dyn.KindSequence: // Check if the original definition of variable has a type field. @@ -469,7 +443,7 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { return dyn.NewValue(map[string]dyn.Value{ "type": typeV, "default": variable, - }, variable.Location()), nil + }, variable.Locations()), nil } return variable, nil @@ -524,6 +498,17 @@ func (r Root) GetLocation(path string) dyn.Location { return v.Location() } +// Get all locations of the configuration value at the specified path. We need both +// this function and it's singular version (GetLocation) because some diagnostics just need +// the primary location and some need all locations associated with a configuration value. +func (r Root) GetLocations(path string) []dyn.Location { + v, err := dyn.Get(r.value, path) + if err != nil { + return []dyn.Location{} + } + return v.Locations() +} + // Value returns the dynamic configuration value of the root object. This value // is the source of truth and is kept in sync with values in the typed configuration. func (r Root) Value() dyn.Value { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index aed670d6c..c95e6e86c 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -30,22 +30,6 @@ func TestRootLoad(t *testing.T) { assert.Equal(t, "basic", root.Bundle.Name) } -func TestDuplicateIdOnLoadReturnsError(t *testing.T) { - _, diags := Load("./testdata/duplicate_resource_names_in_root/databricks.yml") - assert.ErrorContains(t, diags.Error(), "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)") -} - -func TestDuplicateIdOnMergeReturnsError(t *testing.T) { - root, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml") - require.NoError(t, diags.Error()) - - other, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/resources.yml") - require.NoError(t, diags.Error()) - - err := root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") -} - func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ diff --git a/bundle/config/sync.go b/bundle/config/sync.go index 0580e4c4f..377b1333e 100644 --- a/bundle/config/sync.go +++ b/bundle/config/sync.go @@ -1,6 +1,10 @@ package config type Sync struct { + // Paths contains a list of paths to synchronize relative to the bundle root path. + // If not configured, this defaults to synchronizing everything in the bundle root path (i.e. `.`). + Paths []string `json:"paths,omitempty"` + // Include contains a list of globs evaluated relative to the bundle root path // to explicitly include files that were excluded by the user's gitignore. Include []string `json:"include,omitempty"` diff --git a/bundle/config/target.go b/bundle/config/target.go index acc493574..a2ef4d735 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -20,6 +20,10 @@ type Target struct { // development purposes. Mode Mode `json:"mode,omitempty"` + // Mutator configurations that e.g. change the + // name prefix of deployed resources. + Presets Presets `json:"presets,omitempty"` + // Overrides the compute used for jobs and other supported assets. ComputeID string `json:"compute_id,omitempty"` diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go new file mode 100644 index 000000000..7f96e529a --- /dev/null +++ b/bundle/config/validate/all_resources_have_values.go @@ -0,0 +1,57 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func AllResourcesHaveValues() bundle.Mutator { + return &allResourcesHaveValues{} +} + +type allResourcesHaveValues struct{} + +func (m *allResourcesHaveValues) Name() string { + return "validate:AllResourcesHaveValues" +} + +func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + _, err := dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() != dyn.KindNil { + return v, nil + } + + // Type of the resource, stripped of the trailing 's' to make it + // singular. + rType := strings.TrimSuffix(p[1].Key(), "s") + + // Name of the resource. Eg: "foo" in "jobs.foo". + rName := p[2].Key() + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s %s is not defined", rType, rName), + Locations: v.Locations(), + Paths: []dyn.Path{slices.Clone(p)}, + }) + + return v, nil + }, + ) + if err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + return diags +} diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index d53e38243..7cdad772a 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func FilesToSync() bundle.ReadOnlyMutator { @@ -45,8 +46,10 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag. diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", - Location: loc.Location(), - Path: loc.Path(), + // Show all locations where sync.exclude is defined, since merging + // sync.exclude is additive. + Locations: loc.Locations(), + Paths: []dyn.Path{loc.Path()}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 37ed3f417..368c3edb1 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -41,8 +42,11 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("job_cluster_key %s is not defined", task.JobClusterKey), - Location: loc.Location(), - Path: loc.Path(), + // Show only the location where the job_cluster_key is defined. + // Other associated locations are not relevant since they are + // overridden during merging. + Locations: []dyn.Location{loc.Location()}, + Paths: []dyn.Path{loc.Path()}, }) } } diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go new file mode 100644 index 000000000..d6212b0ac --- /dev/null +++ b/bundle/config/validate/unique_resource_keys.go @@ -0,0 +1,116 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "sort" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// This mutator validates that: +// +// 1. Each resource key is unique across different resource types. No two resources +// of the same type can have the same key. This is because command like "bundle run" +// rely on the resource key to identify the resource to run. +// Eg: jobs.foo and pipelines.foo are not allowed simultaneously. +// +// 2. Each resource definition is contained within a single file, and is not spread +// across multiple files. Note: This is not applicable to resource configuration +// defined in a target override. That is why this mutator MUST run before the target +// overrides are merged. +func UniqueResourceKeys() bundle.Mutator { + return &uniqueResourceKeys{} +} + +type uniqueResourceKeys struct{} + +func (m *uniqueResourceKeys) Name() string { + return "validate:unique_resource_keys" +} + +func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + type metadata struct { + locations []dyn.Location + paths []dyn.Path + } + + // Maps of resource key to the paths and locations the resource is defined at. + resourceMetadata := map[string]*metadata{} + + rv := b.Config.Value().Get("resources") + + // return early if no resources are defined or the resources block is empty. + if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { + return diags + } + + // Gather the paths and locations of all resources. + _, err := dyn.MapByPattern( + rv, + dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource. Eg: "my_job" for jobs.my_job. + k := p[1].Key() + + m, ok := resourceMetadata[k] + if !ok { + m = &metadata{ + paths: []dyn.Path{}, + locations: []dyn.Location{}, + } + } + + // dyn.Path under the hood is a slice. The code that walks the configuration + // tree uses the same underlying slice to track the path as it walks + // the tree. So, we need to clone it here. + m.paths = append(m.paths, slices.Clone(p)) + m.locations = append(m.locations, v.Locations()...) + + resourceMetadata[k] = m + return v, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + for k, v := range resourceMetadata { + if len(v.locations) <= 1 { + continue + } + + // Sort the locations and paths for consistent error messages. This helps + // with unit testing. + sort.Slice(v.locations, func(i, j int) bool { + l1 := v.locations[i] + l2 := v.locations[j] + + if l1.File != l2.File { + return l1.File < l2.File + } + if l1.Line != l2.Line { + return l1.Line < l2.Line + } + return l1.Column < l2.Column + }) + sort.Slice(v.paths, func(i, j int) bool { + return v.paths[i].String() < v.paths[j].String() + }) + + // If there are multiple resources with the same key, report an error. + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("multiple resources have been defined with the same key: %s", k), + Locations: v.locations, + Paths: v.paths, + }) + } + + return diags +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index af7e984a1..b4da0bc05 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -20,6 +20,10 @@ func (l location) Location() dyn.Location { return l.rb.Config().GetLocation(l.path) } +func (l location) Locations() []dyn.Location { + return l.rb.Config().GetLocations(l.path) +} + func (l location) Path() dyn.Path { return dyn.MustPathFromString(l.path) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index a04c10776..52f06835c 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -3,10 +3,12 @@ package validate import ( "context" "fmt" + "strings" "sync" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/fileset" "golang.org/x/sync/errgroup" ) @@ -48,14 +50,20 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di for i, pattern := range patterns { index := i - p := pattern + fullPattern := pattern + // If the pattern is negated, strip the negation prefix + // and check if the pattern matches any files. + // Negation in gitignore syntax means "don't look at this path' + // So if p matches nothing it's useless negation, but if there are matches, + // it means: do not include these files into result set + p := strings.TrimPrefix(fullPattern, "!") errs.Go(func() error { fs, err := fileset.NewGlobSet(rb.BundleRoot(), []string{p}) if err != nil { return err } - all, err := fs.All() + all, err := fs.Files() if err != nil { return err } @@ -64,10 +72,10 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di loc := location{path: fmt.Sprintf("%s[%d]", path, index), rb: rb} mu.Lock() diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("Pattern %s does not match any files", p), - Location: loc.Location(), - Path: loc.Path(), + Severity: diag.Warning, + Summary: fmt.Sprintf("Pattern %s does not match any files", fullPattern), + Locations: []dyn.Location{loc.Location()}, + Paths: []dyn.Path{loc.Path()}, }) mu.Unlock() } diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 56d2ca810..9c85e2a71 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -220,7 +220,7 @@ type resolvers struct { func allResolvers() *resolvers { r := &resolvers{} r.Alert = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.Alerts.GetByName(ctx, name) + entity, err := w.Alerts.GetByDisplayName(ctx, name) if err != nil { return "", err } @@ -284,7 +284,7 @@ func allResolvers() *resolvers { return fmt.Sprint(entity.PipelineId), nil } r.Query = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.Queries.GetByName(ctx, name) + entity, err := w.Queries.GetByDisplayName(ctx, name) if err != nil { return "", err } diff --git a/bundle/deploy/files/delete.go b/bundle/deploy/files/delete.go index 133971449..bb28c2722 100644 --- a/bundle/deploy/files/delete.go +++ b/bundle/deploy/files/delete.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/sync" "github.com/databricks/databricks-sdk-go/service/workspace" - "github.com/fatih/color" ) type delete struct{} @@ -22,24 +21,7 @@ func (m *delete) Name() string { } func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // Do not delete files if terraform destroy was not consented - if !b.Plan.IsEmpty && !b.Plan.ConfirmApply { - return nil - } - - cmdio.LogString(ctx, "Starting deletion of remote bundle files") - cmdio.LogString(ctx, fmt.Sprintf("Bundle remote directory is %s", b.Config.Workspace.RootPath)) - - red := color.New(color.FgRed).SprintFunc() - if !b.AutoApprove { - proceed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\n%s and all files in it will be %s Proceed?", b.Config.Workspace.RootPath, red("deleted permanently!"))) - if err != nil { - return diag.FromErr(err) - } - if !proceed { - return nil - } - } + cmdio.LogString(ctx, "Deleting files...") err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{ Path: b.Config.Workspace.RootPath, @@ -54,8 +36,6 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if err != nil { return diag.FromErr(err) } - - cmdio.LogString(ctx, "Successfully deleted files!") return nil } diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index a308668d3..347ed3079 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -28,10 +28,12 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp } opts := &sync.SyncOptions{ - LocalPath: rb.BundleRoot(), + LocalRoot: rb.SyncRoot(), + Paths: rb.Config().Sync.Paths, + Include: includes, + Exclude: rb.Config().Sync.Exclude, + RemotePath: rb.Config().Workspace.FilePath, - Include: includes, - Exclude: rb.Config().Sync.Exclude, Host: rb.WorkspaceClient().Config.Host, Full: false, diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 034765484..6ab997e27 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -39,7 +39,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - relativePath, err := filepath.Rel(b.RootPath, job.ConfigFilePath) + l := b.Config.GetLocation("resources.jobs." + name) + relativePath, err := filepath.Rel(b.RootPath, l.File) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/state.go b/bundle/deploy/state.go index 97048811b..4f2bc4ee4 100644 --- a/bundle/deploy/state.go +++ b/bundle/deploy/state.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/vfs" + "github.com/google/uuid" ) const DeploymentStateFileName = "deployment.json" @@ -46,6 +47,9 @@ type DeploymentState struct { // Files is a list of files which has been deployed as part of this deployment. Files Filelist `json:"files"` + + // UUID uniquely identifying the deployment. + ID uuid.UUID `json:"id"` } // We use this entry type as a proxy to fs.DirEntry. diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 24ed9d360..5e301a6f3 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } log.Infof(ctx, "Creating new snapshot") - snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.BundleRoot), opts) + snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.SyncRoot), opts) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index 38f0b4021..f75193065 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -64,6 +64,10 @@ func testStatePull(t *testing.T, opts statePullOpts) { b := &bundle.Bundle{ RootPath: tmpDir, BundleRoot: vfs.MustNew(tmpDir), + + SyncRootPath: tmpDir, + SyncRoot: vfs.MustNew(tmpDir), + Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -81,11 +85,11 @@ func testStatePull(t *testing.T, opts statePullOpts) { ctx := context.Background() for _, file := range opts.localFiles { - testutil.Touch(t, b.RootPath, "bar", file) + testutil.Touch(t, b.SyncRootPath, "bar", file) } for _, file := range opts.localNotebooks { - testutil.TouchNotebook(t, b.RootPath, "bar", file) + testutil.TouchNotebook(t, b.SyncRootPath, "bar", file) } if opts.withExistingSnapshot { diff --git a/bundle/deploy/state_test.go b/bundle/deploy/state_test.go index 5e1e54230..d149b0efa 100644 --- a/bundle/deploy/state_test.go +++ b/bundle/deploy/state_test.go @@ -18,7 +18,7 @@ func TestFromSlice(t *testing.T) { testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") - files, err := fileset.All() + files, err := fileset.Files() require.NoError(t, err) f, err := FromSlice(files) @@ -38,7 +38,7 @@ func TestToSlice(t *testing.T) { testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") - files, err := fileset.All() + files, err := fileset.Files() require.NoError(t, err) f, err := FromSlice(files) diff --git a/bundle/deploy/state_update.go b/bundle/deploy/state_update.go index bfdb308c4..9ab1bacf1 100644 --- a/bundle/deploy/state_update.go +++ b/bundle/deploy/state_update.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" + "github.com/google/uuid" ) type stateUpdate struct { @@ -46,6 +47,11 @@ func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost } state.Files = fl + // Generate a UUID for the deployment, if one does not already exist + if state.ID == uuid.Nil { + state.ID = uuid.New() + } + statePath, err := getPathToStateFile(ctx, b) if err != nil { return diag.FromErr(err) diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index ed72439d2..72096d142 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -22,7 +23,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle { testutil.Touch(t, tmpDir, "test1.py") testutil.TouchNotebook(t, tmpDir, "test2.py") - files, err := fileset.New(vfs.MustNew(tmpDir)).All() + files, err := fileset.New(vfs.MustNew(tmpDir)).Files() require.NoError(t, err) return &bundle.Bundle{ @@ -88,6 +89,9 @@ func TestStateUpdate(t *testing.T) { }, }) require.Equal(t, build.GetInfo().Version, state.CliVersion) + + // Valid non-empty UUID is generated. + require.NotEqual(t, uuid.Nil, state.ID) } func TestStateUpdateWithExistingState(t *testing.T) { @@ -109,6 +113,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { LocalPath: "bar/t1.py", }, }, + ID: uuid.MustParse("123e4567-e89b-12d3-a456-426614174000"), } data, err := json.Marshal(state) @@ -135,4 +140,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { }, }) require.Equal(t, build.GetInfo().Version, state.CliVersion) + + // Existing UUID is not overwritten. + require.Equal(t, uuid.MustParse("123e4567-e89b-12d3-a456-426614174000"), state.ID) } diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index ca3536c95..5ea2effa0 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" @@ -18,19 +17,23 @@ func (w *apply) Name() string { } func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // return early if plan is empty + if b.Plan.IsEmpty { + log.Debugf(ctx, "No changes in plan. Skipping terraform apply.") + return nil + } + tf := b.Terraform if tf == nil { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Deploying resources...") - - err := tf.Init(ctx, tfexec.Upgrade(true)) - if err != nil { - return diag.Errorf("terraform init: %v", err) + if b.Plan.Path == "" { + return diag.Errorf("no plan found") } - err = tf.Apply(ctx) + // Apply terraform according to the computed plan + err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { diags := permissions.TryExtendTerraformPermissionError(ctx, b, err) if diags != nil { @@ -39,11 +42,11 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("terraform apply: %v", err) } - log.Infof(ctx, "Resource deployment completed") + log.Infof(ctx, "terraform apply completed") return nil } -// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply` +// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply ./plan` // from the bundle's ephemeral working directory for Terraform. func Apply() bundle.Mutator { return &apply{} diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index a6ec04d9a..f13c241ce 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -66,8 +66,10 @@ func convGrants(acl []resources.Grant) *schema.ResourceGrants { // BundleToTerraform converts resources in a bundle configuration // to the equivalent Terraform JSON representation. // -// NOTE: THIS IS CURRENTLY A HACK. WE NEED A BETTER WAY TO -// CONVERT TO/FROM TERRAFORM COMPATIBLE FORMAT. +// Note: This function is an older implementation of the conversion logic. It is +// no longer used in any code paths. It is kept around to be used in tests. +// New resources do not need to modify this function and can instead can define +// the conversion login in the tfdyn package. func BundleToTerraform(config *config.Root) *schema.Root { tfroot := schema.NewRoot() tfroot.Provider = schema.NewProviders() @@ -382,6 +384,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.QualityMonitors[resource.Name] = cur + case "databricks_schema": + if config.Resources.Schemas == nil { + config.Resources.Schemas = make(map[string]*resources.Schema) + } + cur := config.Resources.Schemas[resource.Name] + if cur == nil { + cur = &resources.Schema{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Schemas[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -426,6 +438,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Schemas { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 7ea448538..e4ef6114a 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -655,6 +655,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -681,6 +689,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -736,6 +747,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + }, }, } var tfState = resourcesState{ @@ -765,6 +783,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -855,6 +876,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + "test_schema_new": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -971,6 +1004,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "test_monitor_old"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1024,6 +1073,14 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus) assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Schemas["test_schema_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/destroy.go b/bundle/deploy/terraform/destroy.go deleted file mode 100644 index 16f074a22..000000000 --- a/bundle/deploy/terraform/destroy.go +++ /dev/null @@ -1,124 +0,0 @@ -package terraform - -import ( - "context" - "fmt" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/fatih/color" - "github.com/hashicorp/terraform-exec/tfexec" - tfjson "github.com/hashicorp/terraform-json" -) - -type PlanResourceChange struct { - ResourceType string `json:"resource_type"` - Action string `json:"action"` - ResourceName string `json:"resource_name"` -} - -func (c *PlanResourceChange) String() string { - result := strings.Builder{} - switch c.Action { - case "delete": - result.WriteString(" delete ") - default: - result.WriteString(c.Action + " ") - } - switch c.ResourceType { - case "databricks_job": - result.WriteString("job ") - case "databricks_pipeline": - result.WriteString("pipeline ") - default: - result.WriteString(c.ResourceType + " ") - } - result.WriteString(c.ResourceName) - return result.String() -} - -func (c *PlanResourceChange) IsInplaceSupported() bool { - return false -} - -func logDestroyPlan(ctx context.Context, changes []*tfjson.ResourceChange) error { - cmdio.LogString(ctx, "The following resources will be removed:") - for _, c := range changes { - if c.Change.Actions.Delete() { - cmdio.Log(ctx, &PlanResourceChange{ - ResourceType: c.Type, - Action: "delete", - ResourceName: c.Name, - }) - } - } - return nil -} - -type destroy struct{} - -func (w *destroy) Name() string { - return "terraform.Destroy" -} - -func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // return early if plan is empty - if b.Plan.IsEmpty { - cmdio.LogString(ctx, "No resources to destroy in plan. Skipping destroy!") - return nil - } - - tf := b.Terraform - if tf == nil { - return diag.Errorf("terraform not initialized") - } - - // read plan file - plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) - if err != nil { - return diag.FromErr(err) - } - - // print the resources that will be destroyed - err = logDestroyPlan(ctx, plan.ResourceChanges) - if err != nil { - return diag.FromErr(err) - } - - // Ask for confirmation, if needed - if !b.Plan.ConfirmApply { - red := color.New(color.FgRed).SprintFunc() - b.Plan.ConfirmApply, err = cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nThis will permanently %s resources! Proceed?", red("destroy"))) - if err != nil { - return diag.FromErr(err) - } - } - - // return if confirmation was not provided - if !b.Plan.ConfirmApply { - return nil - } - - if b.Plan.Path == "" { - return diag.Errorf("no plan found") - } - - cmdio.LogString(ctx, "Starting to destroy resources") - - // Apply terraform according to the computed destroy plan - err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) - if err != nil { - return diag.Errorf("terraform destroy: %v", err) - } - - cmdio.LogString(ctx, "Successfully destroyed resources!") - return nil -} - -// Destroy returns a [bundle.Mutator] that runs the conceptual equivalent of -// `terraform destroy ./plan` from the bundle's ephemeral working directory for Terraform. -func Destroy() bundle.Mutator { - return &destroy{} -} diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index d480242ce..e7f720d08 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" @@ -219,8 +220,10 @@ func setProxyEnvVars(ctx context.Context, environ map[string]string, b *bundle.B } func setUserAgentExtraEnvVar(environ map[string]string, b *bundle.Bundle) error { - var products []string - + // Add "cli" to the user agent in set by the Databricks Terraform provider. + // This will allow us to attribute downstream requests made by the Databricks + // Terraform provider to the CLI. + products := []string{fmt.Sprintf("cli/%s", build.GetInfo().Version)} if experimental := b.Config.Experimental; experimental != nil { if experimental.PyDABs.Enabled { products = append(products, "databricks-pydabs/0.0.0") diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index aa9b2f77f..94e47dbc1 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -262,10 +262,9 @@ func TestSetUserAgentExtraEnvVar(t *testing.T) { env := make(map[string]string, 0) err := setUserAgentExtraEnvVar(env, b) - require.NoError(t, err) assert.Equal(t, map[string]string{ - "DATABRICKS_USER_AGENT_EXTRA": "databricks-pydabs/0.0.0", + "DATABRICKS_USER_AGENT_EXTRA": "cli/0.0.0-dev databricks-pydabs/0.0.0", }, env) } diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 608f1c795..faa098e1c 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -56,6 +56,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...) case dyn.Key("quality_monitors"): path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) + case dyn.Key("schemas"): + path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 9af4a1443..5ceb243bc 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -30,6 +30,7 @@ func TestInterpolate(t *testing.T) { "other_experiment": "${resources.experiments.other_experiment.id}", "other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}", "other_registered_model": "${resources.registered_models.other_registered_model.id}", + "other_schema": "${resources.schemas.other_schema.id}", }, Tasks: []jobs.Task{ { @@ -65,6 +66,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_mlflow_experiment.other_experiment.id}", j.Tags["other_experiment"]) assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"]) assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) + assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/plan.go b/bundle/deploy/terraform/plan.go index 50e0f78ca..72f0b49a8 100644 --- a/bundle/deploy/terraform/plan.go +++ b/bundle/deploy/terraform/plan.go @@ -6,8 +6,8 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/terraform" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -33,8 +33,6 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Starting plan computation") - err := tf.Init(ctx, tfexec.Upgrade(true)) if err != nil { return diag.Errorf("terraform init: %v", err) @@ -55,12 +53,11 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Set plan in main bundle struct for downstream mutators b.Plan = &terraform.Plan{ - Path: planPath, - ConfirmApply: b.AutoApprove, - IsEmpty: !notEmpty, + Path: planPath, + IsEmpty: !notEmpty, } - cmdio.LogString(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) + log.Debugf(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) return nil } diff --git a/bundle/deploy/terraform/state_pull.go b/bundle/deploy/terraform/state_pull.go index cc7d34274..9a5b91007 100644 --- a/bundle/deploy/terraform/state_pull.go +++ b/bundle/deploy/terraform/state_pull.go @@ -1,8 +1,8 @@ package terraform import ( - "bytes" "context" + "encoding/json" "errors" "io" "io/fs" @@ -12,10 +12,14 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) +type tfState struct { + Serial int64 `json:"serial"` + Lineage string `json:"lineage"` +} + type statePull struct { filerFactory deploy.FilerFactory } @@ -24,74 +28,105 @@ func (l *statePull) Name() string { return "terraform:state-pull" } -func (l *statePull) remoteState(ctx context.Context, f filer.Filer) (*bytes.Buffer, error) { - // Download state file from filer to local cache directory. - remote, err := f.Read(ctx, TerraformStateFileName) +func (l *statePull) remoteState(ctx context.Context, b *bundle.Bundle) (*tfState, []byte, error) { + f, err := l.filerFactory(b) if err != nil { - // On first deploy this state file doesn't yet exist. - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, err + return nil, nil, err } - defer remote.Close() + r, err := f.Read(ctx, TerraformStateFileName) + if err != nil { + return nil, nil, err + } + defer r.Close() - var buf bytes.Buffer - _, err = io.Copy(&buf, remote) + content, err := io.ReadAll(r) + if err != nil { + return nil, nil, err + } + + state := &tfState{} + err = json.Unmarshal(content, state) + if err != nil { + return nil, nil, err + } + + return state, content, nil +} + +func (l *statePull) localState(ctx context.Context, b *bundle.Bundle) (*tfState, error) { + dir, err := Dir(ctx, b) if err != nil { return nil, err } - return &buf, nil + content, err := os.ReadFile(filepath.Join(dir, TerraformStateFileName)) + if err != nil { + return nil, err + } + + state := &tfState{} + err = json.Unmarshal(content, state) + if err != nil { + return nil, err + } + + return state, nil } func (l *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - f, err := l.filerFactory(b) - if err != nil { - return diag.FromErr(err) - } - dir, err := Dir(ctx, b) if err != nil { return diag.FromErr(err) } - // Download state file from filer to local cache directory. - log.Infof(ctx, "Opening remote state file") - remote, err := l.remoteState(ctx, f) - if err != nil { - log.Infof(ctx, "Unable to open remote state file: %s", err) - return diag.FromErr(err) - } - if remote == nil { - log.Infof(ctx, "Remote state file does not exist") + localStatePath := filepath.Join(dir, TerraformStateFileName) + + // Case: Remote state file does not exist. In this case we fallback to using the + // local Terraform state. This allows users to change the "root_path" their bundle is + // configured with. + remoteState, remoteContent, err := l.remoteState(ctx, b) + if errors.Is(err, fs.ErrNotExist) { + log.Infof(ctx, "Remote state file does not exist. Using local Terraform state.") return nil } - - // Expect the state file to live under dir. - local, err := os.OpenFile(filepath.Join(dir, TerraformStateFileName), os.O_CREATE|os.O_RDWR, 0600) if err != nil { + return diag.Errorf("failed to read remote state file: %v", err) + } + + // Expected invariant: remote state file should have a lineage UUID. Error + // if that's not the case. + if remoteState.Lineage == "" { + return diag.Errorf("remote state file does not have a lineage") + } + + // Case: Local state file does not exist. In this case we should rely on the remote state file. + localState, err := l.localState(ctx, b) + if errors.Is(err, fs.ErrNotExist) { + log.Infof(ctx, "Local state file does not exist. Using remote Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) return diag.FromErr(err) } - defer local.Close() - - if !IsLocalStateStale(local, bytes.NewReader(remote.Bytes())) { - log.Infof(ctx, "Local state is the same or newer, ignoring remote state") - return nil + if err != nil { + return diag.Errorf("failed to read local state file: %v", err) } - // Truncating the file before writing - local.Truncate(0) - local.Seek(0, 0) - - // Write file to disk. - log.Infof(ctx, "Writing remote state file to local cache directory") - _, err = io.Copy(local, bytes.NewReader(remote.Bytes())) - if err != nil { + // If the lineage does not match, the Terraform state files do not correspond to the same deployment. + if localState.Lineage != remoteState.Lineage { + log.Infof(ctx, "Remote and local state lineages do not match. Using remote Terraform state. Invalidating local Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) return diag.FromErr(err) } + // If the remote state is newer than the local state, we should use the remote state. + if remoteState.Serial > localState.Serial { + log.Infof(ctx, "Remote state is newer than local state. Using remote Terraform state.") + err := os.WriteFile(localStatePath, remoteContent, 0600) + return diag.FromErr(err) + } + + // default: local state is newer or equal to remote state in terms of serial sequence. + // It is also of the same lineage. Keep using the local state. return nil } diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index 26297bfcb..39937a3cc 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/mock" ) -func mockStateFilerForPull(t *testing.T, contents map[string]int, merr error) filer.Filer { +func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) filer.Filer { buf, err := json.Marshal(contents) assert.NoError(t, err) @@ -41,86 +41,123 @@ func statePullTestBundle(t *testing.T) *bundle.Bundle { } } -func TestStatePullLocalMissingRemoteMissing(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, nil, os.ErrNotExist)), - } +func TestStatePullLocalErrorWhenRemoteHasNoLineage(t *testing.T) { + m := &statePull{} - ctx := context.Background() - b := statePullTestBundle(t) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) + t.Run("no local state", func(t *testing.T) { + // setup remote state. + m.filerFactory = identityFiler(mockStateFilerForPull(t, map[string]any{"serial": 5}, nil)) - // Confirm that no local state file has been written. - _, err := os.Stat(localStateFile(t, ctx, b)) - assert.ErrorIs(t, err, fs.ErrNotExist) + ctx := context.Background() + b := statePullTestBundle(t) + diags := bundle.Apply(ctx, b, m) + assert.EqualError(t, diags.Error(), "remote state file does not have a lineage") + }) + + t.Run("local state with lineage", func(t *testing.T) { + // setup remote state. + m.filerFactory = identityFiler(mockStateFilerForPull(t, map[string]any{"serial": 5}, nil)) + + ctx := context.Background() + b := statePullTestBundle(t) + writeLocalState(t, ctx, b, map[string]any{"serial": 5, "lineage": "aaaa"}) + + diags := bundle.Apply(ctx, b, m) + assert.EqualError(t, diags.Error(), "remote state file does not have a lineage") + }) } -func TestStatePullLocalMissingRemotePresent(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5}, nil)), +func TestStatePullLocal(t *testing.T) { + tcases := []struct { + name string + + // remote state before applying the pull mutators + remote map[string]any + + // local state before applying the pull mutators + local map[string]any + + // expected local state after applying the pull mutators + expected map[string]any + }{ + { + name: "remote missing, local missing", + remote: nil, + local: nil, + expected: nil, + }, + { + name: "remote missing, local present", + remote: nil, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // fallback to local state, since remote state is missing. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local stale", + remote: map[string]any{"serial": 10, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // use remote, since remote is newer. + expected: map[string]any{"serial": float64(10), "lineage": "aaaa", "some_other_key": float64(123)}, + }, + { + name: "local equal", + remote: map[string]any{"serial": 5, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 5, "lineage": "aaaa"}, + // use local state, since they are equal in terms of serial sequence. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local newer", + remote: map[string]any{"serial": 5, "lineage": "aaaa", "some_other_key": 123}, + local: map[string]any{"serial": 6, "lineage": "aaaa"}, + // use local state, since local is newer. + expected: map[string]any{"serial": float64(6), "lineage": "aaaa"}, + }, + { + name: "remote and local have different lineages", + remote: map[string]any{"serial": 5, "lineage": "aaaa"}, + local: map[string]any{"serial": 10, "lineage": "bbbb"}, + // use remote, since lineages do not match. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, + { + name: "local is missing lineage", + remote: map[string]any{"serial": 5, "lineage": "aaaa"}, + local: map[string]any{"serial": 10}, + // use remote, since local does not have lineage. + expected: map[string]any{"serial": float64(5), "lineage": "aaaa"}, + }, } - ctx := context.Background() - b := statePullTestBundle(t) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + m := &statePull{} + if tc.remote == nil { + // nil represents no remote state file. + m.filerFactory = identityFiler(mockStateFilerForPull(t, nil, os.ErrNotExist)) + } else { + m.filerFactory = identityFiler(mockStateFilerForPull(t, tc.remote, nil)) + } - // Confirm that the local state file has been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} + ctx := context.Background() + b := statePullTestBundle(t) + if tc.local != nil { + writeLocalState(t, ctx, b, tc.local) + } -func TestStatePullLocalStale(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5}, nil)), + diags := bundle.Apply(ctx, b, m) + assert.NoError(t, diags.Error()) + + if tc.expected == nil { + // nil represents no local state file is expected. + _, err := os.Stat(localStateFile(t, ctx, b)) + assert.ErrorIs(t, err, fs.ErrNotExist) + } else { + localState := readLocalState(t, ctx, b) + assert.Equal(t, tc.expected, localState) + + } + }) } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a stale local state file. - writeLocalState(t, ctx, b, map[string]int{"serial": 4}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} - -func TestStatePullLocalEqual(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5, "some_other_key": 123}, nil)), - } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a local state file with the same serial as the remote. - writeLocalState(t, ctx, b, map[string]int{"serial": 5}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has not been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 5}, localState) -} - -func TestStatePullLocalNewer(t *testing.T) { - m := &statePull{ - identityFiler(mockStateFilerForPull(t, map[string]int{"serial": 5, "some_other_key": 123}, nil)), - } - - ctx := context.Background() - b := statePullTestBundle(t) - - // Write a local state file with a newer serial as the remote. - writeLocalState(t, ctx, b, map[string]int{"serial": 6}) - diags := bundle.Apply(ctx, b, m) - assert.NoError(t, diags.Error()) - - // Confirm that the local state file has not been updated. - localState := readLocalState(t, ctx, b) - assert.Equal(t, map[string]int{"serial": 6}, localState) } diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index b50983bd4..6cdde1371 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -2,6 +2,8 @@ package terraform import ( "context" + "errors" + "io/fs" "os" "path/filepath" @@ -34,6 +36,12 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic // Expect the state file to live under dir. local, err := os.Open(filepath.Join(dir, TerraformStateFileName)) + if errors.Is(err, fs.ErrNotExist) { + // The state file can be absent if terraform apply is skipped because + // there are no changes to apply in the plan. + log.Debugf(ctx, "Local terraform state file does not exist.") + return nil + } if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index e054773f3..ac74f345d 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -55,7 +55,7 @@ func TestStatePush(t *testing.T) { b := statePushTestBundle(t) // Write a stale local state file. - writeLocalState(t, ctx, b, map[string]int{"serial": 4}) + writeLocalState(t, ctx, b, map[string]any{"serial": 4}) diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } diff --git a/bundle/deploy/terraform/state_test.go b/bundle/deploy/terraform/state_test.go index ff3250625..73d7cb0de 100644 --- a/bundle/deploy/terraform/state_test.go +++ b/bundle/deploy/terraform/state_test.go @@ -26,19 +26,19 @@ func localStateFile(t *testing.T, ctx context.Context, b *bundle.Bundle) string return filepath.Join(dir, TerraformStateFileName) } -func readLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle) map[string]int { +func readLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle) map[string]any { f, err := os.Open(localStateFile(t, ctx, b)) require.NoError(t, err) defer f.Close() - var contents map[string]int + var contents map[string]any dec := json.NewDecoder(f) err = dec.Decode(&contents) require.NoError(t, err) return contents } -func writeLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle, contents map[string]int) { +func writeLocalState(t *testing.T, ctx context.Context, b *bundle.Bundle, contents map[string]any) { f, err := os.Create(localStateFile(t, ctx, b)) require.NoError(t, err) defer f.Close() diff --git a/bundle/deploy/terraform/tfdyn/convert_schema.go b/bundle/deploy/terraform/tfdyn/convert_schema.go new file mode 100644 index 000000000..b5e6a88c0 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema.go @@ -0,0 +1,53 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertSchemaResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + v, diags := convert.Normalize(schema.ResourceSchema{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "schema normalization diagnostic: %s", diag.Summary) + } + + // We always set force destroy as it allows DABs to manage the lifecycle + // of the schema. It's the responsibility of the CLI to ensure the user + // is adequately warned when they try to delete a UC schema. + vout, err := dyn.SetByPath(v, dyn.MustPathFromString("force_destroy"), dyn.V(true)) + if err != nil { + return dyn.InvalidValue, err + } + + return vout, nil +} + +type schemaConverter struct{} + +func (schemaConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertSchemaResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Schema[key] = vout.AsAny() + + // Configure grants for this resource. + if grants := convertGrantsResource(ctx, vin); grants != nil { + grants.Schema = fmt.Sprintf("${databricks_schema.%s.id}", key) + out.Grants["schema_"+key] = grants + } + + return nil +} + +func init() { + registerConverter("schemas", schemaConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_schema_test.go b/bundle/deploy/terraform/tfdyn/convert_schema_test.go new file mode 100644 index 000000000..2efbf3e43 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema_test.go @@ -0,0 +1,75 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertSchema(t *testing.T) { + var src = resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "name", + CatalogName: "catalog", + Comment: "comment", + Properties: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + StorageRoot: "root", + }, + Grants: []resources.Grant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = schemaConverter{}.Convert(ctx, "my_schema", vin, out) + require.NoError(t, err) + + // Assert equality on the schema + assert.Equal(t, map[string]any{ + "name": "name", + "catalog_name": "catalog", + "comment": "comment", + "properties": map[string]any{ + "k1": "v1", + "k2": "v2", + }, + "force_destroy": true, + "storage_root": "root", + }, out.Schema["my_schema"]) + + // Assert equality on the grants + assert.Equal(t, &schema.ResourceGrants{ + Schema: "${databricks_schema.my_schema.id}", + Grant: []schema.ResourceGrantsGrant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + }, out.Grants["schema_my_schema"]) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 1a8a83ac7..64d667b5f 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "io" "os" "path/filepath" @@ -22,10 +21,6 @@ type resourcesState struct { const SupportedStateVersion = 4 -type serialState struct { - Serial int `json:"serial"` -} - type stateResource struct { Type string `json:"type"` Name string `json:"name"` @@ -41,34 +36,6 @@ type stateInstanceAttributes struct { ID string `json:"id"` } -func IsLocalStateStale(local io.Reader, remote io.Reader) bool { - localState, err := loadState(local) - if err != nil { - return true - } - - remoteState, err := loadState(remote) - if err != nil { - return false - } - - return localState.Serial < remoteState.Serial -} - -func loadState(input io.Reader) (*serialState, error) { - content, err := io.ReadAll(input) - if err != nil { - return nil, err - } - var s serialState - err = json.Unmarshal(content, &s) - if err != nil { - return nil, err - } - - return &s, nil -} - func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) { cacheDir, err := Dir(ctx, b) if err != nil { diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go index 8949ebca8..251a7c256 100644 --- a/bundle/deploy/terraform/util_test.go +++ b/bundle/deploy/terraform/util_test.go @@ -2,48 +2,15 @@ package terraform import ( "context" - "fmt" "os" "path/filepath" - "strings" "testing" - "testing/iotest" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/stretchr/testify/assert" ) -func TestLocalStateIsNewer(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 4}`) - assert.False(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateIsOlder(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 6}`) - assert.True(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateIsTheSame(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := strings.NewReader(`{"serial": 5}`) - assert.False(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateMarkStaleWhenFailsToLoad(t *testing.T) { - local := iotest.ErrReader(fmt.Errorf("Random error")) - remote := strings.NewReader(`{"serial": 5}`) - assert.True(t, IsLocalStateStale(local, remote)) -} - -func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) { - local := strings.NewReader(`{"serial": 5}`) - remote := iotest.ErrReader(fmt.Errorf("Random error")) - assert.False(t, IsLocalStateStale(local, remote)) -} - func TestParseResourcesStateWithNoFile(t *testing.T) { b := &bundle.Bundle{ RootPath: t.TempDir(), diff --git a/bundle/deployer/deployer.go b/bundle/deployer/deployer.go deleted file mode 100644 index f4f718975..000000000 --- a/bundle/deployer/deployer.go +++ /dev/null @@ -1,192 +0,0 @@ -package deployer - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - - "github.com/databricks/cli/libs/locker" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" - "github.com/hashicorp/terraform-exec/tfexec" -) - -type DeploymentStatus int - -const ( - // Empty plan produced on terraform plan. No changes need to be applied - NoChanges DeploymentStatus = iota - - // Deployment failed. No databricks assets were deployed - Failed - - // Deployment failed/partially succeeded. failed to update remote terraform - // state file. - // The partially deployed resources are thus untracked and in most cases - // will need to be cleaned up manually - PartialButUntracked - - // Deployment failed/partially succeeded. Remote terraform state file is - // updated with any partially deployed resources - Partial - - // Deployment succeeded however the remote terraform state was not updated. - // The deployed resources are thus untracked and in most cases will need to - // be cleaned up manually - CompleteButUntracked - - // Deployment succeeeded with remote terraform state file updated - Complete -) - -// Deployer is a struct to deploy a DAB to a databricks workspace -// -// Here's a high level description of what a deploy looks like: -// -// 1. Client compiles the bundle configuration to a terraform HCL config file -// -// 2. Client tries to acquire a lock on the remote root of the project. -// -- If FAIL: print details about current holder of the deployment lock on -// remote root and terminate deployment -// -// 3. Client reads terraform state from remote root -// -// 4. Client applies the diff in terraform config to the databricks workspace -// -// 5. Client updates terraform state file in remote root -// -// 6. Client releases the deploy lock on remote root -type Deployer struct { - localRoot string - remoteRoot string - env string - locker *locker.Locker - wsc *databricks.WorkspaceClient -} - -func Create(ctx context.Context, env, localRoot, remoteRoot string, wsc *databricks.WorkspaceClient) (*Deployer, error) { - user, err := wsc.CurrentUser.Me(ctx) - if err != nil { - return nil, err - } - newLocker, err := locker.CreateLocker(user.UserName, remoteRoot, wsc) - if err != nil { - return nil, err - } - return &Deployer{ - localRoot: localRoot, - remoteRoot: remoteRoot, - env: env, - locker: newLocker, - wsc: wsc, - }, nil -} - -func (b *Deployer) DefaultTerraformRoot() string { - return filepath.Join(b.localRoot, ".databricks/bundle", b.env) -} - -func (b *Deployer) tfStateRemotePath() string { - // Note: remote paths are scoped to `remoteRoot` through the locker. Also see [Create]. - return ".bundle/terraform.tfstate" -} - -func (b *Deployer) tfStateLocalPath() string { - return filepath.Join(b.DefaultTerraformRoot(), "terraform.tfstate") -} - -func (d *Deployer) LoadTerraformState(ctx context.Context) error { - r, err := d.locker.Read(ctx, d.tfStateRemotePath()) - if errors.Is(err, fs.ErrNotExist) { - // If remote tf state is absent, use local tf state - return nil - } - if err != nil { - return err - } - defer r.Close() - err = os.MkdirAll(d.DefaultTerraformRoot(), os.ModeDir) - if err != nil { - return err - } - b, err := io.ReadAll(r) - if err != nil { - return err - } - return os.WriteFile(d.tfStateLocalPath(), b, os.ModePerm) -} - -func (b *Deployer) SaveTerraformState(ctx context.Context) error { - bytes, err := os.ReadFile(b.tfStateLocalPath()) - if err != nil { - return err - } - return b.locker.Write(ctx, b.tfStateRemotePath(), bytes) -} - -func (d *Deployer) Lock(ctx context.Context, isForced bool) error { - return d.locker.Lock(ctx, isForced) -} - -func (d *Deployer) Unlock(ctx context.Context) error { - return d.locker.Unlock(ctx) -} - -func (d *Deployer) ApplyTerraformConfig(ctx context.Context, configPath, terraformBinaryPath string, isForced bool) (DeploymentStatus, error) { - applyErr := d.Lock(ctx, isForced) - if applyErr != nil { - return Failed, applyErr - } - defer func() { - applyErr = d.Unlock(ctx) - if applyErr != nil { - log.Errorf(ctx, "failed to unlock deployment mutex: %s", applyErr) - } - }() - - applyErr = d.LoadTerraformState(ctx) - if applyErr != nil { - log.Debugf(ctx, "failed to load terraform state from workspace: %s", applyErr) - return Failed, applyErr - } - - tf, applyErr := tfexec.NewTerraform(configPath, terraformBinaryPath) - if applyErr != nil { - log.Debugf(ctx, "failed to construct terraform object: %s", applyErr) - return Failed, applyErr - } - - isPlanNotEmpty, applyErr := tf.Plan(ctx) - if applyErr != nil { - log.Debugf(ctx, "failed to compute terraform plan: %s", applyErr) - return Failed, applyErr - } - - if !isPlanNotEmpty { - log.Debugf(ctx, "terraform plan returned a empty diff") - return NoChanges, nil - } - - applyErr = tf.Apply(ctx) - // upload state even if apply fails to handle partial deployments - saveStateErr := d.SaveTerraformState(ctx) - - if applyErr != nil && saveStateErr != nil { - log.Errorf(ctx, "terraform apply failed: %s", applyErr) - log.Errorf(ctx, "failed to upload terraform state after partial terraform apply: %s", saveStateErr) - return PartialButUntracked, fmt.Errorf("deploymented failed: %s", applyErr) - } - if applyErr != nil { - log.Errorf(ctx, "terraform apply failed: %s", applyErr) - return Partial, fmt.Errorf("deploymented failed: %s", applyErr) - } - if saveStateErr != nil { - log.Errorf(ctx, "failed to upload terraform state after completing terraform apply: %s", saveStateErr) - return CompleteButUntracked, fmt.Errorf("failed to upload terraform state file: %s", saveStateErr) - } - return Complete, nil -} diff --git a/bundle/if.go b/bundle/if.go new file mode 100644 index 000000000..bad1d72d2 --- /dev/null +++ b/bundle/if.go @@ -0,0 +1,40 @@ +package bundle + +import ( + "context" + + "github.com/databricks/cli/libs/diag" +) + +type ifMutator struct { + condition func(context.Context, *Bundle) (bool, error) + onTrueMutator Mutator + onFalseMutator Mutator +} + +func If( + condition func(context.Context, *Bundle) (bool, error), + onTrueMutator Mutator, + onFalseMutator Mutator, +) Mutator { + return &ifMutator{ + condition, onTrueMutator, onFalseMutator, + } +} + +func (m *ifMutator) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { + v, err := m.condition(ctx, b) + if err != nil { + return diag.FromErr(err) + } + + if v { + return Apply(ctx, b, m.onTrueMutator) + } else { + return Apply(ctx, b, m.onFalseMutator) + } +} + +func (m *ifMutator) Name() string { + return "If" +} diff --git a/bundle/if_test.go b/bundle/if_test.go new file mode 100644 index 000000000..b3fc0b9d9 --- /dev/null +++ b/bundle/if_test.go @@ -0,0 +1,53 @@ +package bundle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIfMutatorTrue(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return true, nil + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.NoError(t, diags.Error()) + + assert.Equal(t, 1, m1.applyCalled) + assert.Equal(t, 0, m2.applyCalled) +} + +func TestIfMutatorFalse(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return false, nil + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.NoError(t, diags.Error()) + + assert.Equal(t, 0, m1.applyCalled) + assert.Equal(t, 1, m2.applyCalled) +} + +func TestIfMutatorError(t *testing.T) { + m1 := &testMutator{} + m2 := &testMutator{} + ifMutator := If(func(context.Context, *Bundle) (bool, error) { + return true, assert.AnError + }, m1, m2) + + b := &Bundle{} + diags := Apply(context.Background(), b, ifMutator) + assert.Error(t, diags.Error()) + + assert.Equal(t, 0, m1.applyCalled) + assert.Equal(t, 0, m2.applyCalled) +} diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index 1fd6f968c..380d6e17d 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -14,9 +14,9 @@ func SetLocation(b *bundle.Bundle, prefix string, filePath string) { return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { // If the path has the given prefix, set the location. if p.HasPrefix(start) { - return v.WithLocation(dyn.Location{ + return v.WithLocations([]dyn.Location{{ File: filePath, - }), nil + }}), nil } // The path is not nested under the given prefix. @@ -29,6 +29,4 @@ func SetLocation(b *bundle.Bundle, prefix string, filePath string) { return v, dyn.ErrSkip }) }) - - b.Config.ConfigureConfigFilePath() } diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index a99f15a40..efb297243 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.48.0" +const ProviderVersion = "1.50.0" diff --git a/bundle/internal/tf/schema/config.go b/bundle/internal/tf/schema/config.go index a2de987ec..e807cdc53 100644 --- a/bundle/internal/tf/schema/config.go +++ b/bundle/internal/tf/schema/config.go @@ -3,34 +3,36 @@ package schema type Config struct { - AccountId string `json:"account_id,omitempty"` - AuthType string `json:"auth_type,omitempty"` - AzureClientId string `json:"azure_client_id,omitempty"` - AzureClientSecret string `json:"azure_client_secret,omitempty"` - AzureEnvironment string `json:"azure_environment,omitempty"` - AzureLoginAppId string `json:"azure_login_app_id,omitempty"` - AzureTenantId string `json:"azure_tenant_id,omitempty"` - AzureUseMsi bool `json:"azure_use_msi,omitempty"` - AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` - ClientId string `json:"client_id,omitempty"` - ClientSecret string `json:"client_secret,omitempty"` - ClusterId string `json:"cluster_id,omitempty"` - ConfigFile string `json:"config_file,omitempty"` - DatabricksCliPath string `json:"databricks_cli_path,omitempty"` - DebugHeaders bool `json:"debug_headers,omitempty"` - DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` - GoogleCredentials string `json:"google_credentials,omitempty"` - GoogleServiceAccount string `json:"google_service_account,omitempty"` - Host string `json:"host,omitempty"` - HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` - MetadataServiceUrl string `json:"metadata_service_url,omitempty"` - Password string `json:"password,omitempty"` - Profile string `json:"profile,omitempty"` - RateLimit int `json:"rate_limit,omitempty"` - RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` - ServerlessComputeId string `json:"serverless_compute_id,omitempty"` - SkipVerify bool `json:"skip_verify,omitempty"` - Token string `json:"token,omitempty"` - Username string `json:"username,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + AccountId string `json:"account_id,omitempty"` + ActionsIdTokenRequestToken string `json:"actions_id_token_request_token,omitempty"` + ActionsIdTokenRequestUrl string `json:"actions_id_token_request_url,omitempty"` + AuthType string `json:"auth_type,omitempty"` + AzureClientId string `json:"azure_client_id,omitempty"` + AzureClientSecret string `json:"azure_client_secret,omitempty"` + AzureEnvironment string `json:"azure_environment,omitempty"` + AzureLoginAppId string `json:"azure_login_app_id,omitempty"` + AzureTenantId string `json:"azure_tenant_id,omitempty"` + AzureUseMsi bool `json:"azure_use_msi,omitempty"` + AzureWorkspaceResourceId string `json:"azure_workspace_resource_id,omitempty"` + ClientId string `json:"client_id,omitempty"` + ClientSecret string `json:"client_secret,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ConfigFile string `json:"config_file,omitempty"` + DatabricksCliPath string `json:"databricks_cli_path,omitempty"` + DebugHeaders bool `json:"debug_headers,omitempty"` + DebugTruncateBytes int `json:"debug_truncate_bytes,omitempty"` + GoogleCredentials string `json:"google_credentials,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + Host string `json:"host,omitempty"` + HttpTimeoutSeconds int `json:"http_timeout_seconds,omitempty"` + MetadataServiceUrl string `json:"metadata_service_url,omitempty"` + Password string `json:"password,omitempty"` + Profile string `json:"profile,omitempty"` + RateLimit int `json:"rate_limit,omitempty"` + RetryTimeoutSeconds int `json:"retry_timeout_seconds,omitempty"` + ServerlessComputeId string `json:"serverless_compute_id,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty"` + Token string `json:"token,omitempty"` + Username string `json:"username,omitempty"` + WarehouseId string `json:"warehouse_id,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index fff66dc93..94d67bbfa 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -10,7 +10,9 @@ type DataSourceClusterClusterInfoAutoscale struct { type DataSourceClusterClusterInfoAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -18,10 +20,16 @@ type DataSourceClusterClusterInfoAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type DataSourceClusterClusterInfoAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type DataSourceClusterClusterInfoClusterLogConfDbfs struct { @@ -49,12 +57,12 @@ type DataSourceClusterClusterInfoClusterLogStatus struct { } type DataSourceClusterClusterInfoDockerImageBasicAuth struct { - Password string `json:"password"` - Username string `json:"username"` + Password string `json:"password,omitempty"` + Username string `json:"username,omitempty"` } type DataSourceClusterClusterInfoDockerImage struct { - Url string `json:"url"` + Url string `json:"url,omitempty"` BasicAuth *DataSourceClusterClusterInfoDockerImageBasicAuth `json:"basic_auth,omitempty"` } @@ -139,12 +147,212 @@ type DataSourceClusterClusterInfoInitScripts struct { Workspace *DataSourceClusterClusterInfoInitScriptsWorkspace `json:"workspace,omitempty"` } +type DataSourceClusterClusterInfoSpecAutoscale struct { + MaxWorkers int `json:"max_workers,omitempty"` + MinWorkers int `json:"min_workers,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAwsAttributes struct { + Availability string `json:"availability,omitempty"` + EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` + EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` + EbsVolumeType string `json:"ebs_volume_type,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + SpotBidPricePercent int `json:"spot_bid_price_percent,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributes struct { + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConf struct { + Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { + MountOptions string `json:"mount_options,omitempty"` + ServerAddress string `json:"server_address"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfo struct { + LocalMountDirPath string `json:"local_mount_dir_path"` + RemoteMountDirPath string `json:"remote_mount_dir_path,omitempty"` + NetworkFilesystemInfo *DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo `json:"network_filesystem_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecDockerImageBasicAuth struct { + Password string `json:"password"` + Username string `json:"username"` +} + +type DataSourceClusterClusterInfoSpecDockerImage struct { + Url string `json:"url"` + BasicAuth *DataSourceClusterClusterInfoSpecDockerImageBasicAuth `json:"basic_auth,omitempty"` +} + +type DataSourceClusterClusterInfoSpecGcpAttributes struct { + Availability string `json:"availability,omitempty"` + BootDiskSize int `json:"boot_disk_size,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + LocalSsdCount int `json:"local_ssd_count,omitempty"` + UsePreemptibleExecutors bool `json:"use_preemptible_executors,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsAbfss struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsFile struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsGcs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsVolumes struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsWorkspace struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScripts struct { + Abfss *DataSourceClusterClusterInfoSpecInitScriptsAbfss `json:"abfss,omitempty"` + Dbfs *DataSourceClusterClusterInfoSpecInitScriptsDbfs `json:"dbfs,omitempty"` + File *DataSourceClusterClusterInfoSpecInitScriptsFile `json:"file,omitempty"` + Gcs *DataSourceClusterClusterInfoSpecInitScriptsGcs `json:"gcs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecInitScriptsS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoSpecInitScriptsVolumes `json:"volumes,omitempty"` + Workspace *DataSourceClusterClusterInfoSpecInitScriptsWorkspace `json:"workspace,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceClusterClusterInfoSpecLibraryCran `json:"cran,omitempty"` + Maven *DataSourceClusterClusterInfoSpecLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceClusterClusterInfoSpecLibraryPypi `json:"pypi,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadType struct { + Clients *DataSourceClusterClusterInfoSpecWorkloadTypeClients `json:"clients,omitempty"` +} + +type DataSourceClusterClusterInfoSpec struct { + ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + DataSecurityMode string `json:"data_security_mode,omitempty"` + DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` + DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` + IdempotencyToken string `json:"idempotency_token,omitempty"` + InstancePoolId string `json:"instance_pool_id,omitempty"` + NodeTypeId string `json:"node_type_id,omitempty"` + NumWorkers int `json:"num_workers,omitempty"` + PolicyId string `json:"policy_id,omitempty"` + RuntimeEngine string `json:"runtime_engine,omitempty"` + SingleUserName string `json:"single_user_name,omitempty"` + SparkConf map[string]string `json:"spark_conf,omitempty"` + SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` + SparkVersion string `json:"spark_version"` + SshPublicKeys []string `json:"ssh_public_keys,omitempty"` + Autoscale *DataSourceClusterClusterInfoSpecAutoscale `json:"autoscale,omitempty"` + AwsAttributes *DataSourceClusterClusterInfoSpecAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *DataSourceClusterClusterInfoSpecAzureAttributes `json:"azure_attributes,omitempty"` + ClusterLogConf *DataSourceClusterClusterInfoSpecClusterLogConf `json:"cluster_log_conf,omitempty"` + ClusterMountInfo []DataSourceClusterClusterInfoSpecClusterMountInfo `json:"cluster_mount_info,omitempty"` + DockerImage *DataSourceClusterClusterInfoSpecDockerImage `json:"docker_image,omitempty"` + GcpAttributes *DataSourceClusterClusterInfoSpecGcpAttributes `json:"gcp_attributes,omitempty"` + InitScripts []DataSourceClusterClusterInfoSpecInitScripts `json:"init_scripts,omitempty"` + Library []DataSourceClusterClusterInfoSpecLibrary `json:"library,omitempty"` + WorkloadType *DataSourceClusterClusterInfoSpecWorkloadType `json:"workload_type,omitempty"` +} + type DataSourceClusterClusterInfoTerminationReason struct { Code string `json:"code,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` Type string `json:"type,omitempty"` } +type DataSourceClusterClusterInfoWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoWorkloadType struct { + Clients *DataSourceClusterClusterInfoWorkloadTypeClients `json:"clients,omitempty"` +} + type DataSourceClusterClusterInfo struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterCores int `json:"cluster_cores,omitempty"` @@ -155,14 +363,14 @@ type DataSourceClusterClusterInfo struct { CreatorUserName string `json:"creator_user_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags"` + DefaultTags map[string]string `json:"default_tags,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` JdbcPort int `json:"jdbc_port,omitempty"` - LastActivityTime int `json:"last_activity_time,omitempty"` + LastRestartedTime int `json:"last_restarted_time,omitempty"` LastStateLossTime int `json:"last_state_loss_time,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` @@ -172,12 +380,12 @@ type DataSourceClusterClusterInfo struct { SparkConf map[string]string `json:"spark_conf,omitempty"` SparkContextId int `json:"spark_context_id,omitempty"` SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` + SparkVersion string `json:"spark_version,omitempty"` SshPublicKeys []string `json:"ssh_public_keys,omitempty"` StartTime int `json:"start_time,omitempty"` - State string `json:"state"` + State string `json:"state,omitempty"` StateMessage string `json:"state_message,omitempty"` - TerminateTime int `json:"terminate_time,omitempty"` + TerminatedTime int `json:"terminated_time,omitempty"` Autoscale *DataSourceClusterClusterInfoAutoscale `json:"autoscale,omitempty"` AwsAttributes *DataSourceClusterClusterInfoAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *DataSourceClusterClusterInfoAzureAttributes `json:"azure_attributes,omitempty"` @@ -188,7 +396,9 @@ type DataSourceClusterClusterInfo struct { Executors []DataSourceClusterClusterInfoExecutors `json:"executors,omitempty"` GcpAttributes *DataSourceClusterClusterInfoGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []DataSourceClusterClusterInfoInitScripts `json:"init_scripts,omitempty"` + Spec *DataSourceClusterClusterInfoSpec `json:"spec,omitempty"` TerminationReason *DataSourceClusterClusterInfoTerminationReason `json:"termination_reason,omitempty"` + WorkloadType *DataSourceClusterClusterInfoWorkloadType `json:"workload_type,omitempty"` } type DataSourceCluster struct { diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go index 727848ced..91806d670 100644 --- a/bundle/internal/tf/schema/data_source_job.go +++ b/bundle/internal/tf/schema/data_source_job.go @@ -1224,6 +1224,11 @@ type DataSourceJobJobSettingsSettingsTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } +type DataSourceJobJobSettingsSettingsTriggerPeriodic struct { + Interval int `json:"interval"` + Unit string `json:"unit"` +} + type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1234,6 +1239,7 @@ type DataSourceJobJobSettingsSettingsTriggerTableUpdate struct { type DataSourceJobJobSettingsSettingsTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *DataSourceJobJobSettingsSettingsTriggerFileArrival `json:"file_arrival,omitempty"` + Periodic *DataSourceJobJobSettingsSettingsTriggerPeriodic `json:"periodic,omitempty"` TableUpdate *DataSourceJobJobSettingsSettingsTriggerTableUpdate `json:"table_update,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_notebook.go b/bundle/internal/tf/schema/data_source_notebook.go index ebfbe2dfb..bf97c19a8 100644 --- a/bundle/internal/tf/schema/data_source_notebook.go +++ b/bundle/internal/tf/schema/data_source_notebook.go @@ -3,11 +3,12 @@ package schema type DataSourceNotebook struct { - Content string `json:"content,omitempty"` - Format string `json:"format"` - Id string `json:"id,omitempty"` - Language string `json:"language,omitempty"` - ObjectId int `json:"object_id,omitempty"` - ObjectType string `json:"object_type,omitempty"` - Path string `json:"path"` + Content string `json:"content,omitempty"` + Format string `json:"format"` + Id string `json:"id,omitempty"` + Language string `json:"language,omitempty"` + ObjectId int `json:"object_id,omitempty"` + ObjectType string `json:"object_type,omitempty"` + Path string `json:"path"` + WorkspacePath string `json:"workspace_path,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_schema.go b/bundle/internal/tf/schema/data_source_schema.go new file mode 100644 index 000000000..9d778cc88 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_schema.go @@ -0,0 +1,36 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag struct { + InheritedFromName string `json:"inherited_from_name,omitempty"` + InheritedFromType string `json:"inherited_from_type,omitempty"` + Value string `json:"value"` +} + +type DataSourceSchemaSchemaInfo struct { + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + CatalogType string `json:"catalog_type,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + SchemaId string `json:"schema_id,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + EffectivePredictiveOptimizationFlag *DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"` +} + +type DataSourceSchema struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + SchemaInfo *DataSourceSchemaSchemaInfo `json:"schema_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_user.go b/bundle/internal/tf/schema/data_source_user.go index 78981f29b..ea20c066e 100644 --- a/bundle/internal/tf/schema/data_source_user.go +++ b/bundle/internal/tf/schema/data_source_user.go @@ -4,6 +4,7 @@ package schema type DataSourceUser struct { AclPrincipalId string `json:"acl_principal_id,omitempty"` + Active bool `json:"active,omitempty"` Alphanumeric string `json:"alphanumeric,omitempty"` ApplicationId string `json:"application_id,omitempty"` DisplayName string `json:"display_name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_volume.go b/bundle/internal/tf/schema/data_source_volume.go new file mode 100644 index 000000000..67e6100f6 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_volume.go @@ -0,0 +1,38 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails struct { + Algorithm string `json:"algorithm,omitempty"` + AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"` +} + +type DataSourceVolumeVolumeInfoEncryptionDetails struct { + SseEncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"` +} + +type DataSourceVolumeVolumeInfo struct { + AccessPoint string `json:"access_point,omitempty"` + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + VolumeId string `json:"volume_id,omitempty"` + VolumeType string `json:"volume_type,omitempty"` + EncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetails `json:"encryption_details,omitempty"` +} + +type DataSourceVolume struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + VolumeInfo *DataSourceVolumeVolumeInfo `json:"volume_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index b68df2b40..4ac78613f 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -36,6 +36,7 @@ type DataSources struct { Notebook map[string]any `json:"databricks_notebook,omitempty"` NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` Schemas map[string]any `json:"databricks_schemas,omitempty"` ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` @@ -50,6 +51,7 @@ type DataSources struct { Tables map[string]any `json:"databricks_tables,omitempty"` User map[string]any `json:"databricks_user,omitempty"` Views map[string]any `json:"databricks_views,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` Volumes map[string]any `json:"databricks_volumes,omitempty"` Zones map[string]any `json:"databricks_zones,omitempty"` } @@ -89,6 +91,7 @@ func NewDataSources() *DataSources { Notebook: make(map[string]any), NotebookPaths: make(map[string]any), Pipelines: make(map[string]any), + Schema: make(map[string]any), Schemas: make(map[string]any), ServicePrincipal: make(map[string]any), ServicePrincipals: make(map[string]any), @@ -103,6 +106,7 @@ func NewDataSources() *DataSources { Tables: make(map[string]any), User: make(map[string]any), Views: make(map[string]any), + Volume: make(map[string]any), Volumes: make(map[string]any), Zones: make(map[string]any), } diff --git a/bundle/internal/tf/schema/resource_cluster_policy.go b/bundle/internal/tf/schema/resource_cluster_policy.go index d8111fef2..7e15a7b12 100644 --- a/bundle/internal/tf/schema/resource_cluster_policy.go +++ b/bundle/internal/tf/schema/resource_cluster_policy.go @@ -33,7 +33,7 @@ type ResourceClusterPolicy struct { Description string `json:"description,omitempty"` Id string `json:"id,omitempty"` MaxClustersPerUser int `json:"max_clusters_per_user,omitempty"` - Name string `json:"name"` + Name string `json:"name,omitempty"` PolicyFamilyDefinitionOverrides string `json:"policy_family_definition_overrides,omitempty"` PolicyFamilyId string `json:"policy_family_id,omitempty"` PolicyId string `json:"policy_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_dashboard.go b/bundle/internal/tf/schema/resource_dashboard.go new file mode 100644 index 000000000..0c2fa4a0f --- /dev/null +++ b/bundle/internal/tf/schema/resource_dashboard.go @@ -0,0 +1,21 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceDashboard struct { + CreateTime string `json:"create_time,omitempty"` + DashboardChangeDetected bool `json:"dashboard_change_detected,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DisplayName string `json:"display_name"` + EmbedCredentials bool `json:"embed_credentials,omitempty"` + Etag string `json:"etag,omitempty"` + FilePath string `json:"file_path,omitempty"` + Id string `json:"id,omitempty"` + LifecycleState string `json:"lifecycle_state,omitempty"` + Md5 string `json:"md5,omitempty"` + ParentPath string `json:"parent_path"` + Path string `json:"path,omitempty"` + SerializedDashboard string `json:"serialized_dashboard,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + WarehouseId string `json:"warehouse_id"` +} diff --git a/bundle/internal/tf/schema/resource_external_location.go b/bundle/internal/tf/schema/resource_external_location.go index af64c677c..da28271bc 100644 --- a/bundle/internal/tf/schema/resource_external_location.go +++ b/bundle/internal/tf/schema/resource_external_location.go @@ -18,6 +18,7 @@ type ResourceExternalLocation struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` diff --git a/bundle/internal/tf/schema/resource_metastore_data_access.go b/bundle/internal/tf/schema/resource_metastore_data_access.go index 155730055..ef8c34aa7 100644 --- a/bundle/internal/tf/schema/resource_metastore_data_access.go +++ b/bundle/internal/tf/schema/resource_metastore_data_access.go @@ -20,6 +20,12 @@ type ResourceMetastoreDataAccessAzureServicePrincipal struct { DirectoryId string `json:"directory_id"` } +type ResourceMetastoreDataAccessCloudflareApiToken struct { + AccessKeyId string `json:"access_key_id"` + AccountId string `json:"account_id"` + SecretAccessKey string `json:"secret_access_key"` +} + type ResourceMetastoreDataAccessDatabricksGcpServiceAccount struct { CredentialId string `json:"credential_id,omitempty"` Email string `json:"email,omitempty"` @@ -37,6 +43,7 @@ type ResourceMetastoreDataAccess struct { ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` IsDefault bool `json:"is_default,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` @@ -45,6 +52,7 @@ type ResourceMetastoreDataAccess struct { AwsIamRole *ResourceMetastoreDataAccessAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceMetastoreDataAccessAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceMetastoreDataAccessAzureServicePrincipal `json:"azure_service_principal,omitempty"` + CloudflareApiToken *ResourceMetastoreDataAccessCloudflareApiToken `json:"cloudflare_api_token,omitempty"` DatabricksGcpServiceAccount *ResourceMetastoreDataAccessDatabricksGcpServiceAccount `json:"databricks_gcp_service_account,omitempty"` GcpServiceAccountKey *ResourceMetastoreDataAccessGcpServiceAccountKey `json:"gcp_service_account_key,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index f5ffbbe5e..379807a5d 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -10,43 +10,60 @@ type ResourceModelServingConfigAutoCaptureConfig struct { } type ResourceModelServingConfigServedEntitiesExternalModelAi21LabsConfig struct { - Ai21LabsApiKey string `json:"ai21labs_api_key"` + Ai21LabsApiKey string `json:"ai21labs_api_key,omitempty"` + Ai21LabsApiKeyPlaintext string `json:"ai21labs_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelAmazonBedrockConfig struct { - AwsAccessKeyId string `json:"aws_access_key_id"` - AwsRegion string `json:"aws_region"` - AwsSecretAccessKey string `json:"aws_secret_access_key"` - BedrockProvider string `json:"bedrock_provider"` + AwsAccessKeyId string `json:"aws_access_key_id,omitempty"` + AwsAccessKeyIdPlaintext string `json:"aws_access_key_id_plaintext,omitempty"` + AwsRegion string `json:"aws_region"` + AwsSecretAccessKey string `json:"aws_secret_access_key,omitempty"` + AwsSecretAccessKeyPlaintext string `json:"aws_secret_access_key_plaintext,omitempty"` + BedrockProvider string `json:"bedrock_provider"` } type ResourceModelServingConfigServedEntitiesExternalModelAnthropicConfig struct { - AnthropicApiKey string `json:"anthropic_api_key"` + AnthropicApiKey string `json:"anthropic_api_key,omitempty"` + AnthropicApiKeyPlaintext string `json:"anthropic_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelCohereConfig struct { - CohereApiKey string `json:"cohere_api_key"` + CohereApiBase string `json:"cohere_api_base,omitempty"` + CohereApiKey string `json:"cohere_api_key,omitempty"` + CohereApiKeyPlaintext string `json:"cohere_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServingConfig struct { - DatabricksApiToken string `json:"databricks_api_token"` - DatabricksWorkspaceUrl string `json:"databricks_workspace_url"` + DatabricksApiToken string `json:"databricks_api_token,omitempty"` + DatabricksApiTokenPlaintext string `json:"databricks_api_token_plaintext,omitempty"` + DatabricksWorkspaceUrl string `json:"databricks_workspace_url"` +} + +type ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig struct { + PrivateKey string `json:"private_key,omitempty"` + PrivateKeyPlaintext string `json:"private_key_plaintext,omitempty"` + ProjectId string `json:"project_id,omitempty"` + Region string `json:"region,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct { - MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"` - MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"` - MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"` - OpenaiApiBase string `json:"openai_api_base,omitempty"` - OpenaiApiKey string `json:"openai_api_key,omitempty"` - OpenaiApiType string `json:"openai_api_type,omitempty"` - OpenaiApiVersion string `json:"openai_api_version,omitempty"` - OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` - OpenaiOrganization string `json:"openai_organization,omitempty"` + MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"` + MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"` + MicrosoftEntraClientSecretPlaintext string `json:"microsoft_entra_client_secret_plaintext,omitempty"` + MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"` + OpenaiApiBase string `json:"openai_api_base,omitempty"` + OpenaiApiKey string `json:"openai_api_key,omitempty"` + OpenaiApiKeyPlaintext string `json:"openai_api_key_plaintext,omitempty"` + OpenaiApiType string `json:"openai_api_type,omitempty"` + OpenaiApiVersion string `json:"openai_api_version,omitempty"` + OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` + OpenaiOrganization string `json:"openai_organization,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelPalmConfig struct { - PalmApiKey string `json:"palm_api_key"` + PalmApiKey string `json:"palm_api_key,omitempty"` + PalmApiKeyPlaintext string `json:"palm_api_key_plaintext,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModel struct { @@ -58,6 +75,7 @@ type ResourceModelServingConfigServedEntitiesExternalModel struct { AnthropicConfig *ResourceModelServingConfigServedEntitiesExternalModelAnthropicConfig `json:"anthropic_config,omitempty"` CohereConfig *ResourceModelServingConfigServedEntitiesExternalModelCohereConfig `json:"cohere_config,omitempty"` DatabricksModelServingConfig *ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServingConfig `json:"databricks_model_serving_config,omitempty"` + GoogleCloudVertexAiConfig *ResourceModelServingConfigServedEntitiesExternalModelGoogleCloudVertexAiConfig `json:"google_cloud_vertex_ai_config,omitempty"` OpenaiConfig *ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig `json:"openai_config,omitempty"` PalmConfig *ResourceModelServingConfigServedEntitiesExternalModelPalmConfig `json:"palm_config,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_notebook.go b/bundle/internal/tf/schema/resource_notebook.go index 8fb5a5387..4e5d4cbc3 100644 --- a/bundle/internal/tf/schema/resource_notebook.go +++ b/bundle/internal/tf/schema/resource_notebook.go @@ -13,4 +13,5 @@ type ResourceNotebook struct { Path string `json:"path"` Source string `json:"source,omitempty"` Url string `json:"url,omitempty"` + WorkspacePath string `json:"workspace_path,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_notification_destination.go b/bundle/internal/tf/schema/resource_notification_destination.go new file mode 100644 index 000000000..0ed9cff60 --- /dev/null +++ b/bundle/internal/tf/schema/resource_notification_destination.go @@ -0,0 +1,46 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceNotificationDestinationConfigEmail struct { + Addresses []string `json:"addresses,omitempty"` +} + +type ResourceNotificationDestinationConfigGenericWebhook struct { + Password string `json:"password,omitempty"` + PasswordSet bool `json:"password_set,omitempty"` + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` + Username string `json:"username,omitempty"` + UsernameSet bool `json:"username_set,omitempty"` +} + +type ResourceNotificationDestinationConfigMicrosoftTeams struct { + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` +} + +type ResourceNotificationDestinationConfigPagerduty struct { + IntegrationKey string `json:"integration_key,omitempty"` + IntegrationKeySet bool `json:"integration_key_set,omitempty"` +} + +type ResourceNotificationDestinationConfigSlack struct { + Url string `json:"url,omitempty"` + UrlSet bool `json:"url_set,omitempty"` +} + +type ResourceNotificationDestinationConfig struct { + Email *ResourceNotificationDestinationConfigEmail `json:"email,omitempty"` + GenericWebhook *ResourceNotificationDestinationConfigGenericWebhook `json:"generic_webhook,omitempty"` + MicrosoftTeams *ResourceNotificationDestinationConfigMicrosoftTeams `json:"microsoft_teams,omitempty"` + Pagerduty *ResourceNotificationDestinationConfigPagerduty `json:"pagerduty,omitempty"` + Slack *ResourceNotificationDestinationConfigSlack `json:"slack,omitempty"` +} + +type ResourceNotificationDestination struct { + DestinationType string `json:"destination_type,omitempty"` + DisplayName string `json:"display_name"` + Id string `json:"id,omitempty"` + Config *ResourceNotificationDestinationConfig `json:"config,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index 5d8df11e7..ee94a1a8f 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -13,6 +13,7 @@ type ResourcePermissions struct { Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` DirectoryId string `json:"directory_id,omitempty"` DirectoryPath string `json:"directory_path,omitempty"` ExperimentId string `json:"experiment_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 20c25c1e2..154686463 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -3,15 +3,17 @@ package schema type ResourcePipelineClusterAutoscale struct { - MaxWorkers int `json:"max_workers,omitempty"` - MinWorkers int `json:"min_workers,omitempty"` + MaxWorkers int `json:"max_workers"` + MinWorkers int `json:"min_workers"` Mode string `json:"mode,omitempty"` } type ResourcePipelineClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -19,10 +21,16 @@ type ResourcePipelineClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourcePipelineClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourcePipelineClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourcePipelineClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourcePipelineClusterClusterLogConfDbfs struct { @@ -127,8 +135,69 @@ type ResourcePipelineFilters struct { Include []string `json:"include,omitempty"` } +type ResourcePipelineGatewayDefinition struct { + ConnectionId string `json:"connection_id,omitempty"` + GatewayStorageCatalog string `json:"gateway_storage_catalog,omitempty"` + GatewayStorageName string `json:"gateway_storage_name,omitempty"` + GatewayStorageSchema string `json:"gateway_storage_schema,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsSchema struct { + DestinationCatalog string `json:"destination_catalog,omitempty"` + DestinationSchema string `json:"destination_schema,omitempty"` + SourceCatalog string `json:"source_catalog,omitempty"` + SourceSchema string `json:"source_schema,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsTable struct { + DestinationCatalog string `json:"destination_catalog,omitempty"` + DestinationSchema string `json:"destination_schema,omitempty"` + DestinationTable string `json:"destination_table,omitempty"` + SourceCatalog string `json:"source_catalog,omitempty"` + SourceSchema string `json:"source_schema,omitempty"` + SourceTable string `json:"source_table,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjects struct { + Schema *ResourcePipelineIngestionDefinitionObjectsSchema `json:"schema,omitempty"` + Table *ResourcePipelineIngestionDefinitionObjectsTable `json:"table,omitempty"` +} + +type ResourcePipelineIngestionDefinitionTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` +} + +type ResourcePipelineIngestionDefinition struct { + ConnectionName string `json:"connection_name,omitempty"` + IngestionGatewayId string `json:"ingestion_gateway_id,omitempty"` + Objects []ResourcePipelineIngestionDefinitionObjects `json:"objects,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionTableConfiguration `json:"table_configuration,omitempty"` +} + +type ResourcePipelineLatestUpdates struct { + CreationTime string `json:"creation_time,omitempty"` + State string `json:"state,omitempty"` + UpdateId string `json:"update_id,omitempty"` +} + type ResourcePipelineLibraryFile struct { - Path string `json:"path"` + Path string `json:"path,omitempty"` } type ResourcePipelineLibraryMaven struct { @@ -138,7 +207,7 @@ type ResourcePipelineLibraryMaven struct { } type ResourcePipelineLibraryNotebook struct { - Path string `json:"path"` + Path string `json:"path,omitempty"` } type ResourcePipelineLibrary struct { @@ -150,28 +219,53 @@ type ResourcePipelineLibrary struct { } type ResourcePipelineNotification struct { - Alerts []string `json:"alerts"` - EmailRecipients []string `json:"email_recipients"` + Alerts []string `json:"alerts,omitempty"` + EmailRecipients []string `json:"email_recipients,omitempty"` +} + +type ResourcePipelineTriggerCron struct { + QuartzCronSchedule string `json:"quartz_cron_schedule,omitempty"` + TimezoneId string `json:"timezone_id,omitempty"` +} + +type ResourcePipelineTriggerManual struct { +} + +type ResourcePipelineTrigger struct { + Cron *ResourcePipelineTriggerCron `json:"cron,omitempty"` + Manual *ResourcePipelineTriggerManual `json:"manual,omitempty"` } type ResourcePipeline struct { - AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` - Catalog string `json:"catalog,omitempty"` - Channel string `json:"channel,omitempty"` - Configuration map[string]string `json:"configuration,omitempty"` - Continuous bool `json:"continuous,omitempty"` - Development bool `json:"development,omitempty"` - Edition string `json:"edition,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Photon bool `json:"photon,omitempty"` - Serverless bool `json:"serverless,omitempty"` - Storage string `json:"storage,omitempty"` - Target string `json:"target,omitempty"` - Url string `json:"url,omitempty"` - Cluster []ResourcePipelineCluster `json:"cluster,omitempty"` - Deployment *ResourcePipelineDeployment `json:"deployment,omitempty"` - Filters *ResourcePipelineFilters `json:"filters,omitempty"` - Library []ResourcePipelineLibrary `json:"library,omitempty"` - Notification []ResourcePipelineNotification `json:"notification,omitempty"` + AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` + Catalog string `json:"catalog,omitempty"` + Cause string `json:"cause,omitempty"` + Channel string `json:"channel,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + Configuration map[string]string `json:"configuration,omitempty"` + Continuous bool `json:"continuous,omitempty"` + CreatorUserName string `json:"creator_user_name,omitempty"` + Development bool `json:"development,omitempty"` + Edition string `json:"edition,omitempty"` + ExpectedLastModified int `json:"expected_last_modified,omitempty"` + Health string `json:"health,omitempty"` + Id string `json:"id,omitempty"` + LastModified int `json:"last_modified,omitempty"` + Name string `json:"name,omitempty"` + Photon bool `json:"photon,omitempty"` + RunAsUserName string `json:"run_as_user_name,omitempty"` + Serverless bool `json:"serverless,omitempty"` + State string `json:"state,omitempty"` + Storage string `json:"storage,omitempty"` + Target string `json:"target,omitempty"` + Url string `json:"url,omitempty"` + Cluster []ResourcePipelineCluster `json:"cluster,omitempty"` + Deployment *ResourcePipelineDeployment `json:"deployment,omitempty"` + Filters *ResourcePipelineFilters `json:"filters,omitempty"` + GatewayDefinition *ResourcePipelineGatewayDefinition `json:"gateway_definition,omitempty"` + IngestionDefinition *ResourcePipelineIngestionDefinition `json:"ingestion_definition,omitempty"` + LatestUpdates []ResourcePipelineLatestUpdates `json:"latest_updates,omitempty"` + Library []ResourcePipelineLibrary `json:"library,omitempty"` + Notification []ResourcePipelineNotification `json:"notification,omitempty"` + Trigger *ResourcePipelineTrigger `json:"trigger,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_storage_credential.go b/bundle/internal/tf/schema/resource_storage_credential.go index b565a5c78..7278c2193 100644 --- a/bundle/internal/tf/schema/resource_storage_credential.go +++ b/bundle/internal/tf/schema/resource_storage_credential.go @@ -20,6 +20,12 @@ type ResourceStorageCredentialAzureServicePrincipal struct { DirectoryId string `json:"directory_id"` } +type ResourceStorageCredentialCloudflareApiToken struct { + AccessKeyId string `json:"access_key_id"` + AccountId string `json:"account_id"` + SecretAccessKey string `json:"secret_access_key"` +} + type ResourceStorageCredentialDatabricksGcpServiceAccount struct { CredentialId string `json:"credential_id,omitempty"` Email string `json:"email,omitempty"` @@ -36,6 +42,7 @@ type ResourceStorageCredential struct { ForceDestroy bool `json:"force_destroy,omitempty"` ForceUpdate bool `json:"force_update,omitempty"` Id string `json:"id,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name"` Owner string `json:"owner,omitempty"` @@ -45,6 +52,7 @@ type ResourceStorageCredential struct { AwsIamRole *ResourceStorageCredentialAwsIamRole `json:"aws_iam_role,omitempty"` AzureManagedIdentity *ResourceStorageCredentialAzureManagedIdentity `json:"azure_managed_identity,omitempty"` AzureServicePrincipal *ResourceStorageCredentialAzureServicePrincipal `json:"azure_service_principal,omitempty"` + CloudflareApiToken *ResourceStorageCredentialCloudflareApiToken `json:"cloudflare_api_token,omitempty"` DatabricksGcpServiceAccount *ResourceStorageCredentialDatabricksGcpServiceAccount `json:"databricks_gcp_service_account,omitempty"` GcpServiceAccountKey *ResourceStorageCredentialGcpServiceAccountKey `json:"gcp_service_account_key,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_workspace_binding.go b/bundle/internal/tf/schema/resource_workspace_binding.go new file mode 100644 index 000000000..f0be7a41f --- /dev/null +++ b/bundle/internal/tf/schema/resource_workspace_binding.go @@ -0,0 +1,12 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceWorkspaceBinding struct { + BindingType string `json:"binding_type,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Id string `json:"id,omitempty"` + SecurableName string `json:"securable_name,omitempty"` + SecurableType string `json:"securable_type,omitempty"` + WorkspaceId int `json:"workspace_id,omitempty"` +} diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79d71a65f..737b77a2a 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -16,6 +16,7 @@ type Resources struct { ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` Connection map[string]any `json:"databricks_connection,omitempty"` + Dashboard map[string]any `json:"databricks_dashboard,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -58,6 +59,7 @@ type Resources struct { MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` Notebook map[string]any `json:"databricks_notebook,omitempty"` + NotificationDestination map[string]any `json:"databricks_notification_destination,omitempty"` OboToken map[string]any `json:"databricks_obo_token,omitempty"` OnlineTable map[string]any `json:"databricks_online_table,omitempty"` PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` @@ -96,6 +98,7 @@ type Resources struct { VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } @@ -115,6 +118,7 @@ func NewResources() *Resources { ClusterPolicy: make(map[string]any), ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), Connection: make(map[string]any), + Dashboard: make(map[string]any), DbfsFile: make(map[string]any), DefaultNamespaceSetting: make(map[string]any), Directory: make(map[string]any), @@ -157,6 +161,7 @@ func NewResources() *Resources { MwsVpcEndpoint: make(map[string]any), MwsWorkspaces: make(map[string]any), Notebook: make(map[string]any), + NotificationDestination: make(map[string]any), OboToken: make(map[string]any), OnlineTable: make(map[string]any), PermissionAssignment: make(map[string]any), @@ -195,6 +200,7 @@ func NewResources() *Resources { VectorSearchEndpoint: make(map[string]any), VectorSearchIndex: make(map[string]any), Volume: make(map[string]any), + WorkspaceBinding: make(map[string]any), WorkspaceConf: make(map[string]any), WorkspaceFile: make(map[string]any), } diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 39db3ea2f..ebdb7f095 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.48.0" +const ProviderVersion = "1.50.0" func NewRoot() *Root { return &Root{ diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go new file mode 100644 index 000000000..9e90a2a17 --- /dev/null +++ b/bundle/libraries/expand_glob_references.go @@ -0,0 +1,221 @@ +package libraries + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expand struct { +} + +func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { + return diag.Diagnostic{ + Severity: diag.Error, + Summary: message, + Paths: []dyn.Path{ + p.Append(), + }, + Locations: l, + } +} + +func getLibDetails(v dyn.Value) (string, string, bool) { + m := v.MustMap() + whl, ok := m.GetByString("whl") + if ok { + return whl.MustString(), "whl", true + } + + jar, ok := m.GetByString("jar") + if ok { + return jar.MustString(), "jar", true + } + + return "", "", false +} + +func findMatches(b *bundle.Bundle, path string) ([]string, error) { + matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) + if err != nil { + return nil, err + } + + if len(matches) == 0 { + if isGlobPattern(path) { + return nil, fmt.Errorf("no files match pattern: %s", path) + } else { + return nil, fmt.Errorf("file doesn't exist %s", path) + } + } + + // We make the matched path relative to the root path before storing it + // to allow upload mutator to distinguish between local and remote paths + for i, match := range matches { + matches[i], err = filepath.Rel(b.RootPath, match) + if err != nil { + return nil, err + } + } + + return matches, nil +} + +// Checks if the path is a glob pattern +// It can contain *, [] or ? characters +func isGlobPattern(path string) bool { + return strings.ContainsAny(path, "*?[") +} + +func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + libs := v.MustSequence() + for i, lib := range libs { + lp := p.Append(dyn.Index(i)) + path, libType, supported := getLibDetails(lib) + if !supported || !IsLibraryLocal(path) { + output = append(output, lib) + continue + } + + lp = lp.Append(dyn.Key(libType)) + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(map[string]dyn.Value{ + libType: dyn.V(match), + }, lib.Locations())) + } + } + + return diags, output +} + +func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + deps := v.MustSequence() + for i, dep := range deps { + lp := p.Append(dyn.Index(i)) + path := dep.MustString() + if !IsLibraryLocal(path) { + output = append(output, dep) + continue + } + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(match, dep.Locations())) + } + } + + return diags, output +} + +type expandPattern struct { + pattern dyn.Pattern + fn func(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) +} + +var taskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("libraries"), +) + +var forEachTaskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("for_each_task"), + dyn.Key("task"), + dyn.Key("libraries"), +) + +var envDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("dependencies"), +) + +func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + expanders := []expandPattern{ + { + pattern: taskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: forEachTaskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: envDepsPattern, + fn: expandEnvironmentDeps, + }, + } + + var diags diag.Diagnostics + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, expander := range expanders { + v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + d, output := expander.fn(b, p, lv) + diags = diags.Extend(d) + return dyn.V(output), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (e *expand) Name() string { + return "libraries.ExpandGlobReferences" +} + +// ExpandGlobReferences expands any glob references in the libraries or environments section +// to corresponding local paths. +// We only expand local paths (i.e. paths that are relative to the root path). +// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to +// distinguish between local and remote paths. +func ExpandGlobReferences() bundle.Mutator { + return &expand{} +} diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go new file mode 100644 index 000000000..34855b539 --- /dev/null +++ b/bundle/libraries/expand_glob_references_test.go @@ -0,0 +1,239 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0] + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0].ForEachTask.Task + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + EnvironmentKey: "env", + }, + }, + Environments: []jobs.JobEnvironment{ + { + EnvironmentKey: "env", + Spec: &compute.Environment{ + Dependencies: []string{ + "./whl/*.whl", + "/Workspace/path/to/whl/my.whl", + "./jar/*.jar", + "/some/local/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + env := job.JobSettings.Environments[0] + require.Equal(t, []string{ + filepath.Join("whl", "my1.whl"), + filepath.Join("whl", "my2.whl"), + "/Workspace/path/to/whl/my.whl", + filepath.Join("jar", "my1.jar"), + filepath.Join("jar", "my2.jar"), + "/some/local/path/to/whl/*.whl", + }, env.Spec.Dependencies) +} diff --git a/bundle/libraries/helpers.go b/bundle/libraries/helpers.go index 89679c91a..b7e707ccf 100644 --- a/bundle/libraries/helpers.go +++ b/bundle/libraries/helpers.go @@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string { if library.Egg != "" { return library.Egg } + if library.Requirements != "" { + return library.Requirements + } return "" } diff --git a/bundle/libraries/helpers_test.go b/bundle/libraries/helpers_test.go index adc20a246..e4bd32770 100644 --- a/bundle/libraries/helpers_test.go +++ b/bundle/libraries/helpers_test.go @@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) { assert.Equal(t, path, libraryPath(&compute.Library{Whl: path})) assert.Equal(t, path, libraryPath(&compute.Library{Jar: path})) assert.Equal(t, path, libraryPath(&compute.Library{Egg: path})) + assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path})) assert.Equal(t, "", libraryPath(&compute.Library{})) } diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 84ead052b..33b848dd9 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -35,7 +35,7 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { } for _, l := range e.Spec.Dependencies { - if IsEnvironmentDependencyLocal(l) { + if IsLibraryLocal(l) { return true } } @@ -44,34 +44,30 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { return false } -func FindAllWheelTasksWithLocalLibraries(b *bundle.Bundle) []*jobs.Task { +func FindTasksWithLocalLibraries(b *bundle.Bundle) []jobs.Task { tasks := findAllTasks(b) envs := FindAllEnvironments(b) - wheelTasks := make([]*jobs.Task, 0) + allTasks := make([]jobs.Task, 0) for k, jobTasks := range tasks { for i := range jobTasks { - task := &jobTasks[i] - if task.PythonWheelTask == nil { - continue + task := jobTasks[i] + if isTaskWithLocalLibraries(task) { + allTasks = append(allTasks, task) } + } - if isTaskWithLocalLibraries(*task) { - wheelTasks = append(wheelTasks, task) - } - - if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { - wheelTasks = append(wheelTasks, task) - } + if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { + allTasks = append(allTasks, jobTasks...) } } - return wheelTasks + return allTasks } func isTaskWithLocalLibraries(task jobs.Task) bool { for _, l := range task.Libraries { - if IsLocalLibrary(&l) { + if IsLibraryLocal(libraryPath(&l)) { return true } } diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index f1e3788f2..417bce10e 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -3,9 +3,8 @@ package libraries import ( "net/url" "path" + "regexp" "strings" - - "github.com/databricks/databricks-sdk-go/service/compute" ) // IsLocalPath returns true if the specified path indicates that @@ -38,12 +37,12 @@ func IsLocalPath(p string) bool { return !path.IsAbs(p) } -// IsEnvironmentDependencyLocal returns true if the specified dependency +// IsLibraryLocal returns true if the specified library or environment dependency // should be interpreted as a local path. -// We use this to check if the dependency in environment spec is local. +// We use this to check if the dependency in environment spec is local or that library is local. // We can't use IsLocalPath beacuse environment dependencies can be // a pypi package name which can be misinterpreted as a local path by IsLocalPath. -func IsEnvironmentDependencyLocal(dep string) bool { +func IsLibraryLocal(dep string) bool { possiblePrefixes := []string{ ".", } @@ -54,7 +53,40 @@ func IsEnvironmentDependencyLocal(dep string) bool { } } - return false + // If the dependency is a requirements file, it's not a valid local path + if strings.HasPrefix(dep, "-r") { + return false + } + + // If the dependency has no extension, it's a PyPi package name + if isPackage(dep) { + return false + } + + return IsLocalPath(dep) +} + +// ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_). +// \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security]. +// ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?)?: Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1). +// Spec for package name and version specifier: https://pip.pypa.io/en/stable/reference/requirement-specifiers/ +var packageRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_]+\s?(\[.*\])?\s?((==|!=|<=|>=|~=|==|>|<)\s?\d+(\.\d+){0,2}(\.\*)?)?$`) + +func isPackage(name string) bool { + if packageRegex.MatchString(name) { + return true + } + + return isUrlBasedLookup(name) +} + +func isUrlBasedLookup(name string) bool { + parts := strings.Split(name, " @ ") + if len(parts) != 2 { + return false + } + + return packageRegex.MatchString(parts[0]) && isRemoteStorageScheme(parts[1]) } func isRemoteStorageScheme(path string) bool { @@ -67,16 +99,6 @@ func isRemoteStorageScheme(path string) bool { return false } - // If the path starts with scheme:/ format, it's a correct remote storage scheme - return strings.HasPrefix(path, url.Scheme+":/") -} - -// IsLocalLibrary returns true if the specified library refers to a local path. -func IsLocalLibrary(library *compute.Library) bool { - path := libraryPath(library) - if path == "" { - return false - } - - return IsLocalPath(path) + // If the path starts with scheme:/ format (not file), it's a correct remote storage scheme + return strings.HasPrefix(path, url.Scheme+":/") && url.Scheme != "file" } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index d2492d6b1..7f84b3244 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -3,13 +3,13 @@ package libraries import ( "testing" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestIsLocalPath(t *testing.T) { // Relative paths, paths with the file scheme, and Windows paths. + assert.True(t, IsLocalPath("some/local/path")) assert.True(t, IsLocalPath("./some/local/path")) assert.True(t, IsLocalPath("file://path/to/package")) assert.True(t, IsLocalPath("C:\\path\\to\\package")) @@ -30,24 +30,13 @@ func TestIsLocalPath(t *testing.T) { assert.False(t, IsLocalPath("abfss://path/to/package")) } -func TestIsLocalLibrary(t *testing.T) { - // Local paths. - assert.True(t, IsLocalLibrary(&compute.Library{Whl: "./file.whl"})) - assert.True(t, IsLocalLibrary(&compute.Library{Jar: "../target/some.jar"})) - - // Non-local paths. - assert.False(t, IsLocalLibrary(&compute.Library{Whl: "/Workspace/path/to/file.whl"})) - assert.False(t, IsLocalLibrary(&compute.Library{Jar: "s3:/bucket/path/some.jar"})) - - // Empty. - assert.False(t, IsLocalLibrary(&compute.Library{})) -} - -func TestIsEnvironmentDependencyLocal(t *testing.T) { +func TestIsLibraryLocal(t *testing.T) { testCases := [](struct { path string expected bool }){ + {path: "local/*.whl", expected: true}, + {path: "local/test.whl", expected: true}, {path: "./local/*.whl", expected: true}, {path: ".\\local\\*.whl", expected: true}, {path: "./local/mypath.whl", expected: true}, @@ -58,15 +47,26 @@ func TestIsEnvironmentDependencyLocal(t *testing.T) { {path: ".\\..\\local\\*.whl", expected: true}, {path: "../../local/*.whl", expected: true}, {path: "..\\..\\local\\*.whl", expected: true}, + {path: "file://path/to/package/whl.whl", expected: true}, {path: "pypipackage", expected: false}, - {path: "pypipackage/test.whl", expected: false}, - {path: "pypipackage/*.whl", expected: false}, {path: "/Volumes/catalog/schema/volume/path.whl", expected: false}, {path: "/Workspace/my_project/dist.whl", expected: false}, {path: "-r /Workspace/my_project/requirements.txt", expected: false}, + {path: "s3://mybucket/path/to/package", expected: false}, + {path: "dbfs:/mnt/path/to/package", expected: false}, + {path: "beautifulsoup4", expected: false}, + {path: "beautifulsoup4==4.12.3", expected: false}, + {path: "beautifulsoup4 >= 4.12.3", expected: false}, + {path: "beautifulsoup4 < 4.12.3", expected: false}, + {path: "beautifulsoup4 ~= 4.12.3", expected: false}, + {path: "beautifulsoup4[security, tests]", expected: false}, + {path: "beautifulsoup4[security, tests] ~= 4.12.3", expected: false}, + {path: "https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, + {path: "pip @ https://github.com/pypa/pip/archive/22.0.2.zip", expected: false}, + {path: "requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip", expected: false}, } - for _, tc := range testCases { - require.Equal(t, IsEnvironmentDependencyLocal(tc.path), tc.expected) + for i, tc := range testCases { + require.Equalf(t, tc.expected, IsLibraryLocal(tc.path), "failed case: %d, path: %s", i, tc.path) } } diff --git a/bundle/libraries/match.go b/bundle/libraries/match.go deleted file mode 100644 index 4feb4225d..000000000 --- a/bundle/libraries/match.go +++ /dev/null @@ -1,82 +0,0 @@ -package libraries - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -type match struct { -} - -func ValidateLocalLibrariesExist() bundle.Mutator { - return &match{} -} - -func (a *match) Name() string { - return "libraries.ValidateLocalLibrariesExist" -} - -func (a *match) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - for _, job := range b.Config.Resources.Jobs { - err := validateEnvironments(job.Environments, b) - if err != nil { - return diag.FromErr(err) - } - - for _, task := range job.JobSettings.Tasks { - err := validateTaskLibraries(task.Libraries, b) - if err != nil { - return diag.FromErr(err) - } - } - } - - return nil -} - -func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error { - for _, lib := range libs { - path := libraryPath(&lib) - if path == "" || !IsLocalPath(path) { - continue - } - - matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) - if err != nil { - return err - } - - if len(matches) == 0 { - return fmt.Errorf("file %s is referenced in libraries section but doesn't exist on the local file system", libraryPath(&lib)) - } - } - - return nil -} - -func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error { - for _, env := range envs { - if env.Spec == nil { - continue - } - - for _, dep := range env.Spec.Dependencies { - matches, err := filepath.Glob(filepath.Join(b.RootPath, dep)) - if err != nil { - return err - } - - if len(matches) == 0 && IsEnvironmentDependencyLocal(dep) { - return fmt.Errorf("file %s is referenced in environments section but doesn't exist on the local file system", dep) - } - } - } - - return nil -} diff --git a/bundle/libraries/match_test.go b/bundle/libraries/match_test.go index bb4b15107..e60504c84 100644 --- a/bundle/libraries/match_test.go +++ b/bundle/libraries/match_test.go @@ -42,7 +42,7 @@ func TestValidateEnvironments(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -74,9 +74,9 @@ func TestValidateEnvironmentsNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in environments section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } func TestValidateTaskLibraries(t *testing.T) { @@ -109,7 +109,7 @@ func TestValidateTaskLibraries(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -142,7 +142,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in libraries section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go new file mode 100644 index 000000000..be7cc41db --- /dev/null +++ b/bundle/libraries/upload.go @@ -0,0 +1,238 @@ +package libraries + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + + "github.com/databricks/databricks-sdk-go" + + "golang.org/x/sync/errgroup" +) + +// The Files API backend has a rate limit of 10 concurrent +// requests and 100 QPS. We limit the number of concurrent requests to 5 to +// avoid hitting the rate limit. +var maxFilesRequestsInFlight = 5 + +func Upload() bundle.Mutator { + return &upload{} +} + +func UploadWithClient(client filer.Filer) bundle.Mutator { + return &upload{ + client: client, + } +} + +type upload struct { + client filer.Filer +} + +type configLocation struct { + configPath dyn.Path + location dyn.Location +} + +// Collect all libraries from the bundle configuration and their config paths. +// By this stage all glob references are expanded and we have a list of all libraries that need to be uploaded. +// We collect them from task libraries, foreach task libraries, environment dependencies, and artifacts. +// We return a map of library source to a list of config paths and locations where the library is used. +// We use map so we don't upload the same library multiple times. +// Instead we upload it once and update all the config paths to point to the uploaded location. +func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error) { + libs := make(map[string]([]configLocation)) + + patterns := []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + envDepsPattern.Append(dyn.AnyIndex()), + } + + for _, pattern := range patterns { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + source, ok := v.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + if !IsLibraryLocal(source) { + return v, nil + } + + source = filepath.Join(b.RootPath, source) + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(), // Hack to get the copy of path + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + } + + artifactPattern := dyn.NewPattern( + dyn.Key("artifacts"), + dyn.AnyKey(), + dyn.Key("files"), + dyn.AnyIndex(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, artifactPattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + file, ok := v.AsMap() + if !ok { + return v, fmt.Errorf("expected map, got %s", v.Kind()) + } + + sv, ok := file.GetByString("source") + if !ok { + return v, nil + } + + source, ok := sv.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(dyn.Key("remote_path")), + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + + return libs, nil +} + +func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + uploadPath, err := GetUploadBasePath(b) + if err != nil { + return diag.FromErr(err) + } + + // If the client is not initialized, initialize it + // We use client field in mutator to allow for mocking client in testing + if u.client == nil { + filer, err := GetFilerForLibraries(b.WorkspaceClient(), uploadPath) + if err != nil { + return diag.FromErr(err) + } + + u.client = filer + } + + var diags diag.Diagnostics + + libs, err := collectLocalLibraries(b) + if err != nil { + return diag.FromErr(err) + } + + errs, errCtx := errgroup.WithContext(ctx) + errs.SetLimit(maxFilesRequestsInFlight) + + for source := range libs { + errs.Go(func() error { + return UploadFile(errCtx, source, u.client) + }) + } + + if err := errs.Wait(); err != nil { + return diag.FromErr(err) + } + + // Update all the config paths to point to the uploaded location + for source, locations := range libs { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + remotePath := path.Join(uploadPath, filepath.Base(source)) + + // If the remote path does not start with /Workspace or /Volumes, prepend /Workspace + if !strings.HasPrefix(remotePath, "/Workspace") && !strings.HasPrefix(remotePath, "/Volumes") { + remotePath = "/Workspace" + remotePath + } + for _, location := range locations { + v, err = dyn.SetByPath(v, location.configPath, dyn.NewValue(remotePath, []dyn.Location{location.location})) + if err != nil { + return v, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + } + + return diags +} + +func (u *upload) Name() string { + return "libraries.Upload" +} + +func GetFilerForLibraries(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { + if isVolumesPath(uploadPath) { + return filer.NewFilesClient(w, uploadPath) + } + return filer.NewWorkspaceFilesClient(w, uploadPath) +} + +func isVolumesPath(path string) bool { + return strings.HasPrefix(path, "/Volumes/") +} + +// Function to upload file (a library, artifact and etc) to Workspace or UC volume +func UploadFile(ctx context.Context, file string, client filer.Filer) error { + filename := filepath.Base(file) + cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) + + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("unable to open %s: %w", file, errors.Unwrap(err)) + } + defer f.Close() + + err = client.Write(ctx, filename, f, filer.OverwriteIfExists, filer.CreateParentDirectories) + if err != nil { + return fmt.Errorf("unable to import %s: %w", filename, err) + } + + log.Infof(ctx, "Upload succeeded") + return nil +} + +func GetUploadBasePath(b *bundle.Bundle) (string, error) { + artifactPath := b.Config.Workspace.ArtifactPath + if artifactPath == "" { + return "", fmt.Errorf("remote artifact path not configured") + } + + return path.Join(artifactPath, ".internal"), nil +} diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go new file mode 100644 index 000000000..82fe6e7c7 --- /dev/null +++ b/bundle/libraries/upload_test.go @@ -0,0 +1,331 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestArtifactUploadForWorkspace(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadForVolumes(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadWithNoLibraryReference(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Workspace/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Artifacts["whl"].Files[0].RemotePath) +} + +func TestUploadMultipleLibraries(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source1.whl") + testutil.Touch(t, whlFolder, "source2.whl") + testutil.Touch(t, whlFolder, "source3.whl") + testutil.Touch(t, whlFolder, "source4.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "*.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source1.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source2.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source3.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source4.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source1.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source2.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source3.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source4.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/Users/foo@bar.com/mywheel.whl"}) + + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source1.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source2.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source3.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source4.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/Users/foo@bar.com/mywheel.whl") +} diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go index 24c268124..37eb0f8d5 100644 --- a/bundle/permissions/permission_diagnostics.go +++ b/bundle/permissions/permission_diagnostics.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/set" ) @@ -35,10 +36,10 @@ func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) dia } return diag.Diagnostics{{ - Severity: diag.Warning, - Summary: fmt.Sprintf("permissions section should include %s or one of their groups with CAN_MANAGE permissions", b.Config.Workspace.CurrentUser.UserName), - Location: b.Config.GetLocation("permissions"), - ID: diag.PermissionNotIncluded, + Severity: diag.Warning, + Summary: fmt.Sprintf("permissions section should include %s or one of their groups with CAN_MANAGE permissions", b.Config.Workspace.CurrentUser.UserName), + Locations: []dyn.Location{b.Config.GetLocation("permissions")}, + ID: diag.PermissionNotIncluded, }} } diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 362d23be1..3ddc6b181 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -16,6 +16,7 @@ func Build() bundle.Mutator { scripts.Execute(config.ScriptPreBuild), artifacts.DetectPackages(), artifacts.InferMissingProperties(), + artifacts.PrepareAll(), artifacts.BuildAll(), scripts.Execute(config.ScriptPostBuild), mutator.ResolveVariableReferences( diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 46c389189..ca967c321 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -1,6 +1,9 @@ package phases import ( + "context" + "fmt" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" @@ -14,10 +17,94 @@ import ( "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/libs/cmdio" + terraformlib "github.com/databricks/cli/libs/terraform" ) +func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + actions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + // We only care about destructive actions on UC schema resources. + if rc.Type != "databricks_schema" { + continue + } + + var actionType terraformlib.ActionType + + switch { + case rc.Change.Actions.Delete(): + actionType = terraformlib.ActionTypeDelete + case rc.Change.Actions.Replace(): + actionType = terraformlib.ActionTypeRecreate + default: + // We don't need a prompt for non-destructive actions like creating + // or updating a schema. + continue + } + + actions = append(actions, terraformlib.Action{ + Action: actionType, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + + // No restricted actions planned. No need for approval. + if len(actions) == 0 { + return true, nil + } + + cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") + for _, action := range actions { + cmdio.Log(ctx, action) + } + + if b.AutoApprove { + return true, nil + } + + if !cmdio.IsPromptSupported(ctx) { + return false, fmt.Errorf("the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") + } + + cmdio.LogString(ctx, "") + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The deploy phase deploys artifacts and resources. func Deploy() bundle.Mutator { + // Core mutators that CRUD resources and modify deployment state. These + // mutators need informed consent if they are potentially destructive. + deployCore := bundle.Defer( + bundle.Seq( + bundle.LogString("Deploying resources..."), + terraform.Apply(), + ), + bundle.Seq( + terraform.StatePush(), + terraform.Load(), + metadata.Compute(), + metadata.Upload(), + bundle.LogString("Deployment complete!"), + ), + ) + deployMutator := bundle.Seq( scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), @@ -26,9 +113,9 @@ func Deploy() bundle.Mutator { terraform.StatePull(), deploy.StatePull(), mutator.ValidateGitDetails(), - libraries.ValidateLocalLibrariesExist(), artifacts.CleanUp(), - artifacts.UploadAll(), + libraries.ExpandGlobReferences(), + libraries.Upload(), python.TransformWheelTask(), files.Upload(), deploy.StateUpdate(), @@ -37,20 +124,16 @@ func Deploy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.CheckRunningResource(), - bundle.Defer( - terraform.Apply(), - bundle.Seq( - terraform.StatePush(), - terraform.Load(), - metadata.Compute(), - metadata.Upload(), - ), + terraform.Plan(terraform.PlanGoal("deploy")), + bundle.If( + approvalForUcSchemaDelete, + deployCore, + bundle.LogString("Deployment cancelled!"), ), ), lock.Release(lock.GoalDeploy), ), scripts.Execute(config.ScriptPostDeploy), - bundle.LogString("Deployment complete!"), ) return newPhase( diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index f974a0565..6eb8b6a01 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -1,14 +1,91 @@ package phases import ( + "context" + "errors" + "fmt" + "net/http" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + + "github.com/databricks/cli/libs/cmdio" + + "github.com/databricks/cli/libs/log" + terraformlib "github.com/databricks/cli/libs/terraform" + "github.com/databricks/databricks-sdk-go/apierr" ) +func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { + w := b.WorkspaceClient() + _, err := w.Workspace.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + + var aerr *apierr.APIError + if errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound { + log.Infof(ctx, "Root path does not exist: %s", b.Config.Workspace.RootPath) + return false, nil + } + + return true, err +} + +func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + deleteActions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + if rc.Change.Actions.Delete() { + deleteActions = append(deleteActions, terraformlib.Action{ + Action: terraformlib.ActionTypeDelete, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + } + + if len(deleteActions) > 0 { + cmdio.LogString(ctx, "The following resources will be deleted:") + for _, a := range deleteActions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + + } + + cmdio.LogString(ctx, fmt.Sprintf("All files and directories at the following location will be deleted: %s", b.Config.Workspace.RootPath)) + cmdio.LogString(ctx, "") + + if b.AutoApprove { + return true, nil + } + + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The destroy phase deletes artifacts and resources. func Destroy() bundle.Mutator { + // Core destructive mutators for destroy. These require informed user consent. + destroyCore := bundle.Seq( + terraform.Apply(), + files.Delete(), + bundle.LogString("Destroy complete!"), + ) destroyMutator := bundle.Seq( lock.Acquire(), @@ -18,17 +95,25 @@ func Destroy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("destroy")), - terraform.Destroy(), - terraform.StatePush(), - files.Delete(), + bundle.If( + approvalForDestroy, + destroyCore, + bundle.LogString("Destroy cancelled!"), + ), ), lock.Release(lock.GoalDestroy), ), - bundle.LogString("Destroy complete!"), ) return newPhase( "destroy", - []bundle.Mutator{destroyMutator}, + []bundle.Mutator{ + // Only run deploy mutator if root path exists. + bundle.If( + assertRootPathExists, + destroyMutator, + bundle.LogString("No active deployment found to destroy!"), + ), + }, ) } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 109667289..aab8ccea8 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/permissions" @@ -19,8 +20,21 @@ func Initialize() bundle.Mutator { return newPhase( "initialize", []bundle.Mutator{ + validate.AllResourcesHaveValues(), + + // Update all path fields in the sync block to be relative to the bundle root path. mutator.RewriteSyncPaths(), + + // Configure the default sync path to equal the bundle root if not explicitly configured. + // By default, this means all files in the bundle root directory are synchronized. + mutator.SyncDefaultPath(), + + // Figure out if the sync root path is identical or an ancestor of the bundle root path. + // If it is an ancestor, this updates all paths to be relative to the sync root path. + mutator.SyncInferRoot(), + mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), @@ -45,6 +59,7 @@ func Initialize() bundle.Mutator { mutator.SetRunAs(), mutator.OverrideCompute(), mutator.ProcessTargetMode(), + mutator.ApplyPresets(), mutator.DefaultQueueing(), mutator.ExpandPipelineGlobPaths(), diff --git a/bundle/python/conditional_transform_test.go b/bundle/python/conditional_transform_test.go index 677970d70..1d397f7a7 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/python/conditional_transform_test.go @@ -2,7 +2,6 @@ package python import ( "context" - "path" "path/filepath" "testing" @@ -18,11 +17,15 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", }, + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { @@ -63,11 +66,15 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", }, + Workspace: config.Workspace{ + FilePath: "/Workspace/files", + }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { @@ -102,14 +109,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { task := b.Config.Resources.Jobs["job1"].Tasks[0] require.Nil(t, task.PythonWheelTask) require.NotNil(t, task.NotebookTask) - - dir, err := b.InternalDir(context.Background()) - require.NoError(t, err) - - internalDirRel, err := filepath.Rel(b.RootPath, dir) - require.NoError(t, err) - - require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath) + require.Equal(t, "/Workspace/files/my_bundle/.databricks/bundle/development/.internal/notebook_job1_key1", task.NotebookTask.NotebookPath) require.Len(t, task.Libraries, 1) require.Equal(t, "/Workspace/Users/test@test.com/bundle/dist/test.jar", task.Libraries[0].Jar) diff --git a/bundle/python/transform.go b/bundle/python/transform.go index 457b45f78..9d3b1ab6a 100644 --- a/bundle/python/transform.go +++ b/bundle/python/transform.go @@ -1,6 +1,7 @@ package python import ( + "context" "fmt" "strconv" "strings" @@ -63,9 +64,10 @@ dbutils.notebook.exit(s) // which installs uploaded wheels using %pip and then calling corresponding // entry point. func TransformWheelTask() bundle.Mutator { - return mutator.If( - func(b *bundle.Bundle) bool { - return b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper + return bundle.If( + func(_ context.Context, b *bundle.Bundle) (bool, error) { + res := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper + return res, nil }, mutator.NewTrampoline( "python_wheel", diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index c15feb424..c7bddca14 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -124,9 +123,6 @@ func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: []jobs.Task{ { diff --git a/bundle/python/warning.go b/bundle/python/warning.go index 3da88b0d7..d53796d73 100644 --- a/bundle/python/warning.go +++ b/bundle/python/warning.go @@ -35,7 +35,7 @@ func isPythonWheelWrapperOn(b *bundle.Bundle) bool { } func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) + tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { if task.NewCluster != nil { if lowerThanExpectedVersion(ctx, task.NewCluster.SparkVersion) { diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 37ea188f7..ea0b9a944 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -29,11 +29,11 @@ var renderFuncMap = template.FuncMap{ } const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -43,11 +43,11 @@ const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} ` const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -141,12 +141,18 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) t = warningT } - // Make file relative to bundle root - if d.Location.File != "" { - out, err := filepath.Rel(b.RootPath, d.Location.File) - // if we can't relativize the path, just use path as-is - if err == nil { - d.Location.File = out + for i := range d.Locations { + if b == nil { + break + } + + // Make location relative to bundle root + if d.Locations[i].File != "" { + out, err := filepath.Rel(b.RootPath, d.Locations[i].File) + // if we can't relativize the path, just use path as-is + if err == nil { + d.Locations[i].File = out + } } } @@ -160,16 +166,25 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) return nil } +// RenderOptions contains options for rendering diagnostics. +type RenderOptions struct { + // variable to include leading new line + + RenderSummaryTable bool +} + // RenderTextOutput renders the diagnostics in a human-readable format. -func RenderTextOutput(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { +func RenderTextOutput(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics, opts RenderOptions) error { err := renderDiagnostics(out, b, diags) if err != nil { return fmt.Errorf("failed to render diagnostics: %w", err) } - err = renderSummaryTemplate(out, b, diags) - if err != nil { - return fmt.Errorf("failed to render summary: %w", err) + if opts.RenderSummaryTable { + err = renderSummaryTemplate(out, b, diags) + if err != nil { + return fmt.Errorf("failed to render summary: %w", err) + } } return nil diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 4ae86ded7..976f86e79 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -17,6 +17,7 @@ type renderTestOutputTestCase struct { name string bundle *bundle.Bundle diags diag.Diagnostics + opts RenderOptions expected string } @@ -39,6 +40,7 @@ func TestRenderTextOutput(t *testing.T) { Summary: "failed to load xxx", }, }, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: failed to load xxx\n" + "\n" + "Found 1 error\n", @@ -47,6 +49,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 1 error", bundle: loadingBundle, diags: diag.Errorf("failed to load bundle"), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: failed to load bundle\n" + "\n" + "Name: test-bundle\n" + @@ -58,6 +61,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 1 warning", bundle: loadingBundle, diags: diag.Warningf("failed to load bundle"), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Warning: failed to load bundle\n" + "\n" + "Name: test-bundle\n" + @@ -69,6 +73,7 @@ func TestRenderTextOutput(t *testing.T) { name: "bundle during 'load' and 2 warnings", bundle: loadingBundle, diags: diag.Warningf("warning (1)").Extend(diag.Warningf("warning (2)")), + opts: RenderOptions{RenderSummaryTable: true}, expected: "Warning: warning (1)\n" + "\n" + "Warning: warning (2)\n" + @@ -83,36 +88,25 @@ func TestRenderTextOutput(t *testing.T) { bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (1)", - Detail: "detail (1)", - Location: dyn.Location{ - File: "foo.py", - Line: 1, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (2)", - Detail: "detail (2)", - Location: dyn.Location{ - File: "foo.py", - Line: 2, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning (3)", - Detail: "detail (3)", - Location: dyn.Location{ - File: "foo.py", - Line: 3, - Column: 1, - }, + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, }, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: error (1)\n" + " in foo.py:1:1\n" + "\n" + @@ -153,6 +147,7 @@ func TestRenderTextOutput(t *testing.T) { }, }, diags: nil, + opts: RenderOptions{RenderSummaryTable: true}, expected: "Name: test-bundle\n" + "Target: test-target\n" + "Workspace:\n" + @@ -162,13 +157,42 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "Validation OK!\n", }, + { + name: "nil bundle without summary with 1 error and 1 warning", + bundle: nil, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: false}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:3:1\n" + + "\n" + + "detail (2)\n" + + "\n", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { writer := &bytes.Buffer{} - err := RenderTextOutput(writer, tc.bundle, tc.diags) + err := RenderTextOutput(writer, tc.bundle, tc.diags, tc.opts) require.NoError(t, err) assert.Equal(t, tc.expected, writer.String()) @@ -208,17 +232,42 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Summary: "failed to load xxx", Detail: "'name' is required", - Location: dyn.Location{ + Locations: []dyn.Location{{ File: "foo.yaml", Line: 1, - Column: 2, - }, + Column: 2}}, }, }, expected: "Error: failed to load xxx\n" + " in foo.yaml:1:2\n\n" + "'name' is required\n\n", }, + { + name: "error with multiple source locations", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "failed to load xxx", + Detail: "'name' is required", + Locations: []dyn.Location{ + { + File: "foo.yaml", + Line: 1, + Column: 2, + }, + { + File: "bar.yaml", + Line: 3, + Column: 4, + }, + }, + }, + }, + expected: "Error: failed to load xxx\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "'name' is required\n\n", + }, { name: "error with path", diags: diag.Diagnostics{ @@ -226,7 +275,7 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Path: dyn.MustPathFromString("resources.jobs.xxx"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.xxx")}, }, }, expected: "Error: failed to load xxx\n" + @@ -234,6 +283,27 @@ func TestRenderDiagnostics(t *testing.T) { "\n" + "'name' is required\n\n", }, + { + name: "error with multiple paths", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Detail: "'name' is required", + Summary: "failed to load xxx", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + dyn.MustPathFromString("resources.jobs.zzz"), + }, + }, + }, + expected: "Error: failed to load xxx\n" + + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " resources.jobs.zzz\n" + + "\n" + + "'name' is required\n\n", + }, } for _, tc := range testCases { diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 4e29b9f3f..d684f8388 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -53,7 +53,7 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId string, u // Otherwise for long lived pipelines, there can be a lot of unnecessary // latency due to multiple pagination API calls needed underneath the hood for // ListPipelineEventsAll - res, err := w.Pipelines.Impl().ListPipelineEvents(ctx, pipelines.ListPipelineEventsRequest{ + events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ Filter: `level='ERROR'`, MaxResults: 100, PipelineId: pipelineId, @@ -61,7 +61,7 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId string, u if err != nil { return err } - updateEvents := filterEventsByUpdateId(res.Events, updateId) + updateEvents := filterEventsByUpdateId(events, updateId) // The events API returns most recent events first. We iterate in a reverse order // to print the events chronologically for i := len(updateEvents) - 1; i >= 0; i-- { diff --git a/bundle/run/progress/pipeline.go b/bundle/run/progress/pipeline.go index fb076f680..4a256e76c 100644 --- a/bundle/run/progress/pipeline.go +++ b/bundle/run/progress/pipeline.go @@ -78,7 +78,7 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { } // we only check the most recent 100 events for progress - response, err := l.w.Pipelines.Impl().ListPipelineEvents(ctx, pipelines.ListPipelineEventsRequest{ + events, err := l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ PipelineId: l.PipelineId, MaxResults: 100, Filter: filter, @@ -89,8 +89,8 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { result := make([]ProgressEvent, 0) // we iterate in reverse to return events in chronological order - for i := len(response.Events) - 1; i >= 0; i-- { - event := response.Events[i] + for i := len(events) - 1; i >= 0; i-- { + event := events[i] // filter to only include update_progress and flow_progress events if event.EventType == "flow_progress" || event.EventType == "update_progress" { result = append(result, ProgressEvent(event)) diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index 5b960ea55..6e9289f92 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" ) // A subset of Schema struct @@ -63,7 +62,7 @@ func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) { if err != nil { return nil, err } - spec := &openapi.Specification{} + spec := &Specification{} err = json.Unmarshal(openapiSpec, spec) if err != nil { return nil, err diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json index 380be0545..d888b3663 100644 --- a/bundle/schema/docs/bundle_descriptions.json +++ b/bundle/schema/docs/bundle_descriptions.json @@ -218,7 +218,7 @@ } }, "description": { - "description": "An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding." + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." }, "edit_mode": { "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." @@ -935,7 +935,7 @@ } }, "egg": { - "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." }, "jar": { "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." @@ -1827,13 +1827,16 @@ } }, "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. When an external_model is present, the served\nentities list can only have one served_entity object. For an existing endpoint with external_model, it can not be updated to an endpoint without external_model.\nIf the endpoint is created without external_model, users cannot update it to add external_model later.\n", + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", "properties": { "ai21labs_config": { "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", "properties": { "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21Labs API key." + "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." + }, + "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." } } }, @@ -1841,13 +1844,19 @@ "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", "properties": { "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS Access Key ID with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." }, "aws_region": { "description": "The AWS region to use. Bedrock has to be enabled there." }, "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS Secret Access Key paired with the access key ID, with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." }, "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." @@ -1858,15 +1867,24 @@ "description": "Anthropic Config. Only required if the provider is 'anthropic'.", "properties": { "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key." + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." } } }, "cohere_config": { "description": "Cohere Config. Only required if the provider is 'cohere'.", "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + }, "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key." + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." } } }, @@ -1874,13 +1892,33 @@ "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", "properties": { "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\n" + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" }, "databricks_workspace_url": { "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" } } }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "properties": { + "private_key": { + "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" + }, + "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." + }, + "project_id": { + "description": "This is the Google Cloud project id that the service account is associated with." + }, + "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." + } + } + }, "name": { "description": "The name of the external model." }, @@ -1891,16 +1929,22 @@ "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" }, "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" + }, + "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" }, "microsoft_entra_tenant_id": { "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" }, "openai_api_base": { - "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" + "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" }, "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI or Azure OpenAI API key." + "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." + }, + "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." }, "openai_api_type": { "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" @@ -1920,12 +1964,15 @@ "description": "PaLM Config. Only required if the provider is 'palm'.", "properties": { "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key." + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." + }, + "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." } } }, "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'openai', and 'palm'.\",\n" + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" }, "task": { "description": "The task type of the external model." @@ -2331,6 +2378,9 @@ "driver_node_type_id": { "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster." + }, "gcp_attributes": { "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", "properties": { @@ -2525,7 +2575,7 @@ "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." }, "gateway_storage_name": { - "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" }, "gateway_storage_schema": { "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." @@ -2565,7 +2615,7 @@ "description": "Required. Schema name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -2605,7 +2655,7 @@ "description": "Required. Table name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -2685,6 +2735,9 @@ "description": "The absolute path of the notebook." } } + }, + "whl": { + "description": "URI of the whl to be installed." } } } @@ -2955,6 +3008,49 @@ } } } + }, + "schemas": { + "description": "", + "additionalproperties": { + "description": "", + "properties": { + "catalog_name": { + "description": "" + }, + "comment": { + "description": "" + }, + "grants": { + "description": "", + "items": { + "description": "", + "properties": { + "principal": { + "description": "" + }, + "privileges": { + "description": "", + "items": { + "description": "" + } + } + } + } + }, + "name": { + "description": "" + }, + "properties": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "storage_root": { + "description": "" + } + } + } } } }, @@ -3194,7 +3290,7 @@ } }, "description": { - "description": "An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding." + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." }, "edit_mode": { "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." @@ -3911,7 +4007,7 @@ } }, "egg": { - "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." }, "jar": { "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." @@ -4803,13 +4899,16 @@ } }, "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. When an external_model is present, the served\nentities list can only have one served_entity object. For an existing endpoint with external_model, it can not be updated to an endpoint without external_model.\nIf the endpoint is created without external_model, users cannot update it to add external_model later.\n", + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", "properties": { "ai21labs_config": { "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", "properties": { "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21Labs API key." + "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." + }, + "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." } } }, @@ -4817,13 +4916,19 @@ "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", "properties": { "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS Access Key ID with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." }, "aws_region": { "description": "The AWS region to use. Bedrock has to be enabled there." }, "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS Secret Access Key paired with the access key ID, with permissions to interact with Bedrock services." + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." }, "bedrock_provider": { "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." @@ -4834,15 +4939,24 @@ "description": "Anthropic Config. Only required if the provider is 'anthropic'.", "properties": { "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key." + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." } } }, "cohere_config": { "description": "Cohere Config. Only required if the provider is 'cohere'.", "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" + }, "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key." + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." } } }, @@ -4850,13 +4964,33 @@ "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", "properties": { "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\n" + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" }, "databricks_workspace_url": { "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" } } }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "properties": { + "private_key": { + "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" + }, + "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." + }, + "project_id": { + "description": "This is the Google Cloud project id that the service account is associated with." + }, + "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." + } + } + }, "name": { "description": "The name of the external model." }, @@ -4867,16 +5001,22 @@ "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" }, "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" + }, + "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" }, "microsoft_entra_tenant_id": { "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" }, "openai_api_base": { - "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" + "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" }, "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI or Azure OpenAI API key." + "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." + }, + "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." }, "openai_api_type": { "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" @@ -4896,12 +5036,15 @@ "description": "PaLM Config. Only required if the provider is 'palm'.", "properties": { "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key." + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." + }, + "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." } } }, "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'openai', and 'palm'.\",\n" + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" }, "task": { "description": "The task type of the external model." @@ -5307,6 +5450,9 @@ "driver_node_type_id": { "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster." + }, "gcp_attributes": { "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", "properties": { @@ -5501,7 +5647,7 @@ "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." }, "gateway_storage_name": { - "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" }, "gateway_storage_schema": { "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." @@ -5541,7 +5687,7 @@ "description": "Required. Schema name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -5581,7 +5727,7 @@ "description": "Required. Table name in the source database." }, "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", "properties": { "primary_keys": { "description": "The primary key of the table used to apply changes.", @@ -5661,6 +5807,9 @@ "description": "The absolute path of the notebook." } } + }, + "whl": { + "description": "URI of the whl to be installed." } } } @@ -5931,6 +6080,49 @@ } } } + }, + "schemas": { + "description": "", + "additionalproperties": { + "description": "", + "properties": { + "catalog_name": { + "description": "" + }, + "comment": { + "description": "" + }, + "grants": { + "description": "", + "items": { + "description": "", + "properties": { + "principal": { + "description": "" + }, + "privileges": { + "description": "", + "items": { + "description": "" + } + } + } + } + }, + "name": { + "description": "" + }, + "properties": { + "description": "", + "additionalproperties": { + "description": "" + } + }, + "storage_root": { + "description": "" + } + } + } } } }, @@ -6010,6 +6202,9 @@ "description": "" } } + }, + "type": { + "description": "" } } } @@ -6115,6 +6310,9 @@ "description": "" } } + }, + "type": { + "description": "" } } } diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go index 1756d5165..0d896b87c 100644 --- a/bundle/schema/openapi.go +++ b/bundle/schema/openapi.go @@ -6,12 +6,11 @@ import ( "strings" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" ) type OpenapiReader struct { // OpenAPI spec to read schemas from. - OpenapiSpec *openapi.Specification + OpenapiSpec *Specification // In-memory cache of schemas read from the OpenAPI spec. memo map[string]jsonschema.Schema diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go index 359b1e58a..4d393cf37 100644 --- a/bundle/schema/openapi_test.go +++ b/bundle/schema/openapi_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/databricks/cli/libs/jsonschema" - "github.com/databricks/databricks-sdk-go/openapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,7 +44,7 @@ func TestReadSchemaForObject(t *testing.T) { } } ` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -103,7 +102,7 @@ func TestReadSchemaForArray(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -149,7 +148,7 @@ func TestReadSchemaForMap(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -198,7 +197,7 @@ func TestRootReferenceIsResolved(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -248,7 +247,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -282,7 +281,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -327,7 +326,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -397,7 +396,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), @@ -460,7 +459,7 @@ func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) { } } }` - spec := &openapi.Specification{} + spec := &Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, memo: make(map[string]jsonschema.Schema), diff --git a/bundle/schema/spec.go b/bundle/schema/spec.go new file mode 100644 index 000000000..fdc31a4ca --- /dev/null +++ b/bundle/schema/spec.go @@ -0,0 +1,11 @@ +package schema + +import "github.com/databricks/cli/libs/jsonschema" + +type Specification struct { + Components *Components `json:"components"` +} + +type Components struct { + Schemas map[string]*jsonschema.Schema `json:"schemas,omitempty"` +} diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 38d204f99..629b3a8ab 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -37,7 +37,7 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmd, out, err := executeHook(ctx, executor, b, m.scriptHook) if err != nil { - return diag.FromErr(err) + return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) } if cmd == nil { log.Debugf(ctx, "No script defined for %s, skipping", m.scriptHook) @@ -53,7 +53,12 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { line, err = reader.ReadString('\n') } - return diag.FromErr(cmd.Wait()) + err = cmd.Wait() + if err != nil { + return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) + } + + return nil } func executeHook(ctx context.Context, executor *exec.Executor, b *bundle.Bundle, hook config.ScriptHook) (exec.Command, io.Reader, error) { diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go deleted file mode 100644 index e7f0aa28f..000000000 --- a/bundle/tests/conflicting_resource_ids_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package config_tests - -import ( - "context" - "fmt" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConflictingResourceIdsNoSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/no_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) -} - -func TestConflictingResourceIdsOneSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/one_subconfiguration") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml") - resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) -} - -func TestConflictingResourceIdsTwoSubconfigs(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/two_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - resources1ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources1.yml") - resources2ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources2.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", resources1ConfigPath, resources2ConfigPath)) -} diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index aed3964db..135ef1917 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -18,6 +18,6 @@ func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) - diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist()) + diags = bundle.Apply(context.Background(), b, libraries.ExpandGlobReferences()) require.Empty(t, diags) } diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index a18daf90c..218d2e470 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -15,7 +14,6 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +27,6 @@ func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +39,12 @@ func TestJobAndPipelineProductionWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/include_test.go b/bundle/tests/include_test.go index 5b0235f60..15f8fcec1 100644 --- a/bundle/tests/include_test.go +++ b/bundle/tests/include_test.go @@ -31,7 +31,8 @@ func TestIncludeWithGlob(t *testing.T) { job := b.Config.Resources.Jobs["my_job"] assert.Equal(t, "1", job.ID) - assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(job.ConfigFilePath)) + l := b.Config.GetLocation("resources.jobs.my_job") + assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(l.File)) } func TestIncludeDefault(t *testing.T) { @@ -51,9 +52,11 @@ func TestIncludeForMultipleMatches(t *testing.T) { first := b.Config.Resources.Jobs["my_first_job"] assert.Equal(t, "1", first.ID) - assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(first.ConfigFilePath)) + fl := b.Config.GetLocation("resources.jobs.my_first_job") + assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(fl.File)) second := b.Config.Resources.Jobs["my_second_job"] assert.Equal(t, "2", second.ID) - assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(second.ConfigFilePath)) + sl := b.Config.GetLocation("resources.jobs.my_second_job") + assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(sl.File)) } diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 5e8febc33..65aa5bdc4 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -15,7 +14,6 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +27,6 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +39,12 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 8eddcf9a1..5c48d81cb 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -8,6 +8,10 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -36,9 +40,37 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { diags := bundle.Apply(ctx, b, bundle.Seq( phases.LoadNamedTarget(env), mutator.RewriteSyncPaths(), + mutator.SyncDefaultPath(), + mutator.SyncInferRoot(), mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), )) return b, diags } + +func configureMock(t *testing.T, b *bundle.Bundle) { + // Configure mock workspace client + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &config.Config{ + Host: "https://mock.databricks.workspace.com", + } + m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ + UserName: "user@domain.com", + }, nil) + b.SetWorkpaceClient(m.WorkspaceClient) +} + +func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diagnostics) { + b := load(t, path) + configureMock(t, b) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + mutator.SelectTarget(env), + phases.Initialize(), + )) + + return b, diags +} diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go index bfa1a31b4..b8b800863 100644 --- a/bundle/tests/model_serving_endpoint_test.go +++ b/bundle/tests/model_serving_endpoint_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) { - assert.Equal(t, "model_serving_endpoint/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "model-name", p.Config.ServedModels[0].ModelName) assert.Equal(t, "1", p.Config.ServedModels[0].ModelVersion) assert.Equal(t, "model-name-1", p.Config.TrafficConfig.Routes[0].ServedModelName) diff --git a/bundle/tests/override_job_parameters/databricks.yml b/bundle/tests/override_job_parameters/databricks.yml new file mode 100644 index 000000000..9c333c323 --- /dev/null +++ b/bundle/tests/override_job_parameters/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: override_job_parameters + +workspace: + host: https://acme.cloud.databricks.com/ + +resources: + jobs: + foo: + name: job + parameters: + - name: foo + default: v1 + - name: bar + default: v1 + +targets: + development: + resources: + jobs: + foo: + parameters: + - name: foo + default: v2 + + staging: + resources: + jobs: + foo: + parameters: + - name: bar + default: v2 diff --git a/bundle/tests/override_job_parameters_test.go b/bundle/tests/override_job_parameters_test.go new file mode 100644 index 000000000..21e0e35a6 --- /dev/null +++ b/bundle/tests/override_job_parameters_test.go @@ -0,0 +1,31 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOverrideJobParametersDev(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "development") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v2", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v1", p[1].Default) +} + +func TestOverrideJobParametersStaging(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "staging") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v1", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v2", p[1].Default) +} diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go index bf5039b5f..c1c62cfb6 100644 --- a/bundle/tests/pipeline_glob_paths_test.go +++ b/bundle/tests/pipeline_glob_paths_test.go @@ -1,33 +1,13 @@ package config_tests import ( - "context" "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestExpandPipelineGlobPaths(t *testing.T) { - b := loadTarget(t, "./pipeline_glob_paths", "default") - - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - ctx := context.Background() - diags := bundle.Apply(ctx, b, phases.Initialize()) + b, diags := initializeTarget(t, "./pipeline_glob_paths", "default") require.NoError(t, diags.Error()) require.Equal( t, @@ -37,19 +17,6 @@ func TestExpandPipelineGlobPaths(t *testing.T) { } func TestExpandPipelineGlobPathsWithNonExistent(t *testing.T) { - b := loadTarget(t, "./pipeline_glob_paths", "error") - - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) - - ctx := context.Background() - diags := bundle.Apply(ctx, b, phases.Initialize()) + _, diags := initializeTarget(t, "./pipeline_glob_paths", "error") require.ErrorContains(t, diags.Error(), "notebook ./non-existent not found") } diff --git a/bundle/tests/presets/databricks.yml b/bundle/tests/presets/databricks.yml new file mode 100644 index 000000000..d83d31801 --- /dev/null +++ b/bundle/tests/presets/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: presets + +presets: + tags: + prod: true + team: finance + pipelines_development: true + +targets: + dev: + presets: + name_prefix: "myprefix" + pipelines_development: true + trigger_pause_status: PAUSED + jobs_max_concurrent_runs: 10 + tags: + dev: true + prod: false + prod: + presets: + pipelines_development: false diff --git a/bundle/tests/presets_test.go b/bundle/tests/presets_test.go new file mode 100644 index 000000000..5fcb5d95b --- /dev/null +++ b/bundle/tests/presets_test.go @@ -0,0 +1,28 @@ +package config_tests + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" +) + +func TestPresetsDev(t *testing.T) { + b := loadTarget(t, "./presets", "dev") + + assert.Equal(t, "myprefix", b.Config.Presets.NamePrefix) + assert.Equal(t, config.Paused, b.Config.Presets.TriggerPauseStatus) + assert.Equal(t, 10, b.Config.Presets.JobsMaxConcurrentRuns) + assert.Equal(t, true, *b.Config.Presets.PipelinesDevelopment) + assert.Equal(t, "true", b.Config.Presets.Tags["dev"]) + assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) + assert.Equal(t, "false", b.Config.Presets.Tags["prod"]) +} + +func TestPresetsProd(t *testing.T) { + b := loadTarget(t, "./presets", "prod") + + assert.Equal(t, false, *b.Config.Presets.PipelinesDevelopment) + assert.Equal(t, "finance", b.Config.Presets.Tags["team"]) + assert.Equal(t, "true", b.Config.Presets.Tags["prod"]) +} diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml new file mode 100644 index 000000000..6964c58a4 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml @@ -0,0 +1,25 @@ +bundle: + name: python-wheel + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + build: "python3 setup.py bdist_wheel" + my_test_code_2: + type: whl + path: "./my_test_code" + build: "python3 setup2.py bdist_wheel" + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py new file mode 100644 index 000000000..0bd871dd3 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py new file mode 100644 index 000000000..424bec9f1 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code_2", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py new file mode 100644 index 000000000..909f1f322 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py new file mode 100644 index 000000000..73d045afb --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml index 1bac4ebad..492861969 100644 --- a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml @@ -13,10 +13,3 @@ resources: entry_point: "run" libraries: - whl: ./package/*.whl - - task_key: TestTask2 - existing_cluster_id: "0717-aaaaa-bbbbbb" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./non-existing/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml new file mode 100644 index 000000000..93e4e6918 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml @@ -0,0 +1,14 @@ +bundle: + name: python-wheel-notebook + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + notebook_task: + notebook_path: "/notebook.py" + libraries: + - whl: ./dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py new file mode 100644 index 000000000..909f1f322 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py new file mode 100644 index 000000000..73d045afb --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py new file mode 100644 index 000000000..24dc150ff --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py @@ -0,0 +1,3 @@ +# Databricks notebook source + +print("Hello, World!") diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py new file mode 100644 index 000000000..7a1317b2f --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group_1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore new file mode 100644 index 000000000..f03e23bc2 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml new file mode 100644 index 000000000..91b8b1556 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml @@ -0,0 +1,16 @@ +bundle: + name: python-wheel + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./dist/*.whl + - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 000000000..4bb80477c Binary files /dev/null and b/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 000000000..4bb80477c Binary files /dev/null and b/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 8d0036a7b..c4d85703c 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -8,6 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/phases" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -23,7 +26,7 @@ func TestPythonWheelBuild(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -40,7 +43,24 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") + require.NoError(t, err) + require.Equal(t, 1, len(matches)) + + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -53,7 +73,7 @@ func TestPythonWheelWithDBFSLib(t *testing.T) { diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) require.NoError(t, diags.Error()) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -63,21 +83,23 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup") require.NoError(t, err) - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + b.Config.Workspace.ArtifactPath = "/foo/bar" + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + u := libraries.UploadWithClient(mockFiler) + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u)) require.NoError(t, diags.Error()) + require.Empty(t, diags) - match := libraries.ValidateLocalLibrariesExist() - diags = bundle.Apply(ctx, b, match) - require.ErrorContains(t, diags.Error(), "./non-existing/*.whl") - - require.NotZero(t, len(b.Config.Artifacts)) - - artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] - require.NotNil(t, artifact) - require.Empty(t, artifact.BuildCommand) - require.Contains(t, artifact.Files[0].Source, filepath.Join(b.RootPath, "package", - "my_test_code-0.0.1-py3-none-any.whl", - )) + require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { @@ -92,7 +114,37 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelBuildMultiple(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") + require.NoError(t, err) + require.Equal(t, 2, len(matches)) + + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelNoBuild(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go index 920a2ac78..008db8bdd 100644 --- a/bundle/tests/registered_model_test.go +++ b/bundle/tests/registered_model_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) { - assert.Equal(t, "registered_model/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "main", p.CatalogName) assert.Equal(t, "default", p.SchemaName) assert.Equal(t, "comment", p.Comment) diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go index d5b80bea5..199871d23 100644 --- a/bundle/tests/relative_path_translation_test.go +++ b/bundle/tests/relative_path_translation_test.go @@ -1,36 +1,14 @@ package config_tests import ( - "context" "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) -func configureMock(t *testing.T, b *bundle.Bundle) { - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) -} - func TestRelativePathTranslationDefault(t *testing.T) { - b := loadTarget(t, "./relative_path_translation", "default") - configureMock(t, b) - - diags := bundle.Apply(context.Background(), b, phases.Initialize()) + b, diags := initializeTarget(t, "./relative_path_translation", "default") require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] @@ -40,10 +18,7 @@ func TestRelativePathTranslationDefault(t *testing.T) { } func TestRelativePathTranslationOverride(t *testing.T) { - b := loadTarget(t, "./relative_path_translation", "override") - configureMock(t, b) - - diags := bundle.Apply(context.Background(), b, phases.Initialize()) + b, diags := initializeTarget(t, "./relative_path_translation", "override") require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] diff --git a/bundle/tests/sync/negate/databricks.yml b/bundle/tests/sync/negate/databricks.yml new file mode 100644 index 000000000..3d591d19b --- /dev/null +++ b/bundle/tests/sync/negate/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: sync_negate + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + exclude: + - ./* + - '!*.txt' + include: + - '*.txt' + +targets: + default: + dev: + sync: + exclude: + - ./* + - '!*.txt2' + include: + - '*.txt' diff --git a/libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} b/bundle/tests/sync/negate/test.txt similarity index 100% rename from libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} rename to bundle/tests/sync/negate/test.txt diff --git a/libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} b/bundle/tests/sync/negate/test.yml similarity index 100% rename from libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} rename to bundle/tests/sync/negate/test.yml diff --git a/bundle/tests/sync/paths/databricks.yml b/bundle/tests/sync/paths/databricks.yml new file mode 100644 index 000000000..9ef6fa032 --- /dev/null +++ b/bundle/tests/sync/paths/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: sync_paths + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + paths: + - src + +targets: + development: + sync: + paths: + - development + + staging: + sync: + paths: + - staging diff --git a/bundle/tests/sync/paths_no_root/databricks.yml b/bundle/tests/sync/paths_no_root/databricks.yml new file mode 100644 index 000000000..df15b12b6 --- /dev/null +++ b/bundle/tests/sync/paths_no_root/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: sync_paths + +workspace: + host: https://acme.cloud.databricks.com/ + +targets: + development: + sync: + paths: + - development + + staging: + sync: + paths: + - staging + + undefined: ~ + + nil: + sync: + paths: ~ + + empty: + sync: + paths: [] diff --git a/bundle/tests/sync/shared_code/bundle/databricks.yml b/bundle/tests/sync/shared_code/bundle/databricks.yml new file mode 100644 index 000000000..738b6170c --- /dev/null +++ b/bundle/tests/sync/shared_code/bundle/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: shared_code + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + paths: + - "../common" + - "." diff --git a/bundle/tests/sync/shared_code/common/library.txt b/bundle/tests/sync/shared_code/common/library.txt new file mode 100644 index 000000000..83b323843 --- /dev/null +++ b/bundle/tests/sync/shared_code/common/library.txt @@ -0,0 +1 @@ +Placeholder for files to be deployed as part of multiple bundles. diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 94cedbaa6..0192b61e6 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,10 +22,14 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[0].Severity, diag.Warning) require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") - require.Equal(t, diags[0].Location.File, filepath.Join("sync", "override", "databricks.yml")) - require.Equal(t, diags[0].Location.Line, 17) - require.Equal(t, diags[0].Location.Column, 11) - require.Equal(t, diags[0].Path.String(), "sync.exclude[0]") + + require.Len(t, diags[0].Paths, 1) + require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]") + + assert.Len(t, diags[0].Locations, 1) + require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) + require.Equal(t, diags[0].Locations[0].Line, 17) + require.Equal(t, diags[0].Locations[0].Column, 11) summaries := []string{ fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")), @@ -37,3 +42,22 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[2].Severity, diag.Warning) require.Contains(t, summaries, diags[2].Summary) } + +func TestSyncIncludeWithNegate(t *testing.T) { + b := loadTarget(t, "./sync/negate", "default") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 0) + require.NoError(t, diags.Error()) +} + +func TestSyncIncludeWithNegateNoMatches(t *testing.T) { + b := loadTarget(t, "./sync/negate", "dev") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 1) + require.NoError(t, diags.Error()) + + require.Equal(t, diags[0].Severity, diag.Warning) + require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files") +} diff --git a/bundle/tests/sync_test.go b/bundle/tests/sync_test.go index d08e889c3..15644b67e 100644 --- a/bundle/tests/sync_test.go +++ b/bundle/tests/sync_test.go @@ -12,14 +12,20 @@ func TestSyncOverride(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/override", "development") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override", "staging") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*"), filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override", "prod") + assert.Equal(t, filepath.FromSlash("sync/override"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("src/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) } @@ -28,14 +34,20 @@ func TestSyncOverrideNoRootSync(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/override_no_root", "development") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override_no_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("fixtures/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/override_no_root", "prod") + assert.Equal(t, filepath.FromSlash("sync/override_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{}, b.Config.Sync.Exclude) } @@ -44,10 +56,14 @@ func TestSyncNil(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/nil", "development") + assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.Nil(t, b.Config.Sync.Include) assert.Nil(t, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/nil", "staging") + assert.Equal(t, filepath.FromSlash("sync/nil"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) } @@ -56,10 +72,59 @@ func TestSyncNilRoot(t *testing.T) { var b *bundle.Bundle b = loadTarget(t, "./sync/nil_root", "development") + assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.Nil(t, b.Config.Sync.Include) assert.Nil(t, b.Config.Sync.Exclude) b = loadTarget(t, "./sync/nil_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/nil_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) assert.ElementsMatch(t, []string{filepath.FromSlash("tests/*")}, b.Config.Sync.Include) assert.ElementsMatch(t, []string{filepath.FromSlash("dist")}, b.Config.Sync.Exclude) } + +func TestSyncPaths(t *testing.T) { + var b *bundle.Bundle + + b = loadTarget(t, "./sync/paths", "development") + assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath) + assert.Equal(t, []string{"src", "development"}, b.Config.Sync.Paths) + + b = loadTarget(t, "./sync/paths", "staging") + assert.Equal(t, filepath.FromSlash("sync/paths"), b.SyncRootPath) + assert.Equal(t, []string{"src", "staging"}, b.Config.Sync.Paths) +} + +func TestSyncPathsNoRoot(t *testing.T) { + var b *bundle.Bundle + + b = loadTarget(t, "./sync/paths_no_root", "development") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"development"}, b.Config.Sync.Paths) + + b = loadTarget(t, "./sync/paths_no_root", "staging") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"staging"}, b.Config.Sync.Paths) + + // If not set at all, it defaults to "." + b = loadTarget(t, "./sync/paths_no_root", "undefined") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Equal(t, []string{"."}, b.Config.Sync.Paths) + + // If set to nil, it won't sync anything. + b = loadTarget(t, "./sync/paths_no_root", "nil") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Len(t, b.Config.Sync.Paths, 0) + + // If set to an empty sequence, it won't sync anything. + b = loadTarget(t, "./sync/paths_no_root", "empty") + assert.Equal(t, filepath.FromSlash("sync/paths_no_root"), b.SyncRootPath) + assert.Len(t, b.Config.Sync.Paths, 0) +} + +func TestSyncSharedCode(t *testing.T) { + b := loadTarget(t, "./sync/shared_code/bundle", "default") + assert.Equal(t, filepath.FromSlash("sync/shared_code"), b.SyncRootPath) + assert.ElementsMatch(t, []string{"common", "bundle"}, b.Config.Sync.Paths) +} diff --git a/bundle/tests/undefined_job/databricks.yml b/bundle/tests/undefined_job/databricks.yml deleted file mode 100644 index 12c19f946..000000000 --- a/bundle/tests/undefined_job/databricks.yml +++ /dev/null @@ -1,8 +0,0 @@ -bundle: - name: undefined-job - -resources: - jobs: - undefined: - test: - name: "Test Job" diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go deleted file mode 100644 index ed502c471..000000000 --- a/bundle/tests/undefined_job_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestUndefinedJobLoadsWithError(t *testing.T) { - _, diags := loadTargetWithDiags("./undefined_job", "default") - assert.ErrorContains(t, diags.Error(), "job undefined is not defined") -} diff --git a/bundle/tests/undefined_resources/databricks.yml b/bundle/tests/undefined_resources/databricks.yml new file mode 100644 index 000000000..ffc0e46da --- /dev/null +++ b/bundle/tests/undefined_resources/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: undefined-job + +resources: + jobs: + undefined-job: + test: + name: "Test Job" + + experiments: + undefined-experiment: + + pipelines: + undefined-pipeline: diff --git a/bundle/tests/undefined_resources_test.go b/bundle/tests/undefined_resources_test.go new file mode 100644 index 000000000..3dbacbc25 --- /dev/null +++ b/bundle/tests/undefined_resources_test.go @@ -0,0 +1,50 @@ +package config_tests + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestUndefinedResourcesLoadWithError(t *testing.T) { + b := load(t, "./undefined_resources") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) + + assert.Len(t, diags, 3) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "job undefined-job is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 6, + Column: 19, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.undefined-job")}, + }) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "experiment undefined-experiment is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 11, + Column: 26, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.experiments.undefined-experiment")}, + }) + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "pipeline undefined-pipeline is not defined", + Locations: []dyn.Location{{ + File: filepath.FromSlash("undefined_resources/databricks.yml"), + Line: 14, + Column: 24, + }}, + Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.undefined-pipeline")}, + }) +} diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml similarity index 53% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml index 1e9aa10b1..ebb1f9005 100644 --- a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml @@ -4,10 +4,10 @@ bundle: workspace: profile: test +include: + - ./*.yml + resources: jobs: foo: - name: job foo - pipelines: - foo: - name: pipeline foo + name: job foo 1 diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml similarity index 59% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml index c3dcb6e2f..deb81caa1 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml @@ -1,4 +1,8 @@ resources: + jobs: + foo: + name: job foo 2 + pipelines: foo: name: pipeline foo diff --git a/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml new file mode 100644 index 000000000..4e0a342b3 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + name: job foo 3 + + experiments: + foo: + name: experiment foo diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml similarity index 84% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml index ea4dec2e1..5bec67483 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml @@ -5,7 +5,7 @@ workspace: profile: test include: - - "*.yml" + - ./resources.yml resources: jobs: diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml similarity index 76% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml index a81602920..5bec67483 100644 --- a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml @@ -4,6 +4,9 @@ bundle: workspace: profile: test +include: + - ./resources.yml + resources: jobs: foo: diff --git a/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml new file mode 100644 index 000000000..83fb75735 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + name: job foo 2 diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml diff --git a/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml new file mode 100644 index 000000000..d286f1049 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test + +workspace: + profile: test + +resources: + jobs: + foo: + name: job foo + bar: + name: job bar + pipelines: + baz: + name: pipeline baz + experiments: + foo: + name: experiment foo diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go new file mode 100644 index 000000000..9cd7c201b --- /dev/null +++ b/bundle/tests/validate_test.go @@ -0,0 +1,139 @@ +package config_tests + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateUniqueResourceIdentifiers(t *testing.T) { + tcases := []struct { + name string + diagnostics diag.Diagnostics + }{ + { + name: "duplicate_resource_names_in_root_job_and_pipeline", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 13, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_root_job_and_experiment", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 18, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration_job_and_job", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_different_subconfiguations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_multiple_locations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 8, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./validate/"+tc.name) + require.NoError(t, err) + + // The UniqueResourceKeys mutator is run as part of the Load phase. + diags := bundle.Apply(ctx, b, phases.Load()) + assert.Equal(t, tc.diagnostics, diags) + }) + } +} diff --git a/cmd/account/budgets/budgets.go b/cmd/account/budgets/budgets.go index 82f7b9f01..6b47bb32c 100755 --- a/cmd/account/budgets/budgets.go +++ b/cmd/account/budgets/budgets.go @@ -19,16 +19,15 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "budgets", - Short: `These APIs manage budget configuration including notifications for exceeding a budget for a period.`, - Long: `These APIs manage budget configuration including notifications for exceeding a - budget for a period. They can also retrieve the status of each budget.`, + Short: `These APIs manage budget configurations for this account.`, + Long: `These APIs manage budget configurations for this account. Budgets enable you + to monitor usage across your account. You can set up budgets to either track + account-wide spending, or apply filters to track the spending of specific + teams, projects, or workspaces.`, GroupID: "billing", Annotations: map[string]string{ "package": "billing", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods @@ -52,23 +51,24 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *billing.WrappedBudget, + *billing.CreateBudgetConfigurationRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq billing.WrappedBudget + var createReq billing.CreateBudgetConfigurationRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Use = "create" - cmd.Short = `Create a new budget.` - cmd.Long = `Create a new budget. + cmd.Short = `Create new budget.` + cmd.Long = `Create new budget. - Creates a new budget in the specified account.` + Create a new budget configuration for an account. For full details, see + https://docs.databricks.com/en/admin/account-settings/budgets.html.` cmd.Annotations = make(map[string]string) @@ -111,13 +111,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *billing.DeleteBudgetRequest, + *billing.DeleteBudgetConfigurationRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq billing.DeleteBudgetRequest + var deleteReq billing.DeleteBudgetConfigurationRequest // TODO: short flags @@ -125,35 +125,24 @@ func newDelete() *cobra.Command { cmd.Short = `Delete budget.` cmd.Long = `Delete budget. - Deletes the budget specified by its UUID. + Deletes a budget configuration for an account. Both account and budget + configuration are specified by ID. This cannot be undone. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No BUDGET_ID argument specified. Loading names for Budgets drop-down." - names, err := a.Budgets.BudgetWithStatusNameToBudgetIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Budgets drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Budget ID") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have budget id") - } deleteReq.BudgetId = args[0] err = a.Budgets.Delete(ctx, deleteReq) @@ -181,50 +170,38 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *billing.GetBudgetRequest, + *billing.GetBudgetConfigurationRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq billing.GetBudgetRequest + var getReq billing.GetBudgetConfigurationRequest // TODO: short flags cmd.Use = "get BUDGET_ID" - cmd.Short = `Get budget and its status.` - cmd.Long = `Get budget and its status. + cmd.Short = `Get budget.` + cmd.Long = `Get budget. - Gets the budget specified by its UUID, including noncumulative status for each - day that the budget is configured to include. + Gets a budget configuration for an account. Both account and budget + configuration are specified by ID. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No BUDGET_ID argument specified. Loading names for Budgets drop-down." - names, err := a.Budgets.BudgetWithStatusNameToBudgetIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Budgets drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "Budget ID") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have budget id") - } getReq.BudgetId = args[0] response, err := a.Budgets.Get(ctx, getReq) @@ -252,25 +229,37 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *billing.ListBudgetConfigurationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq billing.ListBudgetConfigurationsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token received from a previous get all budget configurations call.`) + cmd.Use = "list" cmd.Short = `Get all budgets.` cmd.Long = `Get all budgets. - Gets all budgets associated with this account, including noncumulative status - for each day that the budget is configured to include.` + Gets all budgets associated with this account.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.Budgets.List(ctx) + + response := a.Budgets.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -280,7 +269,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -292,13 +281,13 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *billing.WrappedBudget, + *billing.UpdateBudgetConfigurationRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq billing.WrappedBudget + var updateReq billing.UpdateBudgetConfigurationRequest var updateJson flags.JsonFlag // TODO: short flags @@ -308,11 +297,11 @@ func newUpdate() *cobra.Command { cmd.Short = `Modify budget.` cmd.Long = `Modify budget. - Modifies a budget in this account. Budget properties are completely - overwritten. + Updates a budget configuration for an account. Both account and budget + configuration are specified by ID. Arguments: - BUDGET_ID: Budget ID` + BUDGET_ID: The Databricks budget configuration ID.` cmd.Annotations = make(map[string]string) @@ -336,11 +325,11 @@ func newUpdate() *cobra.Command { } updateReq.BudgetId = args[0] - err = a.Budgets.Update(ctx, updateReq) + response, err := a.Budgets.Update(ctx, updateReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. @@ -355,4 +344,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service Budgets +// end service budgets diff --git a/cmd/account/cmd.go b/cmd/account/cmd.go index 627d6d590..9b4bb8139 100644 --- a/cmd/account/cmd.go +++ b/cmd/account/cmd.go @@ -26,6 +26,7 @@ import ( account_settings "github.com/databricks/cli/cmd/account/settings" storage "github.com/databricks/cli/cmd/account/storage" account_storage_credentials "github.com/databricks/cli/cmd/account/storage-credentials" + usage_dashboards "github.com/databricks/cli/cmd/account/usage-dashboards" account_users "github.com/databricks/cli/cmd/account/users" vpc_endpoints "github.com/databricks/cli/cmd/account/vpc-endpoints" workspace_assignment "github.com/databricks/cli/cmd/account/workspace-assignment" @@ -40,7 +41,6 @@ func New() *cobra.Command { cmd.AddCommand(account_access_control.New()) cmd.AddCommand(billable_usage.New()) - cmd.AddCommand(budgets.New()) cmd.AddCommand(credentials.New()) cmd.AddCommand(custom_app_integration.New()) cmd.AddCommand(encryption_keys.New()) @@ -59,10 +59,12 @@ func New() *cobra.Command { cmd.AddCommand(account_settings.New()) cmd.AddCommand(storage.New()) cmd.AddCommand(account_storage_credentials.New()) + cmd.AddCommand(usage_dashboards.New()) cmd.AddCommand(account_users.New()) cmd.AddCommand(vpc_endpoints.New()) cmd.AddCommand(workspace_assignment.New()) cmd.AddCommand(workspaces.New()) + cmd.AddCommand(budgets.New()) // Register all groups with the parent command. groups := Groups() diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index ca9f69a35..5cdf422d7 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -3,8 +3,6 @@ package custom_app_integration import ( - "fmt" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" @@ -19,8 +17,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "custom-app-integration", - Short: `These APIs enable administrators to manage custom oauth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, - Long: `These APIs enable administrators to manage custom oauth app integrations, + Short: `These APIs enable administrators to manage custom OAuth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, + Long: `These APIs enable administrators to manage custom OAuth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.`, GroupID: "oauth2", @@ -62,7 +60,9 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&createReq.Confidential, "confidential", createReq.Confidential, `indicates if an oauth client-secret should be generated.`) + cmd.Flags().BoolVar(&createReq.Confidential, "confidential", createReq.Confidential, `This field indicates whether an OAuth client secret is required to authenticate this client.`) + cmd.Flags().StringVar(&createReq.Name, "name", createReq.Name, `Name of the custom OAuth app.`) + // TODO: array: redirect_urls // TODO: array: scopes // TODO: complex arg: token_access_policy @@ -72,11 +72,16 @@ func newCreate() *cobra.Command { Create Custom OAuth App Integration. - You can retrieve the custom oauth app integration via + You can retrieve the custom OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -87,8 +92,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := a.CustomAppIntegration.Create(ctx, createReq) @@ -131,10 +134,7 @@ func newDelete() *cobra.Command { cmd.Long = `Delete Custom OAuth App Integration. Delete an existing Custom OAuth App Integration. You can retrieve the custom - oauth app integration via :method:CustomAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -189,10 +189,7 @@ func newGet() *cobra.Command { cmd.Short = `Get OAuth Custom App Integration.` cmd.Long = `Get OAuth Custom App Integration. - Gets the Custom OAuth App Integration for the given integration id. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + Gets the Custom OAuth App Integration for the given integration id.` cmd.Annotations = make(map[string]string) @@ -233,25 +230,40 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *oauth2.ListCustomAppIntegrationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq oauth2.ListCustomAppIntegrationsRequest + + // TODO: short flags + + cmd.Flags().BoolVar(&listReq.IncludeCreatorUsername, "include-creator-username", listReq.IncludeCreatorUsername, ``) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" cmd.Short = `Get custom oauth app integrations.` cmd.Long = `Get custom oauth app integrations. - Get the list of custom oauth app integrations for the specified Databricks + Get the list of custom OAuth app integrations for the specified Databricks account` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.CustomAppIntegration.List(ctx) + + response := a.CustomAppIntegration.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -261,7 +273,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -293,10 +305,7 @@ func newUpdate() *cobra.Command { cmd.Long = `Updates Custom OAuth App Integration. Updates an existing custom OAuth App Integration. You can retrieve the custom - oauth app integration via :method:CustomAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + OAuth app integration via :method:CustomAppIntegration/get.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/o-auth-published-apps/o-auth-published-apps.go b/cmd/account/o-auth-published-apps/o-auth-published-apps.go index 6573b0529..f1af17d2e 100755 --- a/cmd/account/o-auth-published-apps/o-auth-published-apps.go +++ b/cmd/account/o-auth-published-apps/o-auth-published-apps.go @@ -54,7 +54,7 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().Int64Var(&listReq.PageSize, "page-size", listReq.PageSize, `The max number of OAuth published apps to return.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The max number of OAuth published apps to return in one page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) cmd.Use = "list" diff --git a/cmd/account/published-app-integration/published-app-integration.go b/cmd/account/published-app-integration/published-app-integration.go index 32fed5cd0..5143d53cc 100755 --- a/cmd/account/published-app-integration/published-app-integration.go +++ b/cmd/account/published-app-integration/published-app-integration.go @@ -17,8 +17,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "published-app-integration", - Short: `These APIs enable administrators to manage published oauth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, - Long: `These APIs enable administrators to manage published oauth app integrations, + Short: `These APIs enable administrators to manage published OAuth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, + Long: `These APIs enable administrators to manage published OAuth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.`, GroupID: "oauth2", @@ -60,7 +60,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&createReq.AppId, "app-id", createReq.AppId, `app_id of the oauth published app integration.`) + cmd.Flags().StringVar(&createReq.AppId, "app-id", createReq.AppId, `App id of the OAuth published app integration.`) // TODO: complex arg: token_access_policy cmd.Use = "create" @@ -69,7 +69,7 @@ func newCreate() *cobra.Command { Create Published OAuth App Integration. - You can retrieve the published oauth app integration via + You can retrieve the published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -131,10 +131,7 @@ func newDelete() *cobra.Command { cmd.Long = `Delete Published OAuth App Integration. Delete an existing Published OAuth App Integration. You can retrieve the - published oauth app integration via :method:PublishedAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) @@ -189,10 +186,7 @@ func newGet() *cobra.Command { cmd.Short = `Get OAuth Published App Integration.` cmd.Long = `Get OAuth Published App Integration. - Gets the Published OAuth App Integration for the given integration id. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + Gets the Published OAuth App Integration for the given integration id.` cmd.Annotations = make(map[string]string) @@ -233,25 +227,39 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *oauth2.ListPublishedAppIntegrationsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq oauth2.ListPublishedAppIntegrationsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" cmd.Short = `Get published oauth app integrations.` cmd.Long = `Get published oauth app integrations. - Get the list of published oauth app integrations for the specified Databricks + Get the list of published OAuth app integrations for the specified Databricks account` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustAccountClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() a := root.AccountClient(ctx) - response := a.PublishedAppIntegration.List(ctx) + + response := a.PublishedAppIntegration.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -261,7 +269,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -292,10 +300,7 @@ func newUpdate() *cobra.Command { cmd.Long = `Updates Published OAuth App Integration. Updates an existing published OAuth App Integration. You can retrieve the - published oauth app integration via :method:PublishedAppIntegration/get. - - Arguments: - INTEGRATION_ID: The oauth app integration ID.` + published OAuth app integration via :method:PublishedAppIntegration/get.` cmd.Annotations = make(map[string]string) diff --git a/cmd/account/usage-dashboards/usage-dashboards.go b/cmd/account/usage-dashboards/usage-dashboards.go new file mode 100755 index 000000000..8a1c32476 --- /dev/null +++ b/cmd/account/usage-dashboards/usage-dashboards.go @@ -0,0 +1,164 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package usage_dashboards + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/billing" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "usage-dashboards", + Short: `These APIs manage usage dashboards for this account.`, + Long: `These APIs manage usage dashboards for this account. Usage dashboards enable + you to gain insights into your usage with pre-built dashboards: visualize + breakdowns, analyze tag attributions, and identify cost drivers.`, + GroupID: "billing", + Annotations: map[string]string{ + "package": "billing", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newGet()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *billing.CreateBillingUsageDashboardRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq billing.CreateBillingUsageDashboardRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().Var(&createReq.DashboardType, "dashboard-type", `Workspace level usage dashboard shows usage data for the specified workspace ID. Supported values: [USAGE_DASHBOARD_TYPE_GLOBAL, USAGE_DASHBOARD_TYPE_WORKSPACE]`) + cmd.Flags().Int64Var(&createReq.WorkspaceId, "workspace-id", createReq.WorkspaceId, `The workspace ID of the workspace in which the usage dashboard is created.`) + + cmd.Use = "create" + cmd.Short = `Create new usage dashboard.` + cmd.Long = `Create new usage dashboard. + + Create a usage dashboard specified by workspaceId, accountId, and dashboard + type.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } + + response, err := a.UsageDashboards.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *billing.GetBillingUsageDashboardRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq billing.GetBillingUsageDashboardRequest + + // TODO: short flags + + cmd.Flags().Var(&getReq.DashboardType, "dashboard-type", `Workspace level usage dashboard shows usage data for the specified workspace ID. Supported values: [USAGE_DASHBOARD_TYPE_GLOBAL, USAGE_DASHBOARD_TYPE_WORKSPACE]`) + cmd.Flags().Int64Var(&getReq.WorkspaceId, "workspace-id", getReq.WorkspaceId, `The workspace ID of the workspace in which the usage dashboard is created.`) + + cmd.Use = "get" + cmd.Short = `Get usage dashboard.` + cmd.Long = `Get usage dashboard. + + Get a usage dashboard specified by workspaceId, accountId, and dashboard type.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.UsageDashboards.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// end service UsageDashboards diff --git a/cmd/account/workspace-assignment/workspace-assignment.go b/cmd/account/workspace-assignment/workspace-assignment.go index b965d31ad..58468d09f 100755 --- a/cmd/account/workspace-assignment/workspace-assignment.go +++ b/cmd/account/workspace-assignment/workspace-assignment.go @@ -66,7 +66,7 @@ func newDelete() *cobra.Command { for the specified principal. Arguments: - WORKSPACE_ID: The workspace ID. + WORKSPACE_ID: The workspace ID for the account. PRINCIPAL_ID: The ID of the user, service principal, or group.` cmd.Annotations = make(map[string]string) @@ -247,6 +247,8 @@ func newUpdate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: array: permissions + cmd.Use = "update WORKSPACE_ID PRINCIPAL_ID" cmd.Short = `Create or update permissions assignment.` cmd.Long = `Create or update permissions assignment. @@ -255,7 +257,7 @@ func newUpdate() *cobra.Command { workspace for the specified principal. Arguments: - WORKSPACE_ID: The workspace ID. + WORKSPACE_ID: The workspace ID for the account. PRINCIPAL_ID: The ID of the user, service principal, or group.` cmd.Annotations = make(map[string]string) @@ -275,8 +277,6 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) if err != nil { diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index 79e1063b1..ceceae25c 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -2,6 +2,7 @@ package auth import ( "context" + "fmt" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" @@ -34,25 +35,23 @@ GCP: https://docs.gcp.databricks.com/dev-tools/auth/index.html`, } func promptForHost(ctx context.Context) (string, error) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Host (e.g. https://.cloud.databricks.com)" - // Validate? - host, err := prompt.Run() - if err != nil { - return "", err + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a host using --host") } - return host, nil + + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks host (e.g. https://.cloud.databricks.com)" + return prompt.Run() } func promptForAccountID(ctx context.Context) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify an account ID using --account-id") + } + prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Account ID" + prompt.Label = "Databricks account ID" prompt.Default = "" prompt.AllowEdit = true - // Validate? - accountId, err := prompt.Run() - if err != nil { - return "", err - } - return accountId, nil + return prompt.Run() } diff --git a/cmd/auth/login.go b/cmd/auth/login.go index 11cba8e5f..f87a2a027 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -17,18 +17,16 @@ import ( "github.com/spf13/cobra" ) -func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, args []string, argIndex int) error { - if len(args) > argIndex { - persistentAuth.Host = args[argIndex] - return nil +func promptForProfile(ctx context.Context, defaultValue string) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a profile using --profile") } - host, err := promptForHost(ctx) - if err != nil { - return err - } - persistentAuth.Host = host - return nil + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks profile name" + prompt.Default = defaultValue + prompt.AllowEdit = true + return prompt.Run() } const minimalDbConnectVersion = "13.1" @@ -93,23 +91,18 @@ depends on the existing profiles you have set in your configuration file cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + profileName := cmd.Flag("profile").Value.String() - var profileName string - profileFlag := cmd.Flag("profile") - if profileFlag != nil && profileFlag.Value.String() != "" { - profileName = profileFlag.Value.String() - } else if cmdio.IsInTTY(ctx) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Profile Name" - prompt.Default = persistentAuth.ProfileName() - prompt.AllowEdit = true - profile, err := prompt.Run() + // If the user has not specified a profile name, prompt for one. + if profileName == "" { + var err error + profileName, err = promptForProfile(ctx, persistentAuth.ProfileName()) if err != nil { return err } - profileName = profile } + // Set the host and account-id based on the provided arguments and flags. err := setHostAndAccountId(ctx, profileName, persistentAuth, args) if err != nil { return err @@ -167,7 +160,23 @@ depends on the existing profiles you have set in your configuration file return cmd } +// Sets the host in the persistentAuth object based on the provided arguments and flags. +// Follows the following precedence: +// 1. [HOST] (first positional argument) or --host flag. Error if both are specified. +// 2. Profile host, if available. +// 3. Prompt the user for the host. +// +// Set the account in the persistentAuth object based on the flags. +// Follows the following precedence: +// 1. --account-id flag. +// 2. account-id from the specified profile, if available. +// 3. Prompt the user for the account-id. func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error { + // If both [HOST] and --host are provided, return an error. + if len(args) > 0 && persistentAuth.Host != "" { + return fmt.Errorf("please only provide a host as an argument or a flag, not both") + } + profiler := profile.GetProfiler(ctx) // If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile. profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName)) @@ -177,17 +186,32 @@ func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth } if persistentAuth.Host == "" { - if len(profiles) > 0 && profiles[0].Host != "" { + if len(args) > 0 { + // If [HOST] is provided, set the host to the provided positional argument. + persistentAuth.Host = args[0] + } else if len(profiles) > 0 && profiles[0].Host != "" { + // If neither [HOST] nor --host are provided, and the profile has a host, use it. persistentAuth.Host = profiles[0].Host } else { - configureHost(ctx, persistentAuth, args, 0) + // If neither [HOST] nor --host are provided, and the profile does not have a host, + // then prompt the user for a host. + hostName, err := promptForHost(ctx) + if err != nil { + return err + } + persistentAuth.Host = hostName } } + + // If the account-id was not provided as a cmd line flag, try to read it from + // the specified profile. isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient() if isAccountClient && persistentAuth.AccountID == "" { if len(profiles) > 0 && profiles[0].AccountID != "" { persistentAuth.AccountID = profiles[0].AccountID } else { + // Prompt user for the account-id if it we could not get it from a + // profile. accountId, err := promptForAccountID(ctx) if err != nil { return err diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go index ce3ca5ae5..d0fa5a16b 100644 --- a/cmd/auth/login_test.go +++ b/cmd/auth/login_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { @@ -15,3 +17,69 @@ func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{}) assert.NoError(t, err) } + +func TestSetHost(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test error when both flag and argument are provided + persistentAuth.Host = "val from --host" + err := setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.EqualError(t, err, "please only provide a host as an argument or a flag, not both") + + // Test setting host from flag + persistentAuth.Host = "val from --host" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "val from --host", persistentAuth.Host) + + // Test setting host from argument + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.NoError(t, err) + assert.Equal(t, "val from [HOST]", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host1.com", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-2", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host2.com", persistentAuth.Host) + + // Test host is not set. Should prompt. + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify a host using --host") +} + +func TestSetAccountId(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test setting account-id from flag + persistentAuth.AccountID = "val from --account-id" + err := setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "val from --account-id", persistentAuth.AccountID) + + // Test setting account_id from profile + persistentAuth.AccountID = "" + err = setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + require.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "id-from-profile", persistentAuth.AccountID) + + // Neither flag nor profile account-id is set, should prompt + persistentAuth.AccountID = "" + persistentAuth.Host = "https://accounts.cloud.databricks.com" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify an account ID using --account-id") +} diff --git a/cmd/auth/testdata/.databrickscfg b/cmd/auth/testdata/.databrickscfg new file mode 100644 index 000000000..06e55224a --- /dev/null +++ b/cmd/auth/testdata/.databrickscfg @@ -0,0 +1,9 @@ +[profile-1] +host = https://www.host1.com + +[profile-2] +host = https://www.host2.com + +[account-profile] +host = https://accounts.cloud.databricks.com +account_id = id-from-profile diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 919b15a72..1166875ab 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -2,9 +2,11 @@ package bundle import ( "context" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" @@ -22,40 +24,52 @@ func newDeployCommand() *cobra.Command { var forceLock bool var failOnActiveRuns bool var computeID string + var autoApprove bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") + cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() b, diags := utils.ConfigureBundleWithVariables(cmd) - if err := diags.Error(); err != nil { - return diags.Error() + + if !diags.HasError() { + bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { + b.Config.Bundle.Force = force + b.Config.Bundle.Deployment.Lock.Force = forceLock + b.AutoApprove = autoApprove + + if cmd.Flag("compute-id").Changed { + b.Config.Bundle.ComputeID = computeID + } + if cmd.Flag("fail-on-active-runs").Changed { + b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns + } + + return nil + }) + + diags = diags.Extend( + bundle.Apply(ctx, b, bundle.Seq( + phases.Initialize(), + phases.Build(), + phases.Deploy(), + )), + ) } - bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { - b.Config.Bundle.Force = force - b.Config.Bundle.Deployment.Lock.Force = forceLock - if cmd.Flag("compute-id").Changed { - b.Config.Bundle.ComputeID = computeID - } - - if cmd.Flag("fail-on-active-runs").Changed { - b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns - } - - return nil - }) - - diags = bundle.Apply(ctx, b, bundle.Seq( - phases.Initialize(), - phases.Build(), - phases.Deploy(), - )) - if err := diags.Error(); err != nil { - return err + renderOpts := render.RenderOptions{RenderSummaryTable: false} + err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) + if err != nil { + return fmt.Errorf("failed to render output: %w", err) } + + if diags.HasError() { + return root.ErrAlreadyPrinted + } + return nil } diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index c8c59c149..7f2c0efc5 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -49,6 +49,12 @@ var nativeTemplates = []nativeTemplate{ description: "The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)", aliases: []string{"mlops-stack"}, }, + { + name: "default-pydabs", + gitUrl: "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git", + hidden: true, + description: "The default PyDABs template", + }, { name: customTemplate, description: "Bring your own template", @@ -142,7 +148,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf var templateDir string var tag string var branch string - cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") + cmd.Flags().StringVar(&configFile, "config-file", "", "JSON file containing key value pairs of input parameters required for template initialization.") cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory path within a Git repository containing the template.") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.") cmd.Flags().StringVar(&branch, "tag", "", "Git tag to use for template initialization") diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index b0d6b3dd5..813aebbae 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -11,54 +11,6 @@ import ( "github.com/spf13/cobra" ) -func overrideVariables(s *jsonschema.Schema) error { - // Override schema for default values to allow for multiple primitive types. - // These are normalized to strings when converted to the typed representation. - err := s.SetByPath("variables.*.default", jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: jsonschema.StringType, - }, - { - Type: jsonschema.BooleanType, - }, - { - Type: jsonschema.NumberType, - }, - { - Type: jsonschema.IntegerType, - }, - }, - }) - if err != nil { - return err - } - - // Override schema for variables in targets to allow just specifying the value - // along side overriding the variable definition if needed. - ns, err := s.GetByPath("variables.*") - if err != nil { - return err - } - return s.SetByPath("targets.*.variables.*", jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: jsonschema.StringType, - }, - { - Type: jsonschema.BooleanType, - }, - { - Type: jsonschema.NumberType, - }, - { - Type: jsonschema.IntegerType, - }, - &ns, - }, - }) -} - func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schema", @@ -79,9 +31,9 @@ func newSchemaCommand() *cobra.Command { return err } - // Override schema for variables to take into account normalization of default - // variable values and variable overrides in a target. - err = overrideVariables(schema) + // Target variable value overrides can be primitives, maps or sequences. + // Set an empty schema for them. + err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) if err != nil { return err } diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 59a977047..496d5d2b5 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -53,7 +53,8 @@ func newValidateCommand() *cobra.Command { switch root.OutputType(cmd) { case flags.OutputText: - err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags) + renderOpts := render.RenderOptions{RenderSummaryTable: true} + err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) if err != nil { return fmt.Errorf("failed to render output: %w", err) } diff --git a/cmd/cmd.go b/cmd/cmd.go index 5d835409f..5b53a4ae5 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/cmd/sync" "github.com/databricks/cli/cmd/version" "github.com/databricks/cli/cmd/workspace" + "github.com/databricks/cli/cmd/workspace/apps" "github.com/spf13/cobra" ) @@ -67,6 +68,7 @@ func New(ctx context.Context) *cobra.Command { // Add other subcommands. cli.AddCommand(api.New()) + cli.AddCommand(apps.New()) cli.AddCommand(auth.New()) cli.AddCommand(bundle.New()) cli.AddCommand(configure.New()) diff --git a/cmd/fs/cat.go b/cmd/fs/cat.go index 7a6f42cba..28df80d70 100644 --- a/cmd/fs/cat.go +++ b/cmd/fs/cat.go @@ -30,5 +30,8 @@ func newCatCommand() *cobra.Command { return cmdio.Render(ctx, r) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 52feb8905..6fb3e5e6f 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -200,5 +200,10 @@ func newCpCommand() *cobra.Command { return c.cpFileToFile(sourcePath, targetPath) } + v := newValidArgs() + // The copy command has two paths that can be completed (SOURCE_PATH & TARGET_PATH) + v.pathArgCount = 2 + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 43d65b5dd..bda3239cf 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -8,6 +8,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/filer/completer" + "github.com/spf13/cobra" ) func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { @@ -46,6 +48,58 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return f, path, err } +const dbfsPrefix string = "dbfs:" + func isDbfsPath(path string) bool { - return strings.HasPrefix(path, "dbfs:/") + return strings.HasPrefix(path, dbfsPrefix) +} + +type validArgs struct { + mustWorkspaceClientFunc func(cmd *cobra.Command, args []string) error + filerForPathFunc func(ctx context.Context, fullPath string) (filer.Filer, string, error) + pathArgCount int + onlyDirs bool +} + +func newValidArgs() *validArgs { + return &validArgs{ + mustWorkspaceClientFunc: root.MustWorkspaceClient, + filerForPathFunc: filerForPath, + pathArgCount: 1, + onlyDirs: false, + } +} + +func (v *validArgs) Validate(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + cmd.SetContext(root.SkipPrompt(cmd.Context())) + + if len(args) >= v.pathArgCount { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + err := v.mustWorkspaceClientFunc(cmd, args) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + filer, toCompletePath, err := v.filerForPathFunc(cmd.Context(), toComplete) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + completer := completer.New(cmd.Context(), filer, v.onlyDirs) + + // Dbfs should have a prefix and always use the "/" separator + isDbfsPath := isDbfsPath(toComplete) + if isDbfsPath { + completer.SetPrefix(dbfsPrefix) + completer.SetIsLocalPath(false) + } + + completions, directive, err := completer.CompletePath(toCompletePath) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + return completions, directive } diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index d86bd46e1..10b4aa160 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -3,9 +3,13 @@ package fs import ( "context" "runtime" + "strings" "testing" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -60,3 +64,88 @@ func TestFilerForWindowsLocalPaths(t *testing.T) { testWindowsFilerForPath(t, ctx, `d:\abc`) testWindowsFilerForPath(t, ctx, `f:\abc\ef`) } + +func mockMustWorkspaceClientFunc(cmd *cobra.Command, args []string) error { + return nil +} + +func setupCommand(t *testing.T) (*cobra.Command, *mocks.MockWorkspaceClient) { + m := mocks.NewMockWorkspaceClient(t) + ctx := context.Background() + ctx = root.SetWorkspaceClient(ctx, m.WorkspaceClient) + + cmd := &cobra.Command{} + cmd.SetContext(ctx) + + return cmd, m +} + +func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceClient) { + cmd, m := setupCommand(t) + + fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) { + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + return fakeFiler, strings.TrimPrefix(fullPath, "dbfs:/"), nil + } + + v := newValidArgs() + v.filerForPathFunc = fakeFilerForPath + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + return v, cmd, m +} + +func TestGetValidArgsFunctionDbfsCompletion(t *testing.T) { + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/", "dbfs:/dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletion(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletionWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dir\\fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionCompletionOnlyDirs(t *testing.T) { + v, cmd, _ := setupTest(t) + v.onlyDirs = true + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionNotCompletedArgument(t *testing.T) { + cmd, _ := setupCommand(t) + + v := newValidArgs() + v.pathArgCount = 0 + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + completions, directive := v.Validate(cmd, []string{}, "dbfs:/") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) +} diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index cec9b98ba..d7eac513a 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -89,5 +89,9 @@ func newLsCommand() *cobra.Command { `)) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/mkdir.go b/cmd/fs/mkdir.go index 074a7543d..5e9ac7842 100644 --- a/cmd/fs/mkdir.go +++ b/cmd/fs/mkdir.go @@ -28,5 +28,9 @@ func newMkdirCommand() *cobra.Command { return f.Mkdir(ctx, path) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/rm.go b/cmd/fs/rm.go index 5f2904e71..a133a8309 100644 --- a/cmd/fs/rm.go +++ b/cmd/fs/rm.go @@ -32,5 +32,8 @@ func newRmCommand() *cobra.Command { return f.Delete(ctx, path) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 92dfe9e7c..041415964 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -132,14 +132,14 @@ func (i *installer) Upgrade(ctx context.Context) error { if err != nil { return fmt.Errorf("record version: %w", err) } - err = i.runInstallHook(ctx) - if err != nil { - return fmt.Errorf("installer: %w", err) - } err = i.installPythonDependencies(ctx, ".") if err != nil { return fmt.Errorf("python dependencies: %w", err) } + err = i.runInstallHook(ctx) + if err != nil { + return fmt.Errorf("installer: %w", err) + } return nil } @@ -272,8 +272,10 @@ func (i *installer) installPythonDependencies(ctx context.Context, spec string) // - python3 -m ensurepip --default-pip // - curl -o https://bootstrap.pypa.io/get-pip.py | python3 var buf bytes.Buffer + // Ensure latest version(s) is installed with the `--upgrade` and `--upgrade-strategy eager` flags + // https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-U _, err := process.Background(ctx, - []string{i.virtualEnvPython(ctx), "-m", "pip", "install", spec}, + []string{i.virtualEnvPython(ctx), "-m", "pip", "install", "--upgrade", "--upgrade-strategy", "eager", spec}, process.WithCombinedOutput(&buf), process.WithDir(libDir)) if err != nil { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index 0e049b4c0..1e45fafe6 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -182,7 +182,7 @@ func TestInstallerWorksForReleases(t *testing.T) { w.Write(raw) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) @@ -199,7 +199,7 @@ func TestInstallerWorksForReleases(t *testing.T) { stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5") // on Unix, we call `python3`, but on Windows it is `python.exe` stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]") - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") // simulate the case of GitHub Actions @@ -249,8 +249,9 @@ func TestInstallerWorksForDevelopment(t *testing.T) { Path: filepath.Dir(t.TempDir()), }) }() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/2.0/clusters/list" { + if r.URL.Path == "/api/2.1/clusters/list" { respondWithJSON(t, w, compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -278,7 +279,7 @@ func TestInstallerWorksForDevelopment(t *testing.T) { }) return } - if r.URL.Path == "/api/2.0/clusters/spark-versions" { + if r.URL.Path == "/api/2.1/clusters/spark-versions" { respondWithJSON(t, w, compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { @@ -289,7 +290,7 @@ func TestInstallerWorksForDevelopment(t *testing.T) { }) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) @@ -387,7 +388,7 @@ func TestUpgraderWorksForReleases(t *testing.T) { w.Write(raw) return } - if r.URL.Path == "/api/2.0/clusters/get" { + if r.URL.Path == "/api/2.1/clusters/get" { respondWithJSON(t, w, &compute.ClusterDetails{ State: compute.StateRunning, }) @@ -406,7 +407,7 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Install stubs for the python calls we need to ensure were run in the // upgrade process. ctx, stub := process.WithStub(ctx) - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") py, _ := python.DetectExecutable(ctx) @@ -430,13 +431,13 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Check if the stub was called with the 'python -m pip install' command pi := false for _, call := range stub.Commands() { - if strings.HasSuffix(call, "-m pip install .") { + if strings.HasSuffix(call, "-m pip install --upgrade --upgrade-strategy eager .") { pi = true break } } if !pi { - t.Logf(`Expected stub command 'python[\S]+ -m pip install .' not found`) + t.Logf(`Expected stub command 'python[\S]+ -m pip install --upgrade --upgrade-strategy eager .' not found`) t.FailNow() } } diff --git a/cmd/root/auth_test.go b/cmd/root/auth_test.go index 486f587ef..9ba2a8fa9 100644 --- a/cmd/root/auth_test.go +++ b/cmd/root/auth_test.go @@ -111,6 +111,10 @@ func TestAccountClientOrPrompt(t *testing.T) { expectPrompts(t, accountPromptFn, &config.Config{ Host: "https://accounts.azuredatabricks.net/", AccountID: "1234", + + // Force SDK to not try and lookup the tenant ID from the host. + // The host above is invalid and will not be reachable. + AzureTenantID: "nonempty", }) }) @@ -165,6 +169,10 @@ func TestWorkspaceClientOrPrompt(t *testing.T) { t.Run("Prompt if no credential provider can be configured", func(t *testing.T) { expectPrompts(t, workspacePromptFn, &config.Config{ Host: "https://adb-1111.11.azuredatabricks.net/", + + // Force SDK to not try and lookup the tenant ID from the host. + // The host above is invalid and will not be reachable. + AzureTenantID: "nonempty", }) }) diff --git a/cmd/root/root.go b/cmd/root/root.go index 61baa4da0..eda873d12 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -92,9 +92,8 @@ func flagErrorFunc(c *cobra.Command, err error) error { // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute(cmd *cobra.Command) { +func Execute(ctx context.Context, cmd *cobra.Command) error { // TODO: deferred panic recovery - ctx := context.Background() // Run the command cmd, err := cmd.ExecuteContextC(ctx) @@ -118,7 +117,5 @@ func Execute(cmd *cobra.Command) { } } - if err != nil { - os.Exit(1) - } + return err } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index bab451593..23a4c018f 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -47,7 +47,11 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn } opts := sync.SyncOptions{ - LocalPath: vfs.MustNew(args[0]), + LocalRoot: vfs.MustNew(args[0]), + Paths: []string{"."}, + Include: nil, + Exclude: nil, + RemotePath: args[1], Full: f.full, PollInterval: f.interval, diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 564aeae56..bd03eec91 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -17,8 +17,10 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tempDir, - BundleRoot: vfs.MustNew(tempDir), + RootPath: tempDir, + BundleRoot: vfs.MustNew(tempDir), + SyncRootPath: tempDir, + SyncRoot: vfs.MustNew(tempDir), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -33,7 +35,7 @@ func TestSyncOptionsFromBundle(t *testing.T) { f := syncFlags{} opts, err := f.syncOptionsFromBundle(New(), []string{}, b) require.NoError(t, err) - assert.Equal(t, tempDir, opts.LocalPath.Native()) + assert.Equal(t, tempDir, opts.LocalRoot.Native()) assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath) assert.Equal(t, filepath.Join(tempDir, ".databricks", "bundle", "default"), opts.SnapshotBasePath) assert.NotNil(t, opts.WorkspaceClient) @@ -59,6 +61,6 @@ func TestSyncOptionsFromArgs(t *testing.T) { cmd.SetContext(root.SetWorkspaceClient(context.Background(), nil)) opts, err := f.syncOptionsFromArgs(cmd, []string{local, remote}) require.NoError(t, err) - assert.Equal(t, local, opts.LocalPath.Native()) + assert.Equal(t, local, opts.LocalRoot.Native()) assert.Equal(t, remote, opts.RemotePath) } diff --git a/cmd/workspace/alerts-legacy/alerts-legacy.go b/cmd/workspace/alerts-legacy/alerts-legacy.go new file mode 100755 index 000000000..1046b1124 --- /dev/null +++ b/cmd/workspace/alerts-legacy/alerts-legacy.go @@ -0,0 +1,388 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package alerts_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "alerts-legacy", + Short: `The alerts API can be used to perform CRUD operations on alerts.`, + Long: `The alerts API can be used to perform CRUD operations on alerts. An alert is a + Databricks SQL object that periodically runs a query, evaluates a condition of + its result, and notifies one or more users and/or notification destinations if + the condition was met. Alerts can be scheduled using the sql_task type of + the Jobs API, e.g. :method:jobs/create. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.CreateAlert, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.CreateAlert + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createReq.Parent, "parent", createReq.Parent, `The identifier of the workspace folder containing the object.`) + cmd.Flags().IntVar(&createReq.Rearm, "rearm", createReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + + cmd.Use = "create" + cmd.Short = `Create an alert.` + cmd.Long = `Create an alert. + + Creates an alert. An alert is a Databricks SQL object that periodically runs a + query, evaluates a condition of its result, and notifies users or notification + destinations if the condition was met. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.AlertsLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteAlertsLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteAlertsLegacyRequest + + // TODO: short flags + + cmd.Use = "delete ALERT_ID" + cmd.Short = `Delete an alert.` + cmd.Long = `Delete an alert. + + Deletes an alert. Deleted alerts are no longer accessible and cannot be + restored. **Note**: Unlike queries and dashboards, alerts cannot be moved to + the trash. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts Legacy drop-down." + names, err := w.AlertsLegacy.LegacyAlertNameToIdMap(ctx) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Alerts Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + deleteReq.AlertId = args[0] + + err = w.AlertsLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *sql.GetAlertsLegacyRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq sql.GetAlertsLegacyRequest + + // TODO: short flags + + cmd.Use = "get ALERT_ID" + cmd.Short = `Get an alert.` + cmd.Long = `Get an alert. + + Gets an alert. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/get instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts Legacy drop-down." + names, err := w.AlertsLegacy.LegacyAlertNameToIdMap(ctx) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Alerts Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + getReq.AlertId = args[0] + + response, err := w.AlertsLegacy.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + cmd.Use = "list" + cmd.Short = `Get alerts.` + cmd.Long = `Get alerts. + + Gets a list of alerts. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/list instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + response, err := w.AlertsLegacy.List(ctx) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.EditAlert, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.EditAlert + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().IntVar(&updateReq.Rearm, "rearm", updateReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + + cmd.Use = "update ALERT_ID" + cmd.Short = `Update an alert.` + cmd.Long = `Update an alert. + + Updates an alert. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:alerts/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + updateReq.AlertId = args[0] + + err = w.AlertsLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service AlertsLegacy diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go index 61c1e0eab..cfaa3f55f 100755 --- a/cmd/workspace/alerts/alerts.go +++ b/cmd/workspace/alerts/alerts.go @@ -24,12 +24,7 @@ func New() *cobra.Command { Databricks SQL object that periodically runs a query, evaluates a condition of its result, and notifies one or more users and/or notification destinations if the condition was met. Alerts can be scheduled using the sql_task type of - the Jobs API, e.g. :method:jobs/create. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + the Jobs API, e.g. :method:jobs/create.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -57,36 +52,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.CreateAlert, + *sql.CreateAlertRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.CreateAlert + var createReq sql.CreateAlertRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&createReq.Parent, "parent", createReq.Parent, `The identifier of the workspace folder containing the object.`) - cmd.Flags().IntVar(&createReq.Rearm, "rearm", createReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + // TODO: complex arg: alert cmd.Use = "create" cmd.Short = `Create an alert.` cmd.Long = `Create an alert. - Creates an alert. An alert is a Databricks SQL object that periodically runs a - query, evaluates a condition of its result, and notifies users or notification - destinations if the condition was met. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Creates an alert.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -97,8 +89,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.Alerts.Create(ctx, createReq) @@ -126,28 +116,23 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteAlertRequest, + *sql.TrashAlertRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteAlertRequest + var deleteReq sql.TrashAlertRequest // TODO: short flags - cmd.Use = "delete ALERT_ID" + cmd.Use = "delete ID" cmd.Short = `Delete an alert.` cmd.Long = `Delete an alert. - Deletes an alert. Deleted alerts are no longer accessible and cannot be - restored. **Note**: Unlike queries and dashboards, alerts cannot be moved to - the trash. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Moves an alert to the trash. Trashed alerts immediately disappear from + searches and list views, and can no longer trigger. You can restore a trashed + alert through the UI. A trashed alert is permanently deleted after 30 days.` cmd.Annotations = make(map[string]string) @@ -158,8 +143,8 @@ func newDelete() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts drop-down." - names, err := w.Alerts.AlertNameToIdMap(ctx) + promptSpinner <- "No ID argument specified. Loading names for Alerts drop-down." + names, err := w.Alerts.ListAlertsResponseAlertDisplayNameToIdMap(ctx, sql.ListAlertsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Alerts drop-down. Please manually specify required arguments. Original error: %w", err) @@ -173,7 +158,7 @@ func newDelete() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - deleteReq.AlertId = args[0] + deleteReq.Id = args[0] err = w.Alerts.Delete(ctx, deleteReq) if err != nil { @@ -210,16 +195,11 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get ALERT_ID" + cmd.Use = "get ID" cmd.Short = `Get an alert.` cmd.Long = `Get an alert. - Gets an alert. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets an alert.` cmd.Annotations = make(map[string]string) @@ -230,8 +210,8 @@ func newGet() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No ALERT_ID argument specified. Loading names for Alerts drop-down." - names, err := w.Alerts.AlertNameToIdMap(ctx) + promptSpinner <- "No ID argument specified. Loading names for Alerts drop-down." + names, err := w.Alerts.ListAlertsResponseAlertDisplayNameToIdMap(ctx, sql.ListAlertsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Alerts drop-down. Please manually specify required arguments. Original error: %w", err) @@ -245,7 +225,7 @@ func newGet() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - getReq.AlertId = args[0] + getReq.Id = args[0] response, err := w.Alerts.Get(ctx, getReq) if err != nil { @@ -272,33 +252,41 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *sql.ListAlertsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq sql.ListAlertsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + cmd.Use = "list" - cmd.Short = `Get alerts.` - cmd.Long = `Get alerts. + cmd.Short = `List alerts.` + cmd.Long = `List alerts. - Gets a list of alerts. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of alerts accessible to the user, ordered by creation time. + **Warning:** Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Alerts.List(ctx) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + + response := w.Alerts.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -307,7 +295,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -319,35 +307,44 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.EditAlert, + *sql.UpdateAlertRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.EditAlert + var updateReq sql.UpdateAlertRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().IntVar(&updateReq.Rearm, "rearm", updateReq.Rearm, `Number of seconds after being triggered before the alert rearms itself and can be triggered again.`) + // TODO: complex arg: alert - cmd.Use = "update ALERT_ID" + cmd.Use = "update ID UPDATE_MASK" cmd.Short = `Update an alert.` cmd.Long = `Update an alert. Updates an alert. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + + Arguments: + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) return check(cmd, args) } @@ -361,16 +358,17 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } - updateReq.AlertId = args[0] + updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } - err = w.Alerts.Update(ctx, updateReq) + response, err := w.Alerts.Update(ctx, updateReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index 1572d4f4b..bc3fbe920 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -9,7 +9,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -24,9 +24,9 @@ func New() *cobra.Command { Long: `Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`, - GroupID: "serving", + GroupID: "apps", Annotations: map[string]string{ - "package": "serving", + "package": "apps", }, // This service is being previewed; hide from help output. @@ -39,12 +39,15 @@ func New() *cobra.Command { cmd.AddCommand(newDeploy()) cmd.AddCommand(newGet()) cmd.AddCommand(newGetDeployment()) - cmd.AddCommand(newGetEnvironment()) + cmd.AddCommand(newGetPermissionLevels()) + cmd.AddCommand(newGetPermissions()) cmd.AddCommand(newList()) cmd.AddCommand(newListDeployments()) + cmd.AddCommand(newSetPermissions()) cmd.AddCommand(newStart()) cmd.AddCommand(newStop()) cmd.AddCommand(newUpdate()) + cmd.AddCommand(newUpdatePermissions()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -60,13 +63,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *serving.CreateAppRequest, + *apps.CreateAppRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq serving.CreateAppRequest + var createReq apps.CreateAppRequest var createJson flags.JsonFlag var createSkipWait bool @@ -126,7 +129,7 @@ func newCreate() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *serving.App) { + info, err := wait.OnProgress(func(i *apps.App) { if i.Status == nil { return } @@ -162,13 +165,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *serving.DeleteAppRequest, + *apps.DeleteAppRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq serving.DeleteAppRequest + var deleteReq apps.DeleteAppRequest // TODO: short flags @@ -220,13 +223,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deployOverrides []func( *cobra.Command, - *serving.CreateAppDeploymentRequest, + *apps.CreateAppDeploymentRequest, ) func newDeploy() *cobra.Command { cmd := &cobra.Command{} - var deployReq serving.CreateAppDeploymentRequest + var deployReq apps.CreateAppDeploymentRequest var deployJson flags.JsonFlag var deploySkipWait bool @@ -237,7 +240,9 @@ func newDeploy() *cobra.Command { // TODO: short flags cmd.Flags().Var(&deployJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH MODE" + cmd.Flags().Var(&deployReq.Mode, "mode", `The mode of which the deployment will manage the source code. Supported values: [AUTO_SYNC, SNAPSHOT]`) + + cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH" cmd.Short = `Create an app deployment.` cmd.Long = `Create an app deployment. @@ -251,8 +256,7 @@ func newDeploy() *cobra.Command { deployed app. The former refers to the original source code location of the app in the workspace during deployment creation, whereas the latter provides a system generated stable snapshotted source code path used by - the deployment. - MODE: The mode of which the deployment will manage the source code.` + the deployment.` cmd.Annotations = make(map[string]string) @@ -260,11 +264,11 @@ func newDeploy() *cobra.Command { if cmd.Flags().Changed("json") { err := root.ExactArgs(1)(cmd, args) if err != nil { - return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path', 'mode' in your JSON input") + return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input") } return nil } - check := root.ExactArgs(3) + check := root.ExactArgs(2) return check(cmd, args) } @@ -283,12 +287,6 @@ func newDeploy() *cobra.Command { if !cmd.Flags().Changed("json") { deployReq.SourceCodePath = args[1] } - if !cmd.Flags().Changed("json") { - _, err = fmt.Sscan(args[2], &deployReq.Mode) - if err != nil { - return fmt.Errorf("invalid MODE: %s", args[2]) - } - } wait, err := w.Apps.Deploy(ctx, deployReq) if err != nil { @@ -298,7 +296,7 @@ func newDeploy() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *serving.AppDeployment) { + info, err := wait.OnProgress(func(i *apps.AppDeployment) { if i.Status == nil { return } @@ -334,13 +332,13 @@ func newDeploy() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *serving.GetAppRequest, + *apps.GetAppRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq serving.GetAppRequest + var getReq apps.GetAppRequest // TODO: short flags @@ -392,13 +390,13 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getDeploymentOverrides []func( *cobra.Command, - *serving.GetAppDeploymentRequest, + *apps.GetAppDeploymentRequest, ) func newGetDeployment() *cobra.Command { cmd := &cobra.Command{} - var getDeploymentReq serving.GetAppDeploymentRequest + var getDeploymentReq apps.GetAppDeploymentRequest // TODO: short flags @@ -447,30 +445,30 @@ func newGetDeployment() *cobra.Command { return cmd } -// start get-environment command +// start get-permission-levels command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getEnvironmentOverrides []func( +var getPermissionLevelsOverrides []func( *cobra.Command, - *serving.GetAppEnvironmentRequest, + *apps.GetAppPermissionLevelsRequest, ) -func newGetEnvironment() *cobra.Command { +func newGetPermissionLevels() *cobra.Command { cmd := &cobra.Command{} - var getEnvironmentReq serving.GetAppEnvironmentRequest + var getPermissionLevelsReq apps.GetAppPermissionLevelsRequest // TODO: short flags - cmd.Use = "get-environment NAME" - cmd.Short = `Get app environment.` - cmd.Long = `Get app environment. + cmd.Use = "get-permission-levels APP_NAME" + cmd.Short = `Get app permission levels.` + cmd.Long = `Get app permission levels. - Retrieves app environment. + Gets the permission levels that a user can have on an object. Arguments: - NAME: The name of the app.` + APP_NAME: The app for which to get or manage permissions.` cmd.Annotations = make(map[string]string) @@ -484,9 +482,9 @@ func newGetEnvironment() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getEnvironmentReq.Name = args[0] + getPermissionLevelsReq.AppName = args[0] - response, err := w.Apps.GetEnvironment(ctx, getEnvironmentReq) + response, err := w.Apps.GetPermissionLevels(ctx, getPermissionLevelsReq) if err != nil { return err } @@ -498,8 +496,67 @@ func newGetEnvironment() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getEnvironmentOverrides { - fn(cmd, &getEnvironmentReq) + for _, fn := range getPermissionLevelsOverrides { + fn(cmd, &getPermissionLevelsReq) + } + + return cmd +} + +// start get-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getPermissionsOverrides []func( + *cobra.Command, + *apps.GetAppPermissionsRequest, +) + +func newGetPermissions() *cobra.Command { + cmd := &cobra.Command{} + + var getPermissionsReq apps.GetAppPermissionsRequest + + // TODO: short flags + + cmd.Use = "get-permissions APP_NAME" + cmd.Short = `Get app permissions.` + cmd.Long = `Get app permissions. + + Gets the permissions of an app. Apps can inherit permissions from their root + object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getPermissionsReq.AppName = args[0] + + response, err := w.Apps.GetPermissions(ctx, getPermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getPermissionsOverrides { + fn(cmd, &getPermissionsReq) } return cmd @@ -511,13 +568,13 @@ func newGetEnvironment() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, - *serving.ListAppsRequest, + *apps.ListAppsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} - var listReq serving.ListAppsRequest + var listReq apps.ListAppsRequest // TODO: short flags @@ -564,13 +621,13 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listDeploymentsOverrides []func( *cobra.Command, - *serving.ListAppDeploymentsRequest, + *apps.ListAppDeploymentsRequest, ) func newListDeployments() *cobra.Command { cmd := &cobra.Command{} - var listDeploymentsReq serving.ListAppDeploymentsRequest + var listDeploymentsReq apps.ListAppDeploymentsRequest // TODO: short flags @@ -616,20 +673,94 @@ func newListDeployments() *cobra.Command { return cmd } +// start set-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var setPermissionsOverrides []func( + *cobra.Command, + *apps.AppPermissionsRequest, +) + +func newSetPermissions() *cobra.Command { + cmd := &cobra.Command{} + + var setPermissionsReq apps.AppPermissionsRequest + var setPermissionsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&setPermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: access_control_list + + cmd.Use = "set-permissions APP_NAME" + cmd.Short = `Set app permissions.` + cmd.Long = `Set app permissions. + + Sets permissions on an app. Apps can inherit permissions from their root + object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = setPermissionsJson.Unmarshal(&setPermissionsReq) + if err != nil { + return err + } + } + setPermissionsReq.AppName = args[0] + + response, err := w.Apps.SetPermissions(ctx, setPermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range setPermissionsOverrides { + fn(cmd, &setPermissionsReq) + } + + return cmd +} + // start start command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. var startOverrides []func( *cobra.Command, - *serving.StartAppRequest, + *apps.StartAppRequest, ) func newStart() *cobra.Command { cmd := &cobra.Command{} - var startReq serving.StartAppRequest + var startReq apps.StartAppRequest + var startSkipWait bool + var startTimeout time.Duration + + cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach SUCCEEDED state`) + cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`) // TODO: short flags cmd.Use = "start NAME" @@ -655,11 +786,30 @@ func newStart() *cobra.Command { startReq.Name = args[0] - response, err := w.Apps.Start(ctx, startReq) + wait, err := w.Apps.Start(ctx, startReq) if err != nil { return err } - return cmdio.Render(ctx, response) + if startSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *apps.AppDeployment) { + if i.Status == nil { + return + } + status := i.Status.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.Status != nil { + statusMessage = i.Status.Message + } + spinner <- statusMessage + }).GetWithTimeout(startTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) } // Disable completions since they are not applicable. @@ -680,13 +830,13 @@ func newStart() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var stopOverrides []func( *cobra.Command, - *serving.StopAppRequest, + *apps.StopAppRequest, ) func newStop() *cobra.Command { cmd := &cobra.Command{} - var stopReq serving.StopAppRequest + var stopReq apps.StopAppRequest // TODO: short flags @@ -738,13 +888,13 @@ func newStop() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *serving.UpdateAppRequest, + *apps.UpdateAppRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq serving.UpdateAppRequest + var updateReq apps.UpdateAppRequest var updateJson flags.JsonFlag // TODO: short flags @@ -801,4 +951,73 @@ func newUpdate() *cobra.Command { return cmd } +// start update-permissions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updatePermissionsOverrides []func( + *cobra.Command, + *apps.AppPermissionsRequest, +) + +func newUpdatePermissions() *cobra.Command { + cmd := &cobra.Command{} + + var updatePermissionsReq apps.AppPermissionsRequest + var updatePermissionsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updatePermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: access_control_list + + cmd.Use = "update-permissions APP_NAME" + cmd.Short = `Update app permissions.` + cmd.Long = `Update app permissions. + + Updates the permissions on an app. Apps can inherit permissions from their + root object. + + Arguments: + APP_NAME: The app for which to get or manage permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if err != nil { + return err + } + } + updatePermissionsReq.AppName = args[0] + + response, err := w.Apps.UpdatePermissions(ctx, updatePermissionsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updatePermissionsOverrides { + fn(cmd, &updatePermissionsReq) + } + + return cmd +} + // end service Apps diff --git a/cmd/workspace/cluster-policies/cluster-policies.go b/cmd/workspace/cluster-policies/cluster-policies.go index 8129db477..830d44ca3 100755 --- a/cmd/workspace/cluster-policies/cluster-policies.go +++ b/cmd/workspace/cluster-policies/cluster-policies.go @@ -90,30 +90,20 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `Additional human-readable description of the cluster policy.`) // TODO: array: libraries cmd.Flags().Int64Var(&createReq.MaxClustersPerUser, "max-clusters-per-user", createReq.MaxClustersPerUser, `Max number of clusters per user that can be active using this policy.`) + cmd.Flags().StringVar(&createReq.Name, "name", createReq.Name, `Cluster Policy name requested by the user.`) cmd.Flags().StringVar(&createReq.PolicyFamilyDefinitionOverrides, "policy-family-definition-overrides", createReq.PolicyFamilyDefinitionOverrides, `Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).`) cmd.Flags().StringVar(&createReq.PolicyFamilyId, "policy-family-id", createReq.PolicyFamilyId, `ID of the policy family.`) - cmd.Use = "create NAME" + cmd.Use = "create" cmd.Short = `Create a new policy.` cmd.Long = `Create a new policy. - Creates a new policy with prescribed settings. - - Arguments: - NAME: Cluster Policy name requested by the user. This has to be unique. Length - must be between 1 and 100 characters.` + Creates a new policy with prescribed settings.` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(0)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input") - } - return nil - } - check := root.ExactArgs(1) + check := root.ExactArgs(0) return check(cmd, args) } @@ -128,9 +118,6 @@ func newCreate() *cobra.Command { return err } } - if !cmd.Flags().Changed("json") { - createReq.Name = args[0] - } response, err := w.ClusterPolicies.Create(ctx, createReq) if err != nil { @@ -264,10 +251,11 @@ func newEdit() *cobra.Command { cmd.Flags().StringVar(&editReq.Description, "description", editReq.Description, `Additional human-readable description of the cluster policy.`) // TODO: array: libraries cmd.Flags().Int64Var(&editReq.MaxClustersPerUser, "max-clusters-per-user", editReq.MaxClustersPerUser, `Max number of clusters per user that can be active using this policy.`) + cmd.Flags().StringVar(&editReq.Name, "name", editReq.Name, `Cluster Policy name requested by the user.`) cmd.Flags().StringVar(&editReq.PolicyFamilyDefinitionOverrides, "policy-family-definition-overrides", editReq.PolicyFamilyDefinitionOverrides, `Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).`) cmd.Flags().StringVar(&editReq.PolicyFamilyId, "policy-family-id", editReq.PolicyFamilyId, `ID of the policy family.`) - cmd.Use = "edit POLICY_ID NAME" + cmd.Use = "edit POLICY_ID" cmd.Short = `Update a cluster policy.` cmd.Long = `Update a cluster policy. @@ -275,9 +263,7 @@ func newEdit() *cobra.Command { governed by the previous policy invalid. Arguments: - POLICY_ID: The ID of the policy to update. - NAME: Cluster Policy name requested by the user. This has to be unique. Length - must be between 1 and 100 characters.` + POLICY_ID: The ID of the policy to update.` cmd.Annotations = make(map[string]string) @@ -285,12 +271,11 @@ func newEdit() *cobra.Command { if cmd.Flags().Changed("json") { err := root.ExactArgs(0)(cmd, args) if err != nil { - return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id', 'name' in your JSON input") + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id' in your JSON input") } return nil } - check := root.ExactArgs(2) - return check(cmd, args) + return nil } cmd.PreRunE = root.MustWorkspaceClient @@ -303,13 +288,26 @@ func newEdit() *cobra.Command { if err != nil { return err } - } - if !cmd.Flags().Changed("json") { + } else { + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No POLICY_ID argument specified. Loading names for Cluster Policies drop-down." + names, err := w.ClusterPolicies.PolicyNameToPolicyIdMap(ctx, compute.ListClusterPoliciesRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Cluster Policies drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "The ID of the policy to update") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have the id of the policy to update") + } editReq.PolicyId = args[0] } - if !cmd.Flags().Changed("json") { - editReq.Name = args[1] - } err = w.ClusterPolicies.Edit(ctx, editReq) if err != nil { @@ -353,7 +351,7 @@ func newGet() *cobra.Command { Get a cluster policy entity. Creation and editing is available to admins only. Arguments: - POLICY_ID: Canonical unique identifier for the cluster policy.` + POLICY_ID: Canonical unique identifier for the Cluster Policy.` cmd.Annotations = make(map[string]string) @@ -370,7 +368,7 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Cluster Policies drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "Canonical unique identifier for the cluster policy") + id, err := cmdio.Select(ctx, names, "Canonical unique identifier for the Cluster Policy") if err != nil { return err } diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index abde1bb71..a64a6ab7c 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -43,11 +43,10 @@ func New() *cobra.Command { manually terminate and restart an all-purpose cluster. Multiple users can share such clusters to do collaborative interactive analysis. - IMPORTANT: Databricks retains cluster configuration information for up to 200 - all-purpose clusters terminated in the last 30 days and up to 30 job clusters - recently terminated by the job scheduler. To keep an all-purpose cluster - configuration even after it has been terminated for more than 30 days, an - administrator can pin a cluster to the cluster list.`, + IMPORTANT: Databricks retains cluster configuration information for terminated + clusters for 30 days. To keep an all-purpose cluster configuration even after + it has been terminated for more than 30 days, an administrator can pin a + cluster to the cluster list.`, GroupID: "compute", Annotations: map[string]string{ "package": "compute", @@ -74,6 +73,7 @@ func New() *cobra.Command { cmd.AddCommand(newSparkVersions()) cmd.AddCommand(newStart()) cmd.AddCommand(newUnpin()) + cmd.AddCommand(newUpdate()) cmd.AddCommand(newUpdatePermissions()) // Apply optional overrides to this command. @@ -885,21 +885,18 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().StringVar(&listReq.CanUseClient, "can-use-client", listReq.CanUseClient, `Filter clusters based on what type of client it can be used for.`) + // TODO: complex arg: filter_by + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Use this field to specify the maximum number of results to be returned by the server.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Use next_page_token or prev_page_token returned from the previous request to list the next or previous page of clusters respectively.`) + // TODO: complex arg: sort_by cmd.Use = "list" - cmd.Short = `List all clusters.` - cmd.Long = `List all clusters. + cmd.Short = `List clusters.` + cmd.Long = `List clusters. - Return information about all pinned clusters, active clusters, up to 200 of - the most recently terminated all-purpose clusters in the past 30 days, and up - to 30 of the most recently terminated job clusters in the past 30 days. - - For example, if there is 1 pinned cluster, 4 active clusters, 45 terminated - all-purpose clusters in the past 30 days, and 50 terminated job clusters in - the past 30 days, then this API returns the 1 pinned cluster, 4 active - clusters, all 45 terminated all-purpose clusters, and the 30 most recently - terminated job clusters.` + Return information about all pinned and active clusters, and all clusters + terminated within the last 30 days. Clusters terminated prior to this period + are not included.` cmd.Annotations = make(map[string]string) @@ -1753,6 +1750,117 @@ func newUnpin() *cobra.Command { return cmd } +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *compute.UpdateCluster, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq compute.UpdateCluster + var updateJson flags.JsonFlag + + var updateSkipWait bool + var updateTimeout time.Duration + + cmd.Flags().BoolVar(&updateSkipWait, "no-wait", updateSkipWait, `do not wait to reach RUNNING state`) + cmd.Flags().DurationVar(&updateTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach RUNNING state`) + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: cluster + + cmd.Use = "update CLUSTER_ID UPDATE_MASK" + cmd.Short = `Update cluster configuration (partial).` + cmd.Long = `Update cluster configuration (partial). + + Updates the configuration of a cluster to match the partial set of attributes + and size. Denote which fields to update using the update_mask field in the + request body. A cluster can be updated if it is in a RUNNING or TERMINATED + state. If a cluster is updated while in a RUNNING state, it will be + restarted so that the new attributes can take effect. If a cluster is updated + while in a TERMINATED state, it will remain TERMINATED. The updated + attributes will take effect the next time the cluster is started using the + clusters/start API. Attempts to update a cluster in any other state will be + rejected with an INVALID_STATE error code. Clusters created by the + Databricks Jobs service cannot be updated. + + Arguments: + CLUSTER_ID: ID of the cluster. + UPDATE_MASK: Specifies which fields of the cluster will be updated. This is required in + the POST request. The update mask should be supplied as a single string. + To specify multiple fields, separate them with commas (no spaces). To + delete a field from a cluster configuration, add it to the update_mask + string but omit it from the cluster object.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'cluster_id', 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + if !cmd.Flags().Changed("json") { + updateReq.ClusterId = args[0] + } + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } + + wait, err := w.Clusters.Update(ctx, updateReq) + if err != nil { + return err + } + if updateSkipWait { + return nil + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *compute.ClusterDetails) { + statusMessage := i.StateMessage + spinner <- statusMessage + }).GetWithTimeout(updateTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + // start update-permissions command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/clusters/overrides.go b/cmd/workspace/clusters/overrides.go index 55976d406..6038978ae 100644 --- a/cmd/workspace/clusters/overrides.go +++ b/cmd/workspace/clusters/overrides.go @@ -1,17 +1,83 @@ package clusters import ( + "strings" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/spf13/cobra" ) -func listOverride(listCmd *cobra.Command, _ *compute.ListClustersRequest) { +// Below we add overrides for filter flags for cluster list command to allow for custom filtering +// Auto generating such flags is not yet supported by the CLI generator +func listOverride(listCmd *cobra.Command, listReq *compute.ListClustersRequest) { listCmd.Annotations["headerTemplate"] = cmdio.Heredoc(` {{header "ID"}} {{header "Name"}} {{header "State"}}`) listCmd.Annotations["template"] = cmdio.Heredoc(` {{range .}}{{.ClusterId | green}} {{.ClusterName | cyan}} {{if eq .State "RUNNING"}}{{green "%s" .State}}{{else if eq .State "TERMINATED"}}{{red "%s" .State}}{{else}}{{blue "%s" .State}}{{end}} {{end}}`) + + listReq.FilterBy = &compute.ListClustersFilterBy{} + listCmd.Flags().BoolVar(&listReq.FilterBy.IsPinned, "is-pinned", false, "Filter clusters by pinned status") + listCmd.Flags().StringVar(&listReq.FilterBy.PolicyId, "policy-id", "", "Filter clusters by policy id") + + sources := &clusterSources{source: &listReq.FilterBy.ClusterSources} + listCmd.Flags().Var(sources, "cluster-sources", "Filter clusters by source") + + states := &clusterStates{state: &listReq.FilterBy.ClusterStates} + listCmd.Flags().Var(states, "cluster-states", "Filter clusters by states") +} + +type clusterSources struct { + source *[]compute.ClusterSource +} + +func (c *clusterSources) String() string { + s := make([]string, len(*c.source)) + for i, source := range *c.source { + s[i] = string(source) + } + + return strings.Join(s, ",") +} + +func (c *clusterSources) Set(value string) error { + splits := strings.Split(value, ",") + for _, split := range splits { + *c.source = append(*c.source, compute.ClusterSource(split)) + } + + return nil +} + +func (c *clusterSources) Type() string { + return "[]string" +} + +type clusterStates struct { + state *[]compute.State +} + +func (c *clusterStates) String() string { + s := make([]string, len(*c.state)) + for i, source := range *c.state { + s[i] = string(source) + } + + return strings.Join(s, ",") +} + +func (c *clusterStates) Set(value string) error { + splits := strings.Split(value, ",") + for _, split := range splits { + *c.state = append(*c.state, compute.State(split)) + } + + return nil +} + +func (c *clusterStates) Type() string { + return "[]string" } func listNodeTypesOverride(listNodeTypesCmd *cobra.Command) { diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 7ad9389a8..75664c79c 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -4,6 +4,7 @@ package workspace import ( alerts "github.com/databricks/cli/cmd/workspace/alerts" + alerts_legacy "github.com/databricks/cli/cmd/workspace/alerts-legacy" apps "github.com/databricks/cli/cmd/workspace/apps" artifact_allowlists "github.com/databricks/cli/cmd/workspace/artifact-allowlists" catalogs "github.com/databricks/cli/cmd/workspace/catalogs" @@ -24,6 +25,7 @@ import ( experiments "github.com/databricks/cli/cmd/workspace/experiments" external_locations "github.com/databricks/cli/cmd/workspace/external-locations" functions "github.com/databricks/cli/cmd/workspace/functions" + genie "github.com/databricks/cli/cmd/workspace/genie" git_credentials "github.com/databricks/cli/cmd/workspace/git-credentials" global_init_scripts "github.com/databricks/cli/cmd/workspace/global-init-scripts" grants "github.com/databricks/cli/cmd/workspace/grants" @@ -37,6 +39,7 @@ import ( metastores "github.com/databricks/cli/cmd/workspace/metastores" model_registry "github.com/databricks/cli/cmd/workspace/model-registry" model_versions "github.com/databricks/cli/cmd/workspace/model-versions" + notification_destinations "github.com/databricks/cli/cmd/workspace/notification-destinations" online_tables "github.com/databricks/cli/cmd/workspace/online-tables" permission_migration "github.com/databricks/cli/cmd/workspace/permission-migration" permissions "github.com/databricks/cli/cmd/workspace/permissions" @@ -52,8 +55,10 @@ import ( providers "github.com/databricks/cli/cmd/workspace/providers" quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors" queries "github.com/databricks/cli/cmd/workspace/queries" + queries_legacy "github.com/databricks/cli/cmd/workspace/queries-legacy" query_history "github.com/databricks/cli/cmd/workspace/query-history" query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations" + query_visualizations_legacy "github.com/databricks/cli/cmd/workspace/query-visualizations-legacy" recipient_activation "github.com/databricks/cli/cmd/workspace/recipient-activation" recipients "github.com/databricks/cli/cmd/workspace/recipients" registered_models "github.com/databricks/cli/cmd/workspace/registered-models" @@ -85,6 +90,7 @@ func All() []*cobra.Command { var out []*cobra.Command out = append(out, alerts.New()) + out = append(out, alerts_legacy.New()) out = append(out, apps.New()) out = append(out, artifact_allowlists.New()) out = append(out, catalogs.New()) @@ -105,6 +111,7 @@ func All() []*cobra.Command { out = append(out, experiments.New()) out = append(out, external_locations.New()) out = append(out, functions.New()) + out = append(out, genie.New()) out = append(out, git_credentials.New()) out = append(out, global_init_scripts.New()) out = append(out, grants.New()) @@ -118,6 +125,7 @@ func All() []*cobra.Command { out = append(out, metastores.New()) out = append(out, model_registry.New()) out = append(out, model_versions.New()) + out = append(out, notification_destinations.New()) out = append(out, online_tables.New()) out = append(out, permission_migration.New()) out = append(out, permissions.New()) @@ -133,8 +141,10 @@ func All() []*cobra.Command { out = append(out, providers.New()) out = append(out, quality_monitors.New()) out = append(out, queries.New()) + out = append(out, queries_legacy.New()) out = append(out, query_history.New()) out = append(out, query_visualizations.New()) + out = append(out, query_visualizations_legacy.New()) out = append(out, recipient_activation.New()) out = append(out, recipients.New()) out = append(out, registered_models.New()) diff --git a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go index 6f3ba4b42..46fd27c6f 100755 --- a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go +++ b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go @@ -22,9 +22,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go index d176e5b39..92f61789f 100755 --- a/cmd/workspace/consumer-installations/consumer-installations.go +++ b/cmd/workspace/consumer-installations/consumer-installations.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go index 18f3fb39e..5a8f76e36 100755 --- a/cmd/workspace/consumer-listings/consumer-listings.go +++ b/cmd/workspace/consumer-listings/consumer-listings.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods @@ -186,14 +183,12 @@ func newList() *cobra.Command { // TODO: array: assets // TODO: array: categories - cmd.Flags().BoolVar(&listReq.IsAscending, "is-ascending", listReq.IsAscending, ``) cmd.Flags().BoolVar(&listReq.IsFree, "is-free", listReq.IsFree, `Filters each listing based on if it is free.`) cmd.Flags().BoolVar(&listReq.IsPrivateExchange, "is-private-exchange", listReq.IsPrivateExchange, `Filters each listing based on if it is a private exchange.`) cmd.Flags().BoolVar(&listReq.IsStaffPick, "is-staff-pick", listReq.IsStaffPick, `Filters each listing based on whether it is a staff pick.`) cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) // TODO: array: provider_ids - cmd.Flags().Var(&listReq.SortBy, "sort-by", `Criteria for sorting the resulting set of listings. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`) // TODO: array: tags cmd.Use = "list" @@ -249,13 +244,11 @@ func newSearch() *cobra.Command { // TODO: array: assets // TODO: array: categories - cmd.Flags().BoolVar(&searchReq.IsAscending, "is-ascending", searchReq.IsAscending, ``) cmd.Flags().BoolVar(&searchReq.IsFree, "is-free", searchReq.IsFree, ``) cmd.Flags().BoolVar(&searchReq.IsPrivateExchange, "is-private-exchange", searchReq.IsPrivateExchange, ``) cmd.Flags().IntVar(&searchReq.PageSize, "page-size", searchReq.PageSize, ``) cmd.Flags().StringVar(&searchReq.PageToken, "page-token", searchReq.PageToken, ``) // TODO: array: provider_ids - cmd.Flags().Var(&searchReq.SortBy, "sort-by", `. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`) cmd.Use = "search QUERY" cmd.Short = `Search listings.` diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go index c55ca4ee1..8b0af3cc6 100755 --- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go +++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/consumer-providers/consumer-providers.go b/cmd/workspace/consumer-providers/consumer-providers.go index 579a89516..ab84249e9 100755 --- a/cmd/workspace/consumer-providers/consumer-providers.go +++ b/cmd/workspace/consumer-providers/consumer-providers.go @@ -24,9 +24,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/data-sources/data-sources.go b/cmd/workspace/data-sources/data-sources.go index f310fe50a..9f8a9dcd7 100755 --- a/cmd/workspace/data-sources/data-sources.go +++ b/cmd/workspace/data-sources/data-sources.go @@ -27,10 +27,10 @@ func New() *cobra.Command { grep to search the response from this API for the name of your SQL warehouse as it appears in Databricks SQL. - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] + **Note**: A new version of the Databricks SQL API is now available. [Learn + more] - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -67,10 +67,10 @@ func newList() *cobra.Command { fields that appear in this API response are enumerated for clarity. However, you need only a SQL warehouse's id to create new queries against it. - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:warehouses/list instead. [Learn more] - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go new file mode 100755 index 000000000..e4a059091 --- /dev/null +++ b/cmd/workspace/genie/genie.go @@ -0,0 +1,437 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package genie + +import ( + "fmt" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie", + Short: `Genie provides a no-code experience for business users, powered by AI/BI.`, + Long: `Genie provides a no-code experience for business users, powered by AI/BI. + Analysts set up spaces that business users can use to ask questions using + natural language. Genie uses data registered to Unity Catalog and requires at + least CAN USE permission on a Pro or Serverless SQL warehouse. Also, + Databricks Assistant must be enabled.`, + GroupID: "dashboards", + Annotations: map[string]string{ + "package": "dashboards", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCreateMessage()) + cmd.AddCommand(newExecuteMessageQuery()) + cmd.AddCommand(newGetMessage()) + cmd.AddCommand(newGetMessageQueryResult()) + cmd.AddCommand(newStartConversation()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create-message command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createMessageOverrides []func( + *cobra.Command, + *dashboards.GenieCreateConversationMessageRequest, +) + +func newCreateMessage() *cobra.Command { + cmd := &cobra.Command{} + + var createMessageReq dashboards.GenieCreateConversationMessageRequest + var createMessageJson flags.JsonFlag + + var createMessageSkipWait bool + var createMessageTimeout time.Duration + + cmd.Flags().BoolVar(&createMessageSkipWait, "no-wait", createMessageSkipWait, `do not wait to reach COMPLETED state`) + cmd.Flags().DurationVar(&createMessageTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach COMPLETED state`) + // TODO: short flags + cmd.Flags().Var(&createMessageJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create-message SPACE_ID CONVERSATION_ID CONTENT" + cmd.Short = `Create conversation message.` + cmd.Long = `Create conversation message. + + Create new message in [conversation](:method:genie/startconversation). The AI + response uses all previously created messages in the conversation to respond. + + Arguments: + SPACE_ID: The ID associated with the Genie space where the conversation is started. + CONVERSATION_ID: The ID associated with the conversation. + CONTENT: User message content.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only SPACE_ID, CONVERSATION_ID as positional arguments. Provide 'content' in your JSON input") + } + return nil + } + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createMessageJson.Unmarshal(&createMessageReq) + if err != nil { + return err + } + } + createMessageReq.SpaceId = args[0] + createMessageReq.ConversationId = args[1] + if !cmd.Flags().Changed("json") { + createMessageReq.Content = args[2] + } + + wait, err := w.Genie.CreateMessage(ctx, createMessageReq) + if err != nil { + return err + } + if createMessageSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *dashboards.GenieMessage) { + status := i.Status + statusMessage := fmt.Sprintf("current status: %s", status) + spinner <- statusMessage + }).GetWithTimeout(createMessageTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createMessageOverrides { + fn(cmd, &createMessageReq) + } + + return cmd +} + +// start execute-message-query command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var executeMessageQueryOverrides []func( + *cobra.Command, + *dashboards.ExecuteMessageQueryRequest, +) + +func newExecuteMessageQuery() *cobra.Command { + cmd := &cobra.Command{} + + var executeMessageQueryReq dashboards.ExecuteMessageQueryRequest + + // TODO: short flags + + cmd.Use = "execute-message-query SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Execute SQL query in a conversation message.` + cmd.Long = `Execute SQL query in a conversation message. + + Execute the SQL query in the message. + + Arguments: + SPACE_ID: Genie space ID + CONVERSATION_ID: Conversation ID + MESSAGE_ID: Message ID` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + executeMessageQueryReq.SpaceId = args[0] + executeMessageQueryReq.ConversationId = args[1] + executeMessageQueryReq.MessageId = args[2] + + response, err := w.Genie.ExecuteMessageQuery(ctx, executeMessageQueryReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range executeMessageQueryOverrides { + fn(cmd, &executeMessageQueryReq) + } + + return cmd +} + +// start get-message command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getMessageOverrides []func( + *cobra.Command, + *dashboards.GenieGetConversationMessageRequest, +) + +func newGetMessage() *cobra.Command { + cmd := &cobra.Command{} + + var getMessageReq dashboards.GenieGetConversationMessageRequest + + // TODO: short flags + + cmd.Use = "get-message SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Get conversation message.` + cmd.Long = `Get conversation message. + + Get message from conversation. + + Arguments: + SPACE_ID: The ID associated with the Genie space where the target conversation is + located. + CONVERSATION_ID: The ID associated with the target conversation. + MESSAGE_ID: The ID associated with the target message from the identified + conversation.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getMessageReq.SpaceId = args[0] + getMessageReq.ConversationId = args[1] + getMessageReq.MessageId = args[2] + + response, err := w.Genie.GetMessage(ctx, getMessageReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getMessageOverrides { + fn(cmd, &getMessageReq) + } + + return cmd +} + +// start get-message-query-result command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getMessageQueryResultOverrides []func( + *cobra.Command, + *dashboards.GenieGetMessageQueryResultRequest, +) + +func newGetMessageQueryResult() *cobra.Command { + cmd := &cobra.Command{} + + var getMessageQueryResultReq dashboards.GenieGetMessageQueryResultRequest + + // TODO: short flags + + cmd.Use = "get-message-query-result SPACE_ID CONVERSATION_ID MESSAGE_ID" + cmd.Short = `Get conversation message SQL query result.` + cmd.Long = `Get conversation message SQL query result. + + Get the result of SQL query if the message has a query attachment. This is + only available if a message has a query attachment and the message status is + EXECUTING_QUERY. + + Arguments: + SPACE_ID: Genie space ID + CONVERSATION_ID: Conversation ID + MESSAGE_ID: Message ID` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getMessageQueryResultReq.SpaceId = args[0] + getMessageQueryResultReq.ConversationId = args[1] + getMessageQueryResultReq.MessageId = args[2] + + response, err := w.Genie.GetMessageQueryResult(ctx, getMessageQueryResultReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getMessageQueryResultOverrides { + fn(cmd, &getMessageQueryResultReq) + } + + return cmd +} + +// start start-conversation command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var startConversationOverrides []func( + *cobra.Command, + *dashboards.GenieStartConversationMessageRequest, +) + +func newStartConversation() *cobra.Command { + cmd := &cobra.Command{} + + var startConversationReq dashboards.GenieStartConversationMessageRequest + var startConversationJson flags.JsonFlag + + var startConversationSkipWait bool + var startConversationTimeout time.Duration + + cmd.Flags().BoolVar(&startConversationSkipWait, "no-wait", startConversationSkipWait, `do not wait to reach COMPLETED state`) + cmd.Flags().DurationVar(&startConversationTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach COMPLETED state`) + // TODO: short flags + cmd.Flags().Var(&startConversationJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "start-conversation SPACE_ID CONTENT" + cmd.Short = `Start conversation.` + cmd.Long = `Start conversation. + + Start a new conversation. + + Arguments: + SPACE_ID: The ID associated with the Genie space where you want to start a + conversation. + CONTENT: The text of the message that starts the conversation.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only SPACE_ID as positional arguments. Provide 'content' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = startConversationJson.Unmarshal(&startConversationReq) + if err != nil { + return err + } + } + startConversationReq.SpaceId = args[0] + if !cmd.Flags().Changed("json") { + startConversationReq.Content = args[1] + } + + wait, err := w.Genie.StartConversation(ctx, startConversationReq) + if err != nil { + return err + } + if startConversationSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *dashboards.GenieMessage) { + status := i.Status + statusMessage := fmt.Sprintf("current status: %s", status) + spinner <- statusMessage + }).GetWithTimeout(startConversationTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range startConversationOverrides { + fn(cmd, &startConversationReq) + } + + return cmd +} + +// end service Genie diff --git a/cmd/workspace/groups.go b/cmd/workspace/groups.go index d8a4dec4f..98e474d33 100644 --- a/cmd/workspace/groups.go +++ b/cmd/workspace/groups.go @@ -68,5 +68,9 @@ func Groups() []cobra.Group { ID: "marketplace", Title: "Marketplace", }, + { + ID: "apps", + Title: "Apps", + }, } } diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 50a045921..2d422fa8c 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -817,6 +817,7 @@ func newGetRun() *cobra.Command { cmd.Flags().BoolVar(&getRunReq.IncludeHistory, "include-history", getRunReq.IncludeHistory, `Whether to include the repair history in the response.`) cmd.Flags().BoolVar(&getRunReq.IncludeResolvedValues, "include-resolved-values", getRunReq.IncludeResolvedValues, `Whether to include resolved parameter values in the response.`) + cmd.Flags().StringVar(&getRunReq.PageToken, "page-token", getRunReq.PageToken, `To list the next page or the previous page of job tasks, set this field to the value of the next_page_token or prev_page_token returned in the GetJob response.`) cmd.Use = "get-run RUN_ID" cmd.Short = `Get a single job run.` diff --git a/cmd/workspace/lakeview/lakeview.go b/cmd/workspace/lakeview/lakeview.go index 36eab0e7f..ef2d6845b 100755 --- a/cmd/workspace/lakeview/lakeview.go +++ b/cmd/workspace/lakeview/lakeview.go @@ -666,7 +666,7 @@ func newList() *cobra.Command { cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `The number of dashboards to return per page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A page token, received from a previous ListDashboards call.`) cmd.Flags().BoolVar(&listReq.ShowTrashed, "show-trashed", listReq.ShowTrashed, `The flag to include dashboards located in the trash.`) - cmd.Flags().Var(&listReq.View, "view", `Indicates whether to include all metadata from the dashboard in the response. Supported values: [DASHBOARD_VIEW_BASIC, DASHBOARD_VIEW_FULL]`) + cmd.Flags().Var(&listReq.View, "view", `DASHBOARD_VIEW_BASIConly includes summary metadata from the dashboard. Supported values: [DASHBOARD_VIEW_BASIC]`) cmd.Use = "list" cmd.Short = `List dashboards.` diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go index 034cea2df..d2f054045 100755 --- a/cmd/workspace/model-versions/model-versions.go +++ b/cmd/workspace/model-versions/model-versions.go @@ -133,6 +133,7 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getReq.IncludeAliases, "include-aliases", getReq.IncludeAliases, `Whether to include aliases associated with the model version in the response.`) cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include model versions in the response for which the principal can only access selective metadata for.`) cmd.Use = "get FULL_NAME VERSION" @@ -203,6 +204,8 @@ func newGetByAlias() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getByAliasReq.IncludeAliases, "include-aliases", getByAliasReq.IncludeAliases, `Whether to include aliases associated with the model version in the response.`) + cmd.Use = "get-by-alias FULL_NAME ALIAS" cmd.Short = `Get Model Version By Alias.` cmd.Long = `Get Model Version By Alias. diff --git a/cmd/workspace/notification-destinations/notification-destinations.go b/cmd/workspace/notification-destinations/notification-destinations.go new file mode 100755 index 000000000..5ad47cc95 --- /dev/null +++ b/cmd/workspace/notification-destinations/notification-destinations.go @@ -0,0 +1,342 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package notification_destinations + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "notification-destinations", + Short: `The notification destinations API lets you programmatically manage a workspace's notification destinations.`, + Long: `The notification destinations API lets you programmatically manage a + workspace's notification destinations. Notification destinations are used to + send notifications for query alerts and jobs to destinations outside of + Databricks. Only workspace admins can create, update, and delete notification + destinations.`, + GroupID: "settings", + Annotations: map[string]string{ + "package": "settings", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *settings.CreateNotificationDestinationRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq settings.CreateNotificationDestinationRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: config + cmd.Flags().StringVar(&createReq.DisplayName, "display-name", createReq.DisplayName, `The display name for the notification destination.`) + + cmd.Use = "create" + cmd.Short = `Create a notification destination.` + cmd.Long = `Create a notification destination. + + Creates a notification destination. Requires workspace admin permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } + + response, err := w.NotificationDestinations.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteNotificationDestinationRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteNotificationDestinationRequest + + // TODO: short flags + + cmd.Use = "delete ID" + cmd.Short = `Delete a notification destination.` + cmd.Long = `Delete a notification destination. + + Deletes a notification destination. Requires workspace admin permissions.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + deleteReq.Id = args[0] + + err = w.NotificationDestinations.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetNotificationDestinationRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetNotificationDestinationRequest + + // TODO: short flags + + cmd.Use = "get ID" + cmd.Short = `Get a notification destination.` + cmd.Long = `Get a notification destination. + + Gets a notification destination.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getReq.Id = args[0] + + response, err := w.NotificationDestinations.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, + *settings.ListNotificationDestinationsRequest, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + var listReq settings.ListNotificationDestinationsRequest + + // TODO: short flags + + cmd.Flags().Int64Var(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) + + cmd.Use = "list" + cmd.Short = `List notification destinations.` + cmd.Long = `List notification destinations. + + Lists notification destinations.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response := w.NotificationDestinations.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd, &listReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateNotificationDestinationRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateNotificationDestinationRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: config + cmd.Flags().StringVar(&updateReq.DisplayName, "display-name", updateReq.DisplayName, `The display name for the notification destination.`) + + cmd.Use = "update ID" + cmd.Short = `Update a notification destination.` + cmd.Long = `Update a notification destination. + + Updates a notification destination. Requires workspace admin permissions. At + least one field is required in the request body.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + updateReq.Id = args[0] + + response, err := w.NotificationDestinations.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service NotificationDestinations diff --git a/cmd/workspace/permission-migration/permission-migration.go b/cmd/workspace/permission-migration/permission-migration.go index 40d3f9a3b..2e50b1231 100755 --- a/cmd/workspace/permission-migration/permission-migration.go +++ b/cmd/workspace/permission-migration/permission-migration.go @@ -19,9 +19,9 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "permission-migration", - Short: `This spec contains undocumented permission migration APIs used in https://github.com/databrickslabs/ucx.`, - Long: `This spec contains undocumented permission migration APIs used in - https://github.com/databrickslabs/ucx.`, + Short: `APIs for migrating acl permissions, used only by the ucx tool: https://github.com/databrickslabs/ucx.`, + Long: `APIs for migrating acl permissions, used only by the ucx tool: + https://github.com/databrickslabs/ucx`, GroupID: "iam", Annotations: map[string]string{ "package": "iam", @@ -48,13 +48,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var migratePermissionsOverrides []func( *cobra.Command, - *iam.PermissionMigrationRequest, + *iam.MigratePermissionsRequest, ) func newMigratePermissions() *cobra.Command { cmd := &cobra.Command{} - var migratePermissionsReq iam.PermissionMigrationRequest + var migratePermissionsReq iam.MigratePermissionsRequest var migratePermissionsJson flags.JsonFlag // TODO: short flags @@ -65,14 +65,10 @@ func newMigratePermissions() *cobra.Command { cmd.Use = "migrate-permissions WORKSPACE_ID FROM_WORKSPACE_GROUP_NAME TO_ACCOUNT_GROUP_NAME" cmd.Short = `Migrate Permissions.` cmd.Long = `Migrate Permissions. - - Migrate a batch of permissions from a workspace local group to an account - group. Arguments: WORKSPACE_ID: WorkspaceId of the associated workspace where the permission migration - will occur. Both workspace group and account group must be in this - workspace. + will occur. FROM_WORKSPACE_GROUP_NAME: The name of the workspace group that permissions will be migrated from. TO_ACCOUNT_GROUP_NAME: The name of the account group that permissions will be migrated to.` diff --git a/cmd/workspace/permissions/permissions.go b/cmd/workspace/permissions/permissions.go index 57a7d1e5e..fd9c1a468 100755 --- a/cmd/workspace/permissions/permissions.go +++ b/cmd/workspace/permissions/permissions.go @@ -21,6 +21,9 @@ func New() *cobra.Command { Long: `Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints. + * **[Apps permissions](:service:apps)** — Manage which users can manage or + use apps. + * **[Cluster permissions](:service:clusters)** — Manage which users can manage, restart, or attach to clusters. @@ -59,7 +62,8 @@ func New() *cobra.Command { create or use tokens. * **[Workspace object permissions](:service:workspace)** — Manage which - users can read, run, edit, or manage directories, files, and notebooks. + users can read, run, edit, or manage alerts, dbsql-dashboards, directories, + files, notebooks and queries. For the mapping of the required permissions for specific actions or abilities and other important information, see [Access Control]. @@ -112,10 +116,10 @@ func newGet() *cobra.Command { parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -240,10 +244,10 @@ func newSet() *cobra.Command { parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) @@ -314,10 +318,10 @@ func newUpdate() *cobra.Command { their parent objects or root object. Arguments: - REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: - authorization, clusters, cluster-policies, directories, experiments, - files, instance-pools, jobs, notebooks, pipelines, registered-models, - repos, serving-endpoints, or warehouses. + REQUEST_OBJECT_TYPE: The type of the request object. Can be one of the following: alerts, + authorization, clusters, cluster-policies, dbsql-dashboards, directories, + experiments, files, instance-pools, jobs, notebooks, pipelines, queries, + registered-models, repos, serving-endpoints, or warehouses. REQUEST_OBJECT_ID: The id of the request object.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/policy-families/policy-families.go b/cmd/workspace/policy-families/policy-families.go index beee6e963..cac23405b 100755 --- a/cmd/workspace/policy-families/policy-families.go +++ b/cmd/workspace/policy-families/policy-families.go @@ -60,11 +60,17 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().Int64Var(&getReq.Version, "version", getReq.Version, `The version number for the family to fetch.`) + cmd.Use = "get POLICY_FAMILY_ID" cmd.Short = `Get policy family information.` cmd.Long = `Get policy family information. - Retrieve the information for an policy family based on its identifier.` + Retrieve the information for an policy family based on its identifier and + version + + Arguments: + POLICY_FAMILY_ID: The family ID about which to retrieve information.` cmd.Annotations = make(map[string]string) @@ -115,14 +121,15 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().Int64Var(&listReq.MaxResults, "max-results", listReq.MaxResults, `The max number of policy families to return.`) + cmd.Flags().Int64Var(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of policy families to return.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) cmd.Use = "list" cmd.Short = `List policy families.` cmd.Long = `List policy families. - Retrieve a list of policy families. This API is paginated.` + Returns the list of policy definition types available to use at their latest + version. This API is paginated.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go index 4ab36b5d0..a3f746214 100755 --- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go +++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go index 7ff73e0d1..b92403755 100755 --- a/cmd/workspace/provider-exchanges/provider-exchanges.go +++ b/cmd/workspace/provider-exchanges/provider-exchanges.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go index 25e1addf5..62dcb6de9 100755 --- a/cmd/workspace/provider-files/provider-files.go +++ b/cmd/workspace/provider-files/provider-files.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go index 0abdf51d8..18c99c53d 100755 --- a/cmd/workspace/provider-listings/provider-listings.go +++ b/cmd/workspace/provider-listings/provider-listings.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go index a38d9f420..d18e2e578 100755 --- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go +++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go @@ -26,9 +26,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go index 8cee6e4eb..bb3ca9666 100755 --- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go +++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go @@ -23,9 +23,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go index b7273a344..94d12d6f0 100755 --- a/cmd/workspace/provider-providers/provider-providers.go +++ b/cmd/workspace/provider-providers/provider-providers.go @@ -25,9 +25,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "marketplace", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index 7305191c8..af2737a0f 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -291,6 +291,8 @@ func newList() *cobra.Command { // TODO: short flags cmd.Flags().StringVar(&listReq.DataProviderGlobalMetastoreId, "data-provider-global-metastore-id", listReq.DataProviderGlobalMetastoreId, `If not provided, all providers will be returned.`) + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of providers to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "list" cmd.Short = `List providers.` @@ -345,6 +347,9 @@ func newListShares() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&listSharesReq.MaxResults, "max-results", listSharesReq.MaxResults, `Maximum number of shares to return.`) + cmd.Flags().StringVar(&listSharesReq.PageToken, "page-token", listSharesReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list-shares NAME" cmd.Short = `List shares by Provider.` cmd.Long = `List shares by Provider. diff --git a/cmd/workspace/queries-legacy/queries-legacy.go b/cmd/workspace/queries-legacy/queries-legacy.go new file mode 100755 index 000000000..fa78bb2b0 --- /dev/null +++ b/cmd/workspace/queries-legacy/queries-legacy.go @@ -0,0 +1,500 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package queries_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "queries-legacy", + Short: `These endpoints are used for CRUD operations on query definitions.`, + Long: `These endpoints are used for CRUD operations on query definitions. Query + definitions include the target SQL warehouse, query text, name, description, + tags, parameters, and visualizations. Queries can be scheduled using the + sql_task type of the Jobs API, e.g. :method:jobs/create. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newList()) + cmd.AddCommand(newRestore()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.QueryPostContent, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.QueryPostContent + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create" + cmd.Short = `Create a new query definition.` + cmd.Long = `Create a new query definition. + + Creates a new query definition. Queries created with this endpoint belong to + the authenticated user making the request. + + The data_source_id field specifies the ID of the SQL warehouse to run this + query against. You can use the Data Sources API to see a complete list of + available SQL warehouses. Or you can copy the data_source_id from an + existing query. + + **Note**: You cannot add a visualization until you create the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.QueriesLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteQueriesLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "delete QUERY_ID" + cmd.Short = `Delete a query.` + cmd.Long = `Delete a query. + + Moves a query to the trash. Trashed queries immediately disappear from + searches and list views, and they cannot be used for alerts. The trash is + deleted after 30 days. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + deleteReq.QueryId = args[0] + + err = w.QueriesLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *sql.GetQueriesLegacyRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq sql.GetQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "get QUERY_ID" + cmd.Short = `Get a query definition.` + cmd.Long = `Get a query definition. + + Retrieve a query object definition along with contextual permissions + information about the currently authenticated user. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/get instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + getReq.QueryId = args[0] + + response, err := w.QueriesLegacy.Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start list command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOverrides []func( + *cobra.Command, + *sql.ListQueriesLegacyRequest, +) + +func newList() *cobra.Command { + cmd := &cobra.Command{} + + var listReq sql.ListQueriesLegacyRequest + + // TODO: short flags + + cmd.Flags().StringVar(&listReq.Order, "order", listReq.Order, `Name of query attribute to order by.`) + cmd.Flags().IntVar(&listReq.Page, "page", listReq.Page, `Page number to retrieve.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Number of queries to return per page.`) + cmd.Flags().StringVar(&listReq.Q, "q", listReq.Q, `Full text search term.`) + + cmd.Use = "list" + cmd.Short = `Get a list of queries.` + cmd.Long = `Get a list of queries. + + Gets a list of queries. Optionally, this list can be filtered by a search + term. + + **Warning**: Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/list instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response := w.QueriesLegacy.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOverrides { + fn(cmd, &listReq) + } + + return cmd +} + +// start restore command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var restoreOverrides []func( + *cobra.Command, + *sql.RestoreQueriesLegacyRequest, +) + +func newRestore() *cobra.Command { + cmd := &cobra.Command{} + + var restoreReq sql.RestoreQueriesLegacyRequest + + // TODO: short flags + + cmd.Use = "restore QUERY_ID" + cmd.Short = `Restore a query.` + cmd.Long = `Restore a query. + + Restore a query that has been moved to the trash. A restored query appears in + list views and searches. You can use restored queries for alerts. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + restoreReq.QueryId = args[0] + + err = w.QueriesLegacy.Restore(ctx, restoreReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range restoreOverrides { + fn(cmd, &restoreReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.QueryEditContent, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.QueryEditContent + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&updateReq.DataSourceId, "data-source-id", updateReq.DataSourceId, `Data source ID maps to the ID of the data source used by the resource and is distinct from the warehouse ID.`) + cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `General description that conveys additional information about this query such as usage notes.`) + cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this query that appears in list views, widget headings, and on the query page.`) + // TODO: any: options + cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`) + cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) + // TODO: array: tags + + cmd.Use = "update QUERY_ID" + cmd.Short = `Change a query definition.` + cmd.Long = `Change a query definition. + + Modify this query definition. + + **Note**: You cannot undo this operation. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queries/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + if len(args) == 0 { + promptSpinner := cmdio.Spinner(ctx) + promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries Legacy drop-down." + names, err := w.QueriesLegacy.LegacyQueryNameToIdMap(ctx, sql.ListQueriesLegacyRequest{}) + close(promptSpinner) + if err != nil { + return fmt.Errorf("failed to load names for Queries Legacy drop-down. Please manually specify required arguments. Original error: %w", err) + } + id, err := cmdio.Select(ctx, names, "") + if err != nil { + return err + } + args = append(args, id) + } + if len(args) != 1 { + return fmt.Errorf("expected to have ") + } + updateReq.QueryId = args[0] + + response, err := w.QueriesLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service QueriesLegacy diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index 650131974..fea01451a 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -19,16 +19,11 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "queries", - Short: `These endpoints are used for CRUD operations on query definitions.`, - Long: `These endpoints are used for CRUD operations on query definitions. Query - definitions include the target SQL warehouse, query text, name, description, - tags, parameters, and visualizations. Queries can be scheduled using the - sql_task type of the Jobs API, e.g. :method:jobs/create. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources`, + Short: `The queries API can be used to perform CRUD operations on queries.`, + Long: `The queries API can be used to perform CRUD operations on queries. A query is + a Databricks SQL object that includes the target SQL warehouse, query text, + name, description, tags, and parameters. Queries can be scheduled using the + sql_task type of the Jobs API, e.g. :method:jobs/create.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -40,7 +35,7 @@ func New() *cobra.Command { cmd.AddCommand(newDelete()) cmd.AddCommand(newGet()) cmd.AddCommand(newList()) - cmd.AddCommand(newRestore()) + cmd.AddCommand(newListVisualizations()) cmd.AddCommand(newUpdate()) // Apply optional overrides to this command. @@ -57,39 +52,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.QueryPostContent, + *sql.CreateQueryRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.QueryPostContent + var createReq sql.CreateQueryRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: query + cmd.Use = "create" - cmd.Short = `Create a new query definition.` - cmd.Long = `Create a new query definition. + cmd.Short = `Create a query.` + cmd.Long = `Create a query. - Creates a new query definition. Queries created with this endpoint belong to - the authenticated user making the request. - - The data_source_id field specifies the ID of the SQL warehouse to run this - query against. You can use the Data Sources API to see a complete list of - available SQL warehouses. Or you can copy the data_source_id from an - existing query. - - **Note**: You cannot add a visualization until you create the query. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Creates a query.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -100,8 +89,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.Queries.Create(ctx, createReq) @@ -129,28 +116,24 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteQueryRequest, + *sql.TrashQueryRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteQueryRequest + var deleteReq sql.TrashQueryRequest // TODO: short flags - cmd.Use = "delete QUERY_ID" + cmd.Use = "delete ID" cmd.Short = `Delete a query.` cmd.Long = `Delete a query. Moves a query to the trash. Trashed queries immediately disappear from - searches and list views, and they cannot be used for alerts. The trash is - deleted after 30 days. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + searches and list views, and cannot be used for alerts. You can restore a + trashed query through the UI. A trashed query is permanently deleted after 30 + days.` cmd.Annotations = make(map[string]string) @@ -161,8 +144,8 @@ func newDelete() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -176,7 +159,7 @@ func newDelete() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - deleteReq.QueryId = args[0] + deleteReq.Id = args[0] err = w.Queries.Delete(ctx, deleteReq) if err != nil { @@ -213,17 +196,11 @@ func newGet() *cobra.Command { // TODO: short flags - cmd.Use = "get QUERY_ID" - cmd.Short = `Get a query definition.` - cmd.Long = `Get a query definition. + cmd.Use = "get ID" + cmd.Short = `Get a query.` + cmd.Long = `Get a query. - Retrieve a query object definition along with contextual permissions - information about the currently authenticated user. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a query.` cmd.Annotations = make(map[string]string) @@ -234,8 +211,8 @@ func newGet() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -249,7 +226,7 @@ func newGet() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - getReq.QueryId = args[0] + getReq.Id = args[0] response, err := w.Queries.Get(ctx, getReq) if err != nil { @@ -286,25 +263,16 @@ func newList() *cobra.Command { // TODO: short flags - cmd.Flags().StringVar(&listReq.Order, "order", listReq.Order, `Name of query attribute to order by.`) - cmd.Flags().IntVar(&listReq.Page, "page", listReq.Page, `Page number to retrieve.`) - cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Number of queries to return per page.`) - cmd.Flags().StringVar(&listReq.Q, "q", listReq.Q, `Full text search term.`) + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) cmd.Use = "list" - cmd.Short = `Get a list of queries.` - cmd.Long = `Get a list of queries. + cmd.Short = `List queries.` + cmd.Long = `List queries. - Gets a list of queries. Optionally, this list can be filtered by a search - term. - - **Warning**: Calling this API concurrently 10 or more times could result in - throttling, service degradation, or a temporary ban. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of queries accessible to the user, ordered by creation time. + **Warning:** Calling this API concurrently 10 or more times could result in + throttling, service degradation, or a temporary ban.` cmd.Annotations = make(map[string]string) @@ -334,33 +302,33 @@ func newList() *cobra.Command { return cmd } -// start restore command +// start list-visualizations command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var restoreOverrides []func( +var listVisualizationsOverrides []func( *cobra.Command, - *sql.RestoreQueryRequest, + *sql.ListVisualizationsForQueryRequest, ) -func newRestore() *cobra.Command { +func newListVisualizations() *cobra.Command { cmd := &cobra.Command{} - var restoreReq sql.RestoreQueryRequest + var listVisualizationsReq sql.ListVisualizationsForQueryRequest // TODO: short flags - cmd.Use = "restore QUERY_ID" - cmd.Short = `Restore a query.` - cmd.Long = `Restore a query. + cmd.Flags().IntVar(&listVisualizationsReq.PageSize, "page-size", listVisualizationsReq.PageSize, ``) + cmd.Flags().StringVar(&listVisualizationsReq.PageToken, "page-token", listVisualizationsReq.PageToken, ``) + + cmd.Use = "list-visualizations ID" + cmd.Short = `List visualizations on a query.` + cmd.Long = `List visualizations on a query. - Restore a query that has been moved to the trash. A restored query appears in - list views and searches. You can use restored queries for alerts. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Gets a list of visualizations on a query.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true cmd.Annotations = make(map[string]string) @@ -371,8 +339,8 @@ func newRestore() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) + promptSpinner <- "No ID argument specified. Loading names for Queries drop-down." + names, err := w.Queries.ListQueryObjectsResponseQueryDisplayNameToIdMap(ctx, sql.ListQueriesRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) @@ -386,13 +354,10 @@ func newRestore() *cobra.Command { if len(args) != 1 { return fmt.Errorf("expected to have ") } - restoreReq.QueryId = args[0] + listVisualizationsReq.Id = args[0] - err = w.Queries.Restore(ctx, restoreReq) - if err != nil { - return err - } - return nil + response := w.Queries.ListVisualizations(ctx, listVisualizationsReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -400,8 +365,8 @@ func newRestore() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range restoreOverrides { - fn(cmd, &restoreReq) + for _, fn := range listVisualizationsOverrides { + fn(cmd, &listVisualizationsReq) } return cmd @@ -413,41 +378,47 @@ func newRestore() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.QueryEditContent, + *sql.UpdateQueryRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.QueryEditContent + var updateReq sql.UpdateQueryRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateReq.DataSourceId, "data-source-id", updateReq.DataSourceId, `Data source ID maps to the ID of the data source used by the resource and is distinct from the warehouse ID.`) - cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `General description that conveys additional information about this query such as usage notes.`) - cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this query that appears in list views, widget headings, and on the query page.`) - // TODO: any: options - cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`) - cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) - // TODO: array: tags + // TODO: complex arg: query - cmd.Use = "update QUERY_ID" - cmd.Short = `Change a query definition.` - cmd.Long = `Change a query definition. + cmd.Use = "update ID UPDATE_MASK" + cmd.Short = `Update a query.` + cmd.Long = `Update a query. - Modify this query definition. - - **Note**: You cannot undo this operation. - - **Note**: A new version of the Databricks SQL API will soon be available. - [Learn more] - - [Learn more]: https://docs.databricks.com/en/whats-coming.html#updates-to-the-databricks-sql-api-for-managing-queries-alerts-and-data-sources` + Updates a query. + + Arguments: + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -459,24 +430,10 @@ func newUpdate() *cobra.Command { return err } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No QUERY_ID argument specified. Loading names for Queries drop-down." - names, err := w.Queries.QueryNameToIdMap(ctx, sql.ListQueriesRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Queries drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "") - if err != nil { - return err - } - args = append(args, id) + updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] } - if len(args) != 1 { - return fmt.Errorf("expected to have ") - } - updateReq.QueryId = args[0] response, err := w.Queries.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/query-history/query-history.go b/cmd/workspace/query-history/query-history.go index 60d6004d9..5155b5cc0 100755 --- a/cmd/workspace/query-history/query-history.go +++ b/cmd/workspace/query-history/query-history.go @@ -15,9 +15,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "query-history", - Short: `Access the history of queries through SQL warehouses.`, - Long: `Access the history of queries through SQL warehouses.`, + Use: "query-history", + Short: `A service responsible for storing and retrieving the list of queries run against SQL endpoints, serverless compute, and DLT.`, + Long: `A service responsible for storing and retrieving the list of queries run + against SQL endpoints, serverless compute, and DLT.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -52,7 +53,6 @@ func newList() *cobra.Command { // TODO: short flags // TODO: complex arg: filter_by - cmd.Flags().BoolVar(&listReq.IncludeMetrics, "include-metrics", listReq.IncludeMetrics, `Whether to include metrics about query.`) cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Limit the number of results returned in one page.`) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `A token that can be used to get the next page of results.`) @@ -60,9 +60,13 @@ func newList() *cobra.Command { cmd.Short = `List Queries.` cmd.Long = `List Queries. - List the history of queries through SQL warehouses. + List the history of queries through SQL warehouses, serverless compute, and + DLT. - You can filter by user ID, warehouse ID, status, and time range.` + You can filter by user ID, warehouse ID, status, and time range. Most recently + started queries are returned first (up to max_results in request). The + pagination token returned in response can be used to list subsequent query + statuses.` cmd.Annotations = make(map[string]string) @@ -76,8 +80,11 @@ func newList() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response := w.QueryHistory.List(ctx, listReq) - return cmdio.RenderIterator(ctx, response) + response, err := w.QueryHistory.List(ctx, listReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go new file mode 100755 index 000000000..4f45ab23e --- /dev/null +++ b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go @@ -0,0 +1,253 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package query_visualizations_legacy + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "query-visualizations-legacy", + Short: `This is an evolving API that facilitates the addition and removal of vizualisations from existing queries within the Databricks Workspace.`, + Long: `This is an evolving API that facilitates the addition and removal of + vizualisations from existing queries within the Databricks Workspace. Data + structures may change over time. + + **Note**: A new version of the Databricks SQL API is now available. Please see + the latest version. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html`, + GroupID: "sql", + Annotations: map[string]string{ + "package": "sql", + }, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newCreate()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start create command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOverrides []func( + *cobra.Command, + *sql.CreateQueryVisualizationsLegacyRequest, +) + +func newCreate() *cobra.Command { + cmd := &cobra.Command{} + + var createReq sql.CreateQueryVisualizationsLegacyRequest + var createJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create" + cmd.Short = `Add visualization to a query.` + cmd.Long = `Add visualization to a query. + + Creates visualization in the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/create instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createJson.Unmarshal(&createReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.QueryVisualizationsLegacy.Create(ctx, createReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOverrides { + fn(cmd, &createReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *sql.DeleteQueryVisualizationsLegacyRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq sql.DeleteQueryVisualizationsLegacyRequest + + // TODO: short flags + + cmd.Use = "delete ID" + cmd.Short = `Remove visualization.` + cmd.Long = `Remove visualization. + + Removes a visualization from the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/delete instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + + Arguments: + ID: Widget ID returned by :method:queryvizualisations/create` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + deleteReq.Id = args[0] + + err = w.QueryVisualizationsLegacy.Delete(ctx, deleteReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *sql.LegacyVisualization, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq sql.LegacyVisualization + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update ID" + cmd.Short = `Edit existing visualization.` + cmd.Long = `Edit existing visualization. + + Updates visualization in the query. + + **Note**: A new version of the Databricks SQL API is now available. Please use + :method:queryvisualizations/update instead. [Learn more] + + [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + + Arguments: + ID: The UUID for this visualization.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + updateReq.Id = args[0] + + response, err := w.QueryVisualizationsLegacy.Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service QueryVisualizationsLegacy diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go index c94d83a82..042594529 100755 --- a/cmd/workspace/query-visualizations/query-visualizations.go +++ b/cmd/workspace/query-visualizations/query-visualizations.go @@ -19,10 +19,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "query-visualizations", - Short: `This is an evolving API that facilitates the addition and removal of vizualisations from existing queries within the Databricks Workspace.`, + Short: `This is an evolving API that facilitates the addition and removal of visualizations from existing queries in the Databricks Workspace.`, Long: `This is an evolving API that facilitates the addition and removal of - vizualisations from existing queries within the Databricks Workspace. Data - structures may change over time.`, + visualizations from existing queries in the Databricks Workspace. Data + structures can change over time.`, GroupID: "sql", Annotations: map[string]string{ "package": "sql", @@ -51,24 +51,33 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *sql.CreateQueryVisualizationRequest, + *sql.CreateVisualizationRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq sql.CreateQueryVisualizationRequest + var createReq sql.CreateVisualizationRequest var createJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: visualization + cmd.Use = "create" - cmd.Short = `Add visualization to a query.` - cmd.Long = `Add visualization to a query.` + cmd.Short = `Add a visualization to a query.` + cmd.Long = `Add a visualization to a query. + + Adds a visualization to a query.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -79,8 +88,6 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } response, err := w.QueryVisualizations.Create(ctx, createReq) @@ -108,22 +115,21 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *sql.DeleteQueryVisualizationRequest, + *sql.DeleteVisualizationRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq sql.DeleteQueryVisualizationRequest + var deleteReq sql.DeleteVisualizationRequest // TODO: short flags cmd.Use = "delete ID" - cmd.Short = `Remove visualization.` - cmd.Long = `Remove visualization. - - Arguments: - ID: Widget ID returned by :method:queryvizualisations/create` + cmd.Short = `Remove a visualization.` + cmd.Long = `Remove a visualization. + + Removes a visualization.` cmd.Annotations = make(map[string]string) @@ -164,29 +170,44 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *sql.Visualization, + *sql.UpdateVisualizationRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq sql.Visualization + var updateReq sql.UpdateVisualizationRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Use = "update ID" - cmd.Short = `Edit existing visualization.` - cmd.Long = `Edit existing visualization. + // TODO: complex arg: visualization + + cmd.Use = "update ID UPDATE_MASK" + cmd.Short = `Update a visualization.` + cmd.Long = `Update a visualization. + + Updates a visualization. Arguments: - ID: The UUID for this visualization.` + ID: + UPDATE_MASK: Field mask is required to be passed into the PATCH request. Field mask + specifies which fields of the setting payload will be updated. The field + mask needs to be supplied as single string. To specify multiple fields in + the field mask, use comma as the separator (no space).` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(1) + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only ID as positional arguments. Provide 'update_mask' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) return check(cmd, args) } @@ -200,10 +221,11 @@ func newUpdate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } updateReq.Id = args[0] + if !cmd.Flags().Changed("json") { + updateReq.UpdateMask = args[1] + } response, err := w.QueryVisualizations.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index c21d8a8c0..f4472cf37 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -80,6 +80,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `Description about the recipient.`) cmd.Flags().StringVar(&createReq.DataRecipientGlobalMetastoreId, "data-recipient-global-metastore-id", createReq.DataRecipientGlobalMetastoreId, `The global Unity Catalog metastore id provided by the data recipient.`) + cmd.Flags().Int64Var(&createReq.ExpirationTime, "expiration-time", createReq.ExpirationTime, `Expiration timestamp of the token, in epoch milliseconds.`) // TODO: complex arg: ip_access_list cmd.Flags().StringVar(&createReq.Owner, "owner", createReq.Owner, `Username of the recipient owner.`) // TODO: complex arg: properties_kvpairs @@ -311,6 +312,8 @@ func newList() *cobra.Command { // TODO: short flags cmd.Flags().StringVar(&listReq.DataRecipientGlobalMetastoreId, "data-recipient-global-metastore-id", listReq.DataRecipientGlobalMetastoreId, `If not provided, all recipients will be returned.`) + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of recipients to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "list" cmd.Short = `List share recipients.` @@ -449,6 +452,9 @@ func newSharePermissions() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&sharePermissionsReq.MaxResults, "max-results", sharePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&sharePermissionsReq.PageToken, "page-token", sharePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "share-permissions NAME" cmd.Short = `Get recipient share permissions.` cmd.Long = `Get recipient share permissions. @@ -523,6 +529,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `Description about the recipient.`) + cmd.Flags().Int64Var(&updateReq.ExpirationTime, "expiration-time", updateReq.ExpirationTime, `Expiration timestamp of the token, in epoch milliseconds.`) // TODO: complex arg: ip_access_list cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the recipient.`) cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of the recipient owner.`) diff --git a/cmd/workspace/registered-models/registered-models.go b/cmd/workspace/registered-models/registered-models.go index 08e11d686..5aa6cdf15 100755 --- a/cmd/workspace/registered-models/registered-models.go +++ b/cmd/workspace/registered-models/registered-models.go @@ -326,6 +326,7 @@ func newGet() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&getReq.IncludeAliases, "include-aliases", getReq.IncludeAliases, `Whether to include registered model aliases in the response.`) cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include registered models in the response for which the principal can only access selective metadata for.`) cmd.Use = "get FULL_NAME" diff --git a/cmd/workspace/schemas/schemas.go b/cmd/workspace/schemas/schemas.go index 710141913..3a398251f 100755 --- a/cmd/workspace/schemas/schemas.go +++ b/cmd/workspace/schemas/schemas.go @@ -147,6 +147,8 @@ func newDelete() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&deleteReq.Force, "force", deleteReq.Force, `Force deletion even if the schema is not empty.`) + cmd.Use = "delete FULL_NAME" cmd.Short = `Delete a schema.` cmd.Long = `Delete a schema. diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index c2fd779a7..67f870177 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -254,11 +254,19 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *sharing.ListSharesRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq sharing.ListSharesRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of shares to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list" cmd.Short = `List shares.` cmd.Long = `List shares. @@ -269,11 +277,17 @@ func newList() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response := w.Shares.List(ctx) + + response := w.Shares.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -283,7 +297,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd @@ -305,6 +319,9 @@ func newSharePermissions() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&sharePermissionsReq.MaxResults, "max-results", sharePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&sharePermissionsReq.PageToken, "page-token", sharePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "share-permissions NAME" cmd.Short = `Get permissions.` cmd.Long = `Get permissions. @@ -455,6 +472,8 @@ func newUpdatePermissions() *cobra.Command { cmd.Flags().Var(&updatePermissionsJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: changes + cmd.Flags().IntVar(&updatePermissionsReq.MaxResults, "max-results", updatePermissionsReq.MaxResults, `Maximum number of permissions to return.`) + cmd.Flags().StringVar(&updatePermissionsReq.PageToken, "page-token", updatePermissionsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Use = "update-permissions NAME" cmd.Short = `Update permissions.` diff --git a/cmd/workspace/system-schemas/system-schemas.go b/cmd/workspace/system-schemas/system-schemas.go index 3fe0580d7..292afbe84 100755 --- a/cmd/workspace/system-schemas/system-schemas.go +++ b/cmd/workspace/system-schemas/system-schemas.go @@ -177,6 +177,9 @@ func newList() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of schemas to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list METASTORE_ID" cmd.Short = `List system schemas.` cmd.Long = `List system schemas. diff --git a/cmd/workspace/workspace-bindings/workspace-bindings.go b/cmd/workspace/workspace-bindings/workspace-bindings.go index b7e0614ea..4993f1aff 100755 --- a/cmd/workspace/workspace-bindings/workspace-bindings.go +++ b/cmd/workspace/workspace-bindings/workspace-bindings.go @@ -3,6 +3,8 @@ package workspace_bindings import ( + "fmt" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" @@ -35,7 +37,8 @@ func New() *cobra.Command { (/api/2.1/unity-catalog/bindings/{securable_type}/{securable_name}) which introduces the ability to bind a securable in READ_ONLY mode (catalogs only). - Securables that support binding: - catalog`, + Securable types that support binding: - catalog - storage_credential - + external_location`, GroupID: "catalog", Annotations: map[string]string{ "package": "catalog", @@ -131,6 +134,9 @@ func newGetBindings() *cobra.Command { // TODO: short flags + cmd.Flags().IntVar(&getBindingsReq.MaxResults, "max-results", getBindingsReq.MaxResults, `Maximum number of workspace bindings to return.`) + cmd.Flags().StringVar(&getBindingsReq.PageToken, "page-token", getBindingsReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "get-bindings SECURABLE_TYPE SECURABLE_NAME" cmd.Short = `Get securable workspace bindings.` cmd.Long = `Get securable workspace bindings. @@ -139,7 +145,7 @@ func newGetBindings() *cobra.Command { or an owner of the securable. Arguments: - SECURABLE_TYPE: The type of the securable. + SECURABLE_TYPE: The type of the securable to bind to a workspace. SECURABLE_NAME: The name of the securable.` cmd.Annotations = make(map[string]string) @@ -154,14 +160,14 @@ func newGetBindings() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getBindingsReq.SecurableType = args[0] + _, err = fmt.Sscan(args[0], &getBindingsReq.SecurableType) + if err != nil { + return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) + } getBindingsReq.SecurableName = args[1] - response, err := w.WorkspaceBindings.GetBindings(ctx, getBindingsReq) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + response := w.WorkspaceBindings.GetBindings(ctx, getBindingsReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -275,7 +281,7 @@ func newUpdateBindings() *cobra.Command { admin or an owner of the securable. Arguments: - SECURABLE_TYPE: The type of the securable. + SECURABLE_TYPE: The type of the securable to bind to a workspace. SECURABLE_NAME: The name of the securable.` cmd.Annotations = make(map[string]string) @@ -296,7 +302,10 @@ func newUpdateBindings() *cobra.Command { return err } } - updateBindingsReq.SecurableType = args[0] + _, err = fmt.Sscan(args[0], &updateBindingsReq.SecurableType) + if err != nil { + return fmt.Errorf("invalid SECURABLE_TYPE: %s", args[0]) + } updateBindingsReq.SecurableName = args[1] response, err := w.WorkspaceBindings.UpdateBindings(ctx, updateBindingsReq) diff --git a/cmd/workspace/workspace/export_dir.go b/cmd/workspace/workspace/export_dir.go index 0b53666f9..0046f46ef 100644 --- a/cmd/workspace/workspace/export_dir.go +++ b/cmd/workspace/workspace/export_dir.go @@ -110,8 +110,7 @@ func newExportDir() *cobra.Command { } workspaceFS := filer.NewFS(ctx, workspaceFiler) - // TODO: print progress events on stderr instead: https://github.com/databricks/cli/issues/448 - err = cmdio.RenderJson(ctx, newExportStartedEvent(opts.sourceDir)) + err = cmdio.RenderWithTemplate(ctx, newExportStartedEvent(opts.sourceDir), "", "Exporting files from {{.SourcePath}}\n") if err != nil { return err } @@ -120,7 +119,7 @@ func newExportDir() *cobra.Command { if err != nil { return err } - return cmdio.RenderJson(ctx, newExportCompletedEvent(opts.targetDir)) + return cmdio.RenderWithTemplate(ctx, newExportCompletedEvent(opts.targetDir), "", "Export complete\n") } return cmd diff --git a/cmd/workspace/workspace/import_dir.go b/cmd/workspace/workspace/import_dir.go index 19d9a0a17..a197d7dd9 100644 --- a/cmd/workspace/workspace/import_dir.go +++ b/cmd/workspace/workspace/import_dir.go @@ -134,8 +134,7 @@ Notebooks will have their extensions (one of .scala, .py, .sql, .ipynb, .r) stri return err } - // TODO: print progress events on stderr instead: https://github.com/databricks/cli/issues/448 - err = cmdio.RenderJson(ctx, newImportStartedEvent(opts.sourceDir)) + err = cmdio.RenderWithTemplate(ctx, newImportStartedEvent(opts.sourceDir), "", "Importing files from {{.SourcePath}}\n") if err != nil { return err } @@ -145,7 +144,7 @@ Notebooks will have their extensions (one of .scala, .py, .sql, .ipynb, .r) stri if err != nil { return err } - return cmdio.RenderJson(ctx, newImportCompletedEvent(opts.targetDir)) + return cmdio.RenderWithTemplate(ctx, newImportCompletedEvent(opts.targetDir), "", "Import complete\n") } return cmd diff --git a/go.mod b/go.mod index 385a93b09..838a45f36 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.43.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.44.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause @@ -22,11 +22,11 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.18.0 - golang.org/x/oauth2 v0.21.0 - golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 - golang.org/x/text v0.16.0 + golang.org/x/mod v0.20.0 + golang.org/x/oauth2 v0.22.0 + golang.org/x/sync v0.8.0 + golang.org/x/term v0.23.0 + golang.org/x/text v0.17.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -59,13 +59,13 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 864b7919b..f55f329f3 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.43.0 h1:x4laolWhYlsQg2t8yWEGyRPZy4/Wv3pKnLEoJfVin7I= -github.com/databricks/databricks-sdk-go v0.43.0/go.mod h1:a9rr0FOHLL26kOjQjZZVFjIYmRABCbrAWVeundDEVG8= +github.com/databricks/databricks-sdk-go v0.44.0 h1:9/FZACv4EFQIOYxfwYVKnY7v46xio9FKCw9tpKB2O/s= +github.com/databricks/databricks-sdk-go v0.44.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -172,32 +172,32 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -208,14 +208,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -240,8 +240,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/internal/acc/workspace.go b/internal/acc/workspace.go index 8944e199f..39374f229 100644 --- a/internal/acc/workspace.go +++ b/internal/acc/workspace.go @@ -2,6 +2,7 @@ package acc import ( "context" + "os" "testing" "github.com/databricks/databricks-sdk-go" @@ -38,6 +39,33 @@ func WorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { return wt.ctx, wt } +// Run the workspace test only on UC workspaces. +func UcWorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { + loadDebugEnvIfRunFromIDE(t, "workspace") + + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skipf("Skipping on non-UC workspaces") + } + if os.Getenv("DATABRICKS_ACCOUNT_ID") != "" { + t.Skipf("Skipping on accounts") + } + + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + wt := &WorkspaceT{ + T: t, + + W: w, + + ctx: context.Background(), + } + + return wt.ctx, wt +} + func (t *WorkspaceT) TestClusterID() string { clusterID := GetEnvOrSkipTest(t.T, "TEST_BRICKS_CLUSTER_ID") err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID) diff --git a/internal/alerts_test.go b/internal/alerts_test.go index f34b404de..6d7544074 100644 --- a/internal/alerts_test.go +++ b/internal/alerts_test.go @@ -9,6 +9,6 @@ import ( func TestAccAlertsCreateErrWhenNoArguments(t *testing.T) { t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) - _, _, err := RequireErrorRun(t, "alerts", "create") + _, _, err := RequireErrorRun(t, "alerts-legacy", "create") assert.Equal(t, "please provide command input in JSON format by specifying the --json flag", err.Error()) } diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 222b23047..bae8073fc 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" "github.com/databricks/databricks-sdk-go/service/compute" @@ -74,7 +74,7 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -138,7 +138,7 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -153,3 +153,72 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) b.Config.Resources.Jobs["test"].JobSettings.Environments[0].Spec.Dependencies[0], ) } + +func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") + } + + volumePath := internal.TemporaryUcVolume(t, w) + + dir := t.TempDir() + whlPath := filepath.Join(dir, "dist", "test.whl") + touchEmptyFile(t, whlPath) + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + }, + Workspace: config.Workspace{ + ArtifactPath: volumePath, + }, + Artifacts: config.Artifacts{ + "test": &config.Artifact{ + Type: "whl", + Files: []config.ArtifactFile{ + { + Source: whlPath, + }, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: "dist/test.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) + require.NoError(t, diags.Error()) + + // The remote path attribute on the artifact file should have been set. + require.Regexp(t, + regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + b.Config.Artifacts["test"].Files[0].RemotePath, + ) + + // The task library path should have been updated to the remote path. + require.Regexp(t, + regexp.MustCompile(path.Join(regexp.QuoteMeta(volumePath), `.internal/test\.whl`)), + b.Config.Resources.Jobs["test"].JobSettings.Tasks[0].Libraries[0].Whl, + ) +} diff --git a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json index 0695eb2ba..c4a74df07 100644 --- a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json +++ b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json @@ -20,6 +20,10 @@ "python_wheel_wrapper": { "type": "boolean", "description": "Whether or not to enable python wheel wrapper" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" } } } diff --git a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl index 8729dcba5..30b0a5eae 100644 --- a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl @@ -20,6 +20,7 @@ resources: spark_version: "{{.spark_version}}" node_type_id: "{{.node_type_id}}" data_security_mode: USER_ISOLATION + instance_pool_id: "{{.instance_pool_id}}" python_wheel_task: package_name: my_test_code entry_point: run diff --git a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json new file mode 100644 index 000000000..1381da1dd --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json @@ -0,0 +1,33 @@ +{ + "properties": { + "project_name": { + "type": "string", + "default": "my_java_project", + "description": "Unique name for this project" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + }, + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "root": { + "type": "string", + "description": "Path to the root of the template" + }, + "artifact_path": { + "type": "string", + "description": "Path to the remote base path for artifacts" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" + } + } +} diff --git a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl new file mode 100644 index 000000000..db451cd93 --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl @@ -0,0 +1,55 @@ +bundle: + name: spark-jar-task + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +artifacts: + my_java_code: + path: ./{{.project_name}} + build: "javac PrintArgs.java && jar cvfm PrintArgs.jar META-INF/MANIFEST.MF PrintArgs.class" + files: + - source: ./{{.project_name}}/PrintArgs.jar + +resources: + jobs: + jar_job: + name: "[${bundle.target}] Test Spark Jar Job {{.unique_id}}" + tasks: + - task_key: TestSparkJarTask + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + instance_pool_id: "{{.instance_pool_id}}" + spark_jar_task: + main_class_name: PrintArgs + libraries: + - jar: ./{{.project_name}}/PrintArgs.jar + +targets: + volume: + # Override the artifact path to upload artifacts to a volume path + workspace: + artifact_path: {{.artifact_path}} + + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in single user mode (force it to be a UC cluster) + data_security_mode: SINGLE_USER + + workspace: + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in no isolation mode (force it to be a non-UC cluster) + data_security_mode: NONE diff --git a/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF new file mode 100644 index 000000000..40b023dbd --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Main-Class: PrintArgs \ No newline at end of file diff --git a/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java new file mode 100644 index 000000000..b7430f25f --- /dev/null +++ b/internal/bundle/bundles/spark_jar_task/template/{{.project_name}}/PrintArgs.java @@ -0,0 +1,8 @@ +import java.util.Arrays; + +public class PrintArgs { + public static void main(String[] args) { + System.out.println("Hello from Jar!"); + System.out.println(Arrays.toString(args)); + } +} diff --git a/internal/bundle/bundles/uc_schema/databricks_template_schema.json b/internal/bundle/bundles/uc_schema/databricks_template_schema.json new file mode 100644 index 000000000..762f4470c --- /dev/null +++ b/internal/bundle/bundles/uc_schema/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema and pipeline names" + } + } +} diff --git a/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl new file mode 100644 index 000000000..961af25e8 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: "bundle-playground" + +resources: + pipelines: + foo: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + +include: + - "*.yml" + +targets: + development: + default: true diff --git a/internal/bundle/bundles/uc_schema/template/nb.sql b/internal/bundle/bundles/uc_schema/template/nb.sql new file mode 100644 index 000000000..199ff5078 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl new file mode 100644 index 000000000..50067036e --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl @@ -0,0 +1,13 @@ +resources: + schemas: + bar: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created from DABs + +targets: + development: + resources: + pipelines: + foo: + target: ${resources.schemas.bar.id} diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go new file mode 100644 index 000000000..269b7c80a --- /dev/null +++ b/internal/bundle/deploy_test.go @@ -0,0 +1,156 @@ +package bundle + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/files" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupUcSchemaBundle(t *testing.T, ctx context.Context, w *databricks.WorkspaceClient, uniqueId string) string { + bundleRoot, err := initTestTemplate(t, ctx, "uc_schema", map[string]any{ + "unique_id": uniqueId, + }) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + // Assert the schema is created + catalogName := "main" + schemaName := "test-schema-" + uniqueId + schema, err := w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + require.NoError(t, err) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), schema.FullName) + require.Equal(t, "This schema was created from DABs", schema.Comment) + + // Assert the pipeline is created + pipelineName := "test-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + require.Equal(t, pipelineName, pipeline.Name) + id := pipeline.PipelineId + + // Assert the pipeline uses the schema + i, err := w.Pipelines.GetByPipelineId(ctx, id) + require.NoError(t, err) + require.Equal(t, catalogName, i.Spec.Catalog) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), i.Spec.Target) + + // Create a volume in the schema, and add a file to it. This ensures that the + // schema has some data in it and deletion will fail unless the generated + // terraform configuration has force_destroy set to true. + volumeName := "test-volume-" + uniqueId + volume, err := w.Volumes.Create(ctx, catalog.CreateVolumeRequestContent{ + CatalogName: catalogName, + SchemaName: schemaName, + Name: volumeName, + VolumeType: catalog.VolumeTypeManaged, + }) + require.NoError(t, err) + require.Equal(t, volume.Name, volumeName) + + fileName := "test-file-" + uniqueId + err = w.Files.Upload(ctx, files.UploadRequest{ + Contents: io.NopCloser(strings.NewReader("Hello, world!")), + FilePath: fmt.Sprintf("/Volumes/%s/%s/%s/%s", catalogName, schemaName, volumeName, fileName), + }) + require.NoError(t, err) + + return bundleRoot +} + +func TestAccBundleDeployUcSchema(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + schemaName := "test-schema-" + uniqueId + catalogName := "main" + + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + // Assert the schema is deleted + _, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + apiErr := &apierr.APIError{} + assert.True(t, errors.As(err, &apiErr)) + assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode) +} + +func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + stdout, _, err := c.Run() + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") +} + +func TestAccDeployBasicBundleLogs(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + root, err := initTestTemplate(t, ctx, "basic", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + t.Cleanup(func() { + err = destroyBundle(t, ctx, root) + require.NoError(t, err) + }) + + currentUser, err := wt.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + stdout, stderr := blackBoxRun(t, root, "bundle", "deploy") + assert.Equal(t, strings.Join([]string{ + fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), + "Deploying resources...", + "Updating deployment state...", + "Deployment complete!\n", + }, "\n"), stderr) + assert.Equal(t, "", stdout) +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index a17964b16..3547c1755 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -1,10 +1,12 @@ package bundle import ( + "bytes" "context" "encoding/json" "fmt" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -12,8 +14,10 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/template" + "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/require" ) @@ -21,9 +25,13 @@ import ( const defaultSparkVersion = "13.3.x-snapshot-scala2.12" func initTestTemplate(t *testing.T, ctx context.Context, templateName string, config map[string]any) (string, error) { + bundleRoot := t.TempDir() + return initTestTemplateWithBundleRoot(t, ctx, templateName, config, bundleRoot) +} + +func initTestTemplateWithBundleRoot(t *testing.T, ctx context.Context, templateName string, config map[string]any, bundleRoot string) (string, error) { templateRoot := filepath.Join("bundles", templateName) - bundleRoot := t.TempDir() configFilePath, err := writeConfigFile(t, config) if err != nil { return "", err @@ -52,21 +60,21 @@ func writeConfigFile(t *testing.T, config map[string]any) (string, error) { } func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, error) { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "validate", "--output", "json") stdout, _, err := c.Run() return stdout.Bytes(), err } func deployBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) - c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + ctx = env.Set(ctx, "BUNDLE_ROOT", path) + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve") _, _, err := c.Run() return err } func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) args := []string{"bundle", "deploy", "--force-lock"} args = append(args, flags...) c := internal.NewCobraTestRunnerWithContext(t, ctx, args...) @@ -75,6 +83,7 @@ func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags } func runResource(t *testing.T, ctx context.Context, path string, key string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "run", key) @@ -83,6 +92,7 @@ func runResource(t *testing.T, ctx context.Context, path string, key string) (st } func runResourceWithParams(t *testing.T, ctx context.Context, path string, key string, params ...string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) args := make([]string, 0) @@ -94,7 +104,7 @@ func runResourceWithParams(t *testing.T, ctx context.Context, path string, key s } func destroyBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "destroy", "--auto-approve") _, _, err := c.Run() return err @@ -107,3 +117,29 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId) return root } + +func blackBoxRun(t *testing.T, root string, args ...string) (stdout string, stderr string) { + cwd := vfs.MustNew(".") + gitRoot, err := vfs.FindLeafInTree(cwd, ".git") + require.NoError(t, err) + + t.Setenv("BUNDLE_ROOT", root) + + // Create the command + cmd := exec.Command("go", append([]string{"run", "main.go"}, args...)...) + cmd.Dir = gitRoot.Native() + + // Create buffers to capture output + var outBuffer, errBuffer bytes.Buffer + cmd.Stdout = &outBuffer + cmd.Stderr = &errBuffer + + // Run the command + err = cmd.Run() + require.NoError(t, err) + + // Get the output + stdout = outBuffer.String() + stderr = errBuffer.String() + return +} diff --git a/internal/bundle/python_wheel_test.go b/internal/bundle/python_wheel_test.go index bf2462920..ed98efecd 100644 --- a/internal/bundle/python_wheel_test.go +++ b/internal/bundle/python_wheel_test.go @@ -14,11 +14,13 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo ctx, _ := acc.WorkspaceTest(t) nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{ "node_type_id": nodeTypeId, "unique_id": uuid.New().String(), "spark_version": sparkVersion, "python_wheel_wrapper": pythonWheelWrapper, + "instance_pool_id": instancePoolId, }) require.NoError(t, err) diff --git a/internal/bundle/spark_jar_test.go b/internal/bundle/spark_jar_test.go new file mode 100644 index 000000000..4b469617c --- /dev/null +++ b/internal/bundle/spark_jar_test.go @@ -0,0 +1,100 @@ +package bundle + +import ( + "context" + "testing" + + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/env" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func runSparkJarTestCommon(t *testing.T, ctx context.Context, sparkVersion string, artifactPath string) { + cloudEnv := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + nodeTypeId := internal.GetNodeTypeId(cloudEnv) + tmpDir := t.TempDir() + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") + bundleRoot, err := initTestTemplateWithBundleRoot(t, ctx, "spark_jar_task", map[string]any{ + "node_type_id": nodeTypeId, + "unique_id": uuid.New().String(), + "spark_version": sparkVersion, + "root": tmpDir, + "artifact_path": artifactPath, + "instance_pool_id": instancePoolId, + }, tmpDir) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + out, err := runResource(t, ctx, bundleRoot, "jar_job") + require.NoError(t, err) + require.Contains(t, out, "Hello from Jar!") +} + +func runSparkJarTestFromVolume(t *testing.T, sparkVersion string) { + ctx, wt := acc.UcWorkspaceTest(t) + volumePath := internal.TemporaryUcVolume(t, wt.W) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "volume") + runSparkJarTestCommon(t, ctx, sparkVersion, volumePath) +} + +func runSparkJarTestFromWorkspace(t *testing.T, sparkVersion string) { + ctx, _ := acc.WorkspaceTest(t) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "workspace") + runSparkJarTestCommon(t, ctx, sparkVersion, "n/a") +} + +func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // JAR installation from Volumes is supported on UC Clusters with DBR >= 13.3. + // Denied library is Jar(/Volumes/main/test-schema-ldgaklhcahlg/my-volume/.internal/PrintArgs.jar) + // + + versions := []string{ + "13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12) + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromVolume(t, version) + }) + } +} + +func TestAccSparkJarTaskDeployAndRunOnWorkspace(t *testing.T) { + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // Library from /Workspace is not allowed on this cluster. + // Please switch to using DBR 14.1+ No Isolation Shared or DBR 13.1+ Shared cluster or 13.2+ Assigned cluster to use /Workspace libraries. + // + + versions := []string{ + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromWorkspace(t, version) + }) + } +} diff --git a/internal/completer_test.go b/internal/completer_test.go new file mode 100644 index 000000000..b2c936886 --- /dev/null +++ b/internal/completer_test.go @@ -0,0 +1,27 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "testing" + + _ "github.com/databricks/cli/cmd/fs" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupCompletionFile(t *testing.T, f filer.Filer) { + err := f.Write(context.Background(), "dir1/file1.txt", strings.NewReader("abc"), filer.CreateParentDirectories) + require.NoError(t, err) +} + +func TestAccFsCompletion(t *testing.T) { + f, tmpDir := setupDbfsFiler(t) + setupCompletionFile(t, f) + + stdout, _ := RequireSuccessfulRun(t, "__complete", "fs", "ls", tmpDir+"/") + expectedOutput := fmt.Sprintf("%s/dir1/\n:2\n", tmpDir) + assert.Equal(t, expectedOutput, stdout.String()) +} diff --git a/internal/filer_test.go b/internal/filer_test.go index 275304256..bc4c94808 100644 --- a/internal/filer_test.go +++ b/internal/filer_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "io/fs" "path" @@ -722,67 +721,6 @@ func TestAccFilerWorkspaceFilesExtensionsStat(t *testing.T) { assert.ErrorIs(t, err, fs.ErrNotExist) } -func TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) { - t.Parallel() - - tcases := []struct { - files []struct{ name, content string } - name string - }{ - { - name: "python", - files: []struct{ name, content string }{ - {"foo.py", "print('foo')"}, - {"foo.py", "# Databricks notebook source\nprint('foo')"}, - }, - }, - { - name: "r", - files: []struct{ name, content string }{ - {"foo.r", "print('foo')"}, - {"foo.r", "# Databricks notebook source\nprint('foo')"}, - }, - }, - { - name: "sql", - files: []struct{ name, content string }{ - {"foo.sql", "SELECT 'foo'"}, - {"foo.sql", "-- Databricks notebook source\nSELECT 'foo'"}, - }, - }, - { - name: "scala", - files: []struct{ name, content string }{ - {"foo.scala", "println('foo')"}, - {"foo.scala", "// Databricks notebook source\nprintln('foo')"}, - }, - }, - // We don't need to test this for ipynb notebooks. The import API - // fails when the file extension is .ipynb but the content is not a - // valid juptyer notebook. - } - - for i := range tcases { - tc := tcases[i] - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - ctx := context.Background() - wf, tmpDir := setupWsfsExtensionsFiler(t) - - for _, f := range tc.files { - err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories) - require.NoError(t, err) - } - - _, err := wf.ReadDir(ctx, ".") - assert.ErrorAs(t, err, &filer.DuplicatePathError{}) - assert.ErrorContains(t, err, fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at %s and FILE at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", path.Join(tmpDir, "foo"), path.Join(tmpDir, tc.files[0].name), tc.files[0].name)) - }) - } - -} - func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) { t.Parallel() diff --git a/internal/helpers.go b/internal/helpers.go index 3923e7e1e..419fa419c 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -19,6 +19,9 @@ import ( "testing" "time" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/cmd" _ "github.com/databricks/cli/cmd/version" "github.com/databricks/cli/libs/cmdio" @@ -84,17 +87,23 @@ type cobraTestRunner struct { } func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan string { - ch := make(chan string, 1000) + ch := make(chan string, 30000) wg.Add(1) go func() { defer close(ch) defer wg.Done() scanner := bufio.NewScanner(r) for scanner.Scan() { + // We expect to be able to always send these lines into the channel. + // If we can't, it means the channel is full and likely there is a problem + // in either the test or the code under test. select { case <-ctx.Done(): return case ch <- scanner.Text(): + continue + default: + panic("line buffer is full") } } }() @@ -105,7 +114,12 @@ func (t *cobraTestRunner) registerFlagCleanup(c *cobra.Command) { // Find target command that will be run. Example: if the command run is `databricks fs cp`, // target command corresponds to `cp` targetCmd, _, err := c.Find(t.args) - require.NoError(t, err) + if err != nil && strings.HasPrefix(err.Error(), "unknown command") { + // even if command is unknown, we can proceed + require.NotNil(t, targetCmd) + } else { + require.NoError(t, err) + } // Force initialization of default flags. // These are initialized by cobra at execution time and would otherwise @@ -169,22 +183,28 @@ func (t *cobraTestRunner) RunBackground() { var stdoutW, stderrW io.WriteCloser stdoutR, stdoutW = io.Pipe() stderrR, stderrW = io.Pipe() - root := cmd.New(t.ctx) - root.SetOut(stdoutW) - root.SetErr(stderrW) - root.SetArgs(t.args) + ctx := cmdio.NewContext(t.ctx, &cmdio.Logger{ + Mode: flags.ModeAppend, + Reader: bufio.Reader{}, + Writer: stderrW, + }) + + cli := cmd.New(ctx) + cli.SetOut(stdoutW) + cli.SetErr(stderrW) + cli.SetArgs(t.args) if t.stdinW != nil { - root.SetIn(t.stdinR) + cli.SetIn(t.stdinR) } // Register cleanup function to restore flags to their original values // once test has been executed. This is needed because flag values reside // in a global singleton data-structure, and thus subsequent tests might // otherwise interfere with each other - t.registerFlagCleanup(root) + t.registerFlagCleanup(cli) errch := make(chan error) - ctx, cancel := context.WithCancel(t.ctx) + ctx, cancel := context.WithCancel(ctx) // Tee stdout/stderr to buffers. stdoutR = io.TeeReader(stdoutR, &t.stdout) @@ -197,7 +217,7 @@ func (t *cobraTestRunner) RunBackground() { // Run command in background. go func() { - cmd, err := root.ExecuteContextC(ctx) + err := root.Execute(ctx, cli) if err != nil { t.Logf("Error running command: %s", err) } @@ -230,7 +250,7 @@ func (t *cobraTestRunner) RunBackground() { // These commands are globals so we have to clean up to the best of our ability after each run. // See https://github.com/spf13/cobra/blob/a6f198b635c4b18fff81930c40d464904e55b161/command.go#L1062-L1066 //lint:ignore SA1012 cobra sets the context and doesn't clear it - cmd.SetContext(nil) + cli.SetContext(nil) // Make caller aware of error. errch <- err @@ -458,7 +478,7 @@ func TemporaryDbfsDir(t *testing.T, w *databricks.WorkspaceClient) string { } // Create a new UC volume in a catalog called "main" in the workspace. -func temporaryUcVolume(t *testing.T, w *databricks.WorkspaceClient) string { +func TemporaryUcVolume(t *testing.T, w *databricks.WorkspaceClient) string { ctx := context.Background() // Create a schema @@ -593,7 +613,7 @@ func setupUcVolumesFiler(t *testing.T) (filer.Filer, string) { w, err := databricks.NewWorkspaceClient() require.NoError(t, err) - tmpDir := temporaryUcVolume(t, w) + tmpDir := TemporaryUcVolume(t, w) f, err := filer.NewFilesClient(w, tmpDir) require.NoError(t, err) diff --git a/internal/testutil/copy.go b/internal/testutil/copy.go new file mode 100644 index 000000000..21faece00 --- /dev/null +++ b/internal/testutil/copy.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// CopyDirectory copies the contents of a directory to another directory. +// The destination directory is created if it does not exist. +func CopyDirectory(t *testing.T, src, dst string) { + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(src, path) + require.NoError(t, err) + + if d.IsDir() { + return os.MkdirAll(filepath.Join(dst, rel), 0755) + } + + // Copy the file to the temporary directory + in, err := os.Open(path) + if err != nil { + return err + } + + defer in.Close() + + out, err := os.Create(filepath.Join(dst, rel)) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, in) + return err + }) + + require.NoError(t, err) +} diff --git a/internal/testutil/jdk.go b/internal/testutil/jdk.go new file mode 100644 index 000000000..05bd7d6d6 --- /dev/null +++ b/internal/testutil/jdk.go @@ -0,0 +1,24 @@ +package testutil + +import ( + "bytes" + "context" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func RequireJDK(t *testing.T, ctx context.Context, version string) { + var stderr bytes.Buffer + + cmd := exec.Command("javac", "-version") + cmd.Stderr = &stderr + err := cmd.Run() + require.NoError(t, err, "Unable to run javac -version") + + // Get the first line of the output + line := strings.Split(stderr.String(), "\n")[0] + require.Contains(t, line, version, "Expected JDK version %s, got %s", version, line) +} diff --git a/internal/unknown_command_test.go b/internal/unknown_command_test.go new file mode 100644 index 000000000..62b84027f --- /dev/null +++ b/internal/unknown_command_test.go @@ -0,0 +1,15 @@ +package internal + +import ( + "testing" + + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestUnknownCommand(t *testing.T) { + stdout, stderr, err := RequireErrorRun(t, "unknown-command") + + assert.Error(t, err, "unknown command", `unknown command "unknown-command" for "databricks"`) + assert.Equal(t, "", stdout.String()) + assert.Contains(t, stderr.String(), "unknown command") +} diff --git a/internal/workspace_test.go b/internal/workspace_test.go index bc354914f..445361654 100644 --- a/internal/workspace_test.go +++ b/internal/workspace_test.go @@ -3,18 +3,17 @@ package internal import ( "context" "encoding/base64" - "errors" + "fmt" "io" - "net/http" "os" "path" "path/filepath" "strings" "testing" + "github.com/databricks/cli/internal/acc" "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -63,21 +62,12 @@ func TestAccWorkpaceExportPrintsContents(t *testing.T) { } func setupWorkspaceImportExportTest(t *testing.T) (context.Context, filer.Filer, string) { - t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + ctx, wt := acc.WorkspaceTest(t) - ctx := context.Background() - w := databricks.Must(databricks.NewWorkspaceClient()) - tmpdir := TemporaryWorkspaceDir(t, w) - f, err := filer.NewWorkspaceFilesClient(w, tmpdir) + tmpdir := TemporaryWorkspaceDir(t, wt.W) + f, err := filer.NewWorkspaceFilesClient(wt.W, tmpdir) require.NoError(t, err) - // Check if we can use this API here, skip test if we cannot. - _, err = f.Read(ctx, "we_use_this_call_to_test_if_this_api_is_enabled") - var aerr *apierr.APIError - if errors.As(err, &aerr) && aerr.StatusCode == http.StatusBadRequest { - t.Skip(aerr.Message) - } - return ctx, f, tmpdir } @@ -122,8 +112,21 @@ func TestAccExportDir(t *testing.T) { err = f.Write(ctx, "a/b/c/file-b", strings.NewReader("def"), filer.CreateParentDirectories) require.NoError(t, err) + expectedLogs := strings.Join([]string{ + fmt.Sprintf("Exporting files from %s", sourceDir), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "a/b/c/file-b"), filepath.Join(targetDir, "a/b/c/file-b")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "file-a"), filepath.Join(targetDir, "file-a")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "pyNotebook"), filepath.Join(targetDir, "pyNotebook.py")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "rNotebook"), filepath.Join(targetDir, "rNotebook.r")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "scalaNotebook"), filepath.Join(targetDir, "scalaNotebook.scala")), + fmt.Sprintf("%s -> %s", path.Join(sourceDir, "sqlNotebook"), filepath.Join(targetDir, "sqlNotebook.sql")), + "Export complete\n", + }, "\n") + // Run Export - RequireSuccessfulRun(t, "workspace", "export-dir", sourceDir, targetDir) + stdout, stderr := RequireSuccessfulRun(t, "workspace", "export-dir", sourceDir, targetDir) + assert.Equal(t, expectedLogs, stdout.String()) + assert.Equal(t, "", stderr.String()) // Assert files were exported assertLocalFileContents(t, filepath.Join(targetDir, "file-a"), "abc") @@ -176,10 +179,24 @@ func TestAccExportDirWithOverwriteFlag(t *testing.T) { assertLocalFileContents(t, filepath.Join(targetDir, "file-a"), "content from workspace") } -// TODO: Add assertions on progress logs for workspace import-dir command. https://github.com/databricks/cli/issues/455 func TestAccImportDir(t *testing.T) { ctx, workspaceFiler, targetDir := setupWorkspaceImportExportTest(t) - RequireSuccessfulRun(t, "workspace", "import-dir", "./testdata/import_dir", targetDir, "--log-level=debug") + stdout, stderr := RequireSuccessfulRun(t, "workspace", "import-dir", "./testdata/import_dir", targetDir, "--log-level=debug") + + expectedLogs := strings.Join([]string{ + fmt.Sprintf("Importing files from %s", "./testdata/import_dir"), + fmt.Sprintf("%s -> %s", filepath.FromSlash("a/b/c/file-b"), path.Join(targetDir, "a/b/c/file-b")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("file-a"), path.Join(targetDir, "file-a")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("jupyterNotebook.ipynb"), path.Join(targetDir, "jupyterNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("pyNotebook.py"), path.Join(targetDir, "pyNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("rNotebook.r"), path.Join(targetDir, "rNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("scalaNotebook.scala"), path.Join(targetDir, "scalaNotebook")), + fmt.Sprintf("%s -> %s", filepath.FromSlash("sqlNotebook.sql"), path.Join(targetDir, "sqlNotebook")), + "Import complete\n", + }, "\n") + + assert.Equal(t, expectedLogs, stdout.String()) + assert.Equal(t, "", stderr.String()) // Assert files are imported assertFilerFileContents(t, ctx, workspaceFiler, "file-a", "hello, world") diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go index 1f3e032de..7c1cb9576 100644 --- a/libs/auth/oauth.go +++ b/libs/auth/oauth.go @@ -105,7 +105,6 @@ func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) { } func (a *PersistentAuth) ProfileName() string { - // TODO: get profile name from interactive input if a.AccountID != "" { return fmt.Sprintf("ACCOUNT-%s", a.AccountID) } diff --git a/libs/auth/user_test.go b/libs/auth/user_test.go index eb579fc98..62b2d29ac 100644 --- a/libs/auth/user_test.go +++ b/libs/auth/user_test.go @@ -22,7 +22,7 @@ func TestGetShortUserName(t *testing.T) { }, { email: "test$.user@example.com", - expected: "test__user", + expected: "test_user", }, { email: `jöhn.dÅ“@domain.com`, // Using non-ASCII characters. @@ -38,7 +38,7 @@ func TestGetShortUserName(t *testing.T) { }, { email: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. - expected: "__quoted_", + expected: "quoted", }, { email: `name-o'mally@website.org`, // Single quote in the local-part. diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index ec851b8ff..4114db5ca 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -280,14 +280,6 @@ func RenderIteratorWithTemplate[T any](ctx context.Context, i listing.Iterator[T return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, headerTemplate, template) } -func RenderJson(ctx context.Context, v any) error { - c := fromContext(ctx) - if _, ok := v.(listingInterface); ok { - panic("use RenderIteratorJson instead") - } - return renderWithTemplate(newRenderer(v), ctx, flags.OutputJSON, c.out, c.headerTemplate, c.template) -} - func RenderIteratorJson[T any](ctx context.Context, i listing.Iterator[T]) error { c := fromContext(ctx) return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, c.headerTemplate, c.template) diff --git a/libs/databrickscfg/cfgpickers/clusters.go b/libs/databrickscfg/cfgpickers/clusters.go index d955be35b..cac1b08a7 100644 --- a/libs/databrickscfg/cfgpickers/clusters.go +++ b/libs/databrickscfg/cfgpickers/clusters.go @@ -134,9 +134,7 @@ func loadInteractiveClusters(ctx context.Context, w *databricks.WorkspaceClient, promptSpinner := cmdio.Spinner(ctx) promptSpinner <- "Loading list of clusters to select from" defer close(promptSpinner) - all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{ - CanUseClient: "NOTEBOOKS", - }) + all, err := w.Clusters.ListAll(ctx, compute.ListClustersRequest{}) if err != nil { return nil, fmt.Errorf("list clusters: %w", err) } diff --git a/libs/databrickscfg/cfgpickers/clusters_test.go b/libs/databrickscfg/cfgpickers/clusters_test.go index 2e62f93a8..d17e86d4a 100644 --- a/libs/databrickscfg/cfgpickers/clusters_test.go +++ b/libs/databrickscfg/cfgpickers/clusters_test.go @@ -70,7 +70,7 @@ func TestFirstCompatibleCluster(t *testing.T) { cfg, server := qa.HTTPFixtures{ { Method: "GET", - Resource: "/api/2.0/clusters/list?can_use_client=NOTEBOOKS", + Resource: "/api/2.1/clusters/list?", Response: compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -100,7 +100,7 @@ func TestFirstCompatibleCluster(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/clusters/spark-versions", + Resource: "/api/2.1/clusters/spark-versions", Response: compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { @@ -125,7 +125,7 @@ func TestNoCompatibleClusters(t *testing.T) { cfg, server := qa.HTTPFixtures{ { Method: "GET", - Resource: "/api/2.0/clusters/list?can_use_client=NOTEBOOKS", + Resource: "/api/2.1/clusters/list?", Response: compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ { @@ -147,7 +147,7 @@ func TestNoCompatibleClusters(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/clusters/spark-versions", + Resource: "/api/2.1/clusters/spark-versions", Response: compute.GetSparkVersionsResponse{ Versions: []compute.SparkVersion{ { diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index f2d72b72f..5c3e4ea78 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -17,13 +17,13 @@ type Diagnostic struct { // This may be multiple lines and may be nil. Detail string - // Location is a source code location associated with the diagnostic message. - // It may be zero if there is no associated location. - Location dyn.Location + // Locations are the source code locations associated with the diagnostic message. + // It may be empty if there are no associated locations. + Locations []dyn.Location - // Path is a path to the value in a configuration tree that the diagnostic is associated with. - // It may be nil if there is no associated path. - Path dyn.Path + // Paths are paths to the values in the configuration tree that the diagnostic is associated with. + // It may be nil if there are no associated paths. + Paths []dyn.Path // A diagnostic ID. Only used for select diagnostic messages. ID ID diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index e8d321f66..cd92ad0eb 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -42,7 +42,7 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, // Dereference pointer if necessary for srcv.Kind() == reflect.Pointer { if srcv.IsNil() { - return dyn.NilValue.WithLocation(ref.Location()), nil + return dyn.NilValue.WithLocations(ref.Locations()), nil } srcv = srcv.Elem() @@ -83,7 +83,7 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, if err != nil { return dyn.InvalidValue, err } - return v.WithLocation(ref.Location()), err + return v.WithLocations(ref.Locations()), err } func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) { diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go index 9141a6948..0cddff3be 100644 --- a/libs/dyn/convert/from_typed_test.go +++ b/libs/dyn/convert/from_typed_test.go @@ -115,16 +115,16 @@ func TestFromTypedStructSetFieldsRetainLocation(t *testing.T) { } ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", dyn.Location{File: "foo"}), - "bar": dyn.NewValue("baz", dyn.Location{File: "bar"}), + "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), + "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their location. - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", dyn.Location{File: "bar"}), nv.Get("bar")) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) + assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) } func TestFromTypedStringMapWithZeroValue(t *testing.T) { @@ -359,16 +359,16 @@ func TestFromTypedMapNonEmptyRetainLocation(t *testing.T) { } ref := dyn.V(map[string]dyn.Value{ - "foo": dyn.NewValue("bar", dyn.Location{File: "foo"}), - "bar": dyn.NewValue("baz", dyn.Location{File: "bar"}), + "foo": dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), + "bar": dyn.NewValue("baz", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv.Get("foo")) - assert.Equal(t, dyn.NewValue("qux", dyn.Location{File: "bar"}), nv.Get("bar")) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv.Get("foo")) + assert.Equal(t, dyn.NewValue("qux", []dyn.Location{{File: "bar"}}), nv.Get("bar")) } func TestFromTypedMapFieldWithZeroValue(t *testing.T) { @@ -432,16 +432,16 @@ func TestFromTypedSliceNonEmptyRetainLocation(t *testing.T) { } ref := dyn.V([]dyn.Value{ - dyn.NewValue("foo", dyn.Location{File: "foo"}), - dyn.NewValue("bar", dyn.Location{File: "bar"}), + dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), + dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), }) nv, err := FromTyped(src, ref) require.NoError(t, err) // Assert foo and bar have retained their locations. - assert.Equal(t, dyn.NewValue("foo", dyn.Location{File: "foo"}), nv.Index(0)) - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "bar"}), nv.Index(1)) + assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv.Index(0)) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "bar"}}), nv.Index(1)) } func TestFromTypedStringEmpty(t *testing.T) { @@ -477,19 +477,19 @@ func TestFromTypedStringNonEmptyOverwrite(t *testing.T) { } func TestFromTypedStringRetainsLocations(t *testing.T) { - var ref = dyn.NewValue("foo", dyn.Location{File: "foo"}) + var ref = dyn.NewValue("foo", []dyn.Location{{File: "foo"}}) // case: value has not been changed var src string = "foo" nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue("foo", dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue("foo", []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = "bar" nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue("bar", dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue("bar", []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedStringTypeError(t *testing.T) { @@ -532,19 +532,19 @@ func TestFromTypedBoolNonEmptyOverwrite(t *testing.T) { } func TestFromTypedBoolRetainsLocations(t *testing.T) { - var ref = dyn.NewValue(true, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(true, []dyn.Location{{File: "foo"}}) // case: value has not been changed var src bool = true nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(true, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(true, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = false nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(false, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedBoolVariableReference(t *testing.T) { @@ -595,19 +595,19 @@ func TestFromTypedIntNonEmptyOverwrite(t *testing.T) { } func TestFromTypedIntRetainsLocations(t *testing.T) { - var ref = dyn.NewValue(1234, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(1234, []dyn.Location{{File: "foo"}}) // case: value has not been changed var src int = 1234 nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1234, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1234, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = 1235 nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(int64(1235), dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(int64(1235), []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedIntVariableReference(t *testing.T) { @@ -659,19 +659,19 @@ func TestFromTypedFloatNonEmptyOverwrite(t *testing.T) { func TestFromTypedFloatRetainsLocations(t *testing.T) { var src float64 - var ref = dyn.NewValue(1.23, dyn.Location{File: "foo"}) + var ref = dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}) // case: value has not been changed src = 1.23 nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.23, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1.23, []dyn.Location{{File: "foo"}}), nv) // case: value has been changed src = 1.24 nv, err = FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(1.24, dyn.Location{File: "foo"}), nv) + assert.Equal(t, dyn.NewValue(1.24, []dyn.Location{{File: "foo"}}), nv) } func TestFromTypedFloatVariableReference(t *testing.T) { @@ -740,27 +740,27 @@ func TestFromTypedNilPointerRetainsLocations(t *testing.T) { } var src *Tmp - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } func TestFromTypedNilMapRetainsLocation(t *testing.T) { var src map[string]string - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } func TestFromTypedNilSliceRetainsLocation(t *testing.T) { var src []string - ref := dyn.NewValue(nil, dyn.Location{File: "foobar"}) + ref := dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}) nv, err := FromTyped(src, ref) require.NoError(t, err) - assert.Equal(t, dyn.NewValue(nil, dyn.Location{File: "foobar"}), nv) + assert.Equal(t, dyn.NewValue(nil, []dyn.Location{{File: "foobar"}}), nv) } diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index ad82e20ef..c80a914f1 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -65,19 +65,19 @@ func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen [] func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected a %s value, found null", expected), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, } } func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, } } @@ -98,8 +98,9 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), - Location: pk.Location(), - Path: path, + // Show all locations the unknown field is defined at. + Locations: pk.Locations(), + Paths: []dyn.Path{path}, }) } continue @@ -120,7 +121,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen // Return the normalized value if missing fields are not included. if !n.includeMissingFields { - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } // Populate missing fields with their zero values. @@ -165,7 +166,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen } } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -203,7 +204,7 @@ func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []r out.Set(pk, nv) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -238,7 +239,7 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [ out = append(out, v) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags case dyn.KindNil: return src, diags @@ -273,7 +274,7 @@ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindString, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -306,7 +307,7 @@ func (n normalizeOptions) normalizeBool(typ reflect.Type, src dyn.Value, path dy return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindBool, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -320,10 +321,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn out = int64(src.MustFloat()) if src.MustFloat() != float64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -336,10 +337,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: @@ -349,7 +350,7 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindInt, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { @@ -363,10 +364,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d out = float64(src.MustInt()) if src.MustInt() != int64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -379,10 +380,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: @@ -392,7 +393,7 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindFloat, src, path)) } - return dyn.NewValue(out, src.Location()), diags + return dyn.NewValue(out, src.Locations()), diags } func (n normalizeOptions) normalizeInterface(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 299ffcabd..c2256615e 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -40,10 +40,10 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -58,23 +58,33 @@ func TestNormalizeStructUnknownField(t *testing.T) { } var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) + + m := dyn.NewMapping() + m.Set(dyn.V("foo"), dyn.V("val-foo")) + // Set the unknown field, with location information. + m.Set(dyn.NewValue("bar", []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }), dyn.V("var-bar")) + + vin := dyn.V(m) vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ Severity: diag.Warning, Summary: `unknown field: bar`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + // Assert location of the unknown field is included in the diagnostic. + Locations: []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) // The field that can be mapped to the struct field is retained. assert.Equal(t, map[string]any{ - "foo": "bar", + "foo": "val-foo", }, vout.AsAny()) } @@ -100,10 +110,10 @@ func TestNormalizeStructError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Get("foo").Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -229,7 +239,7 @@ func TestNormalizeStructVariableReference(t *testing.T) { } var typ Tmp - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -241,14 +251,14 @@ func TestNormalizeStructRandomStringError(t *testing.T) { } var typ Tmp - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -258,14 +268,14 @@ func TestNormalizeStructIntError(t *testing.T) { } var typ Tmp - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -291,10 +301,10 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -317,10 +327,10 @@ func TestNormalizeMapError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -360,7 +370,7 @@ func TestNormalizeMapNestedError(t *testing.T) { func TestNormalizeMapVariableReference(t *testing.T) { var typ map[string]string - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -368,27 +378,27 @@ func TestNormalizeMapVariableReference(t *testing.T) { func TestNormalizeMapRandomStringError(t *testing.T) { var typ map[string]string - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } func TestNormalizeMapIntError(t *testing.T) { var typ map[string]string - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -415,10 +425,10 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Index(2)), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Index(2))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -439,10 +449,10 @@ func TestNormalizeSliceError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -482,7 +492,7 @@ func TestNormalizeSliceNestedError(t *testing.T) { func TestNormalizeSliceVariableReference(t *testing.T) { var typ []string - vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("${var.foo}", []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(typ, vin) assert.Empty(t, err) assert.Equal(t, vin, vout) @@ -490,27 +500,27 @@ func TestNormalizeSliceVariableReference(t *testing.T) { func TestNormalizeSliceRandomStringError(t *testing.T) { var typ []string - vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue("var foo", []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } func TestNormalizeSliceIntError(t *testing.T) { var typ []string - vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -524,39 +534,39 @@ func TestNormalizeString(t *testing.T) { func TestNormalizeStringNil(t *testing.T) { var typ string - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a string value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } func TestNormalizeStringFromBool(t *testing.T) { var typ string - vin := dyn.NewValue(true, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(true, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("true", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("true", vin.Locations()), vout) } func TestNormalizeStringFromInt(t *testing.T) { var typ string - vin := dyn.NewValue(123, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(123, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("123", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("123", vin.Locations()), vout) } func TestNormalizeStringFromFloat(t *testing.T) { var typ string - vin := dyn.NewValue(1.20, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(1.20, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Empty(t, err) - assert.Equal(t, dyn.NewValue("1.2", vin.Location()), vout) + assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) } func TestNormalizeStringError(t *testing.T) { @@ -565,10 +575,10 @@ func TestNormalizeStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -582,14 +592,14 @@ func TestNormalizeBool(t *testing.T) { func TestNormalizeBoolNil(t *testing.T) { var typ bool - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a bool value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -628,10 +638,10 @@ func TestNormalizeBoolFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -641,10 +651,10 @@ func TestNormalizeBoolError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -658,14 +668,14 @@ func TestNormalizeInt(t *testing.T) { func TestNormalizeIntNil(t *testing.T) { var typ int - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a int value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -683,10 +693,10 @@ func TestNormalizeIntFromFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "1.5" as integer due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -712,10 +722,10 @@ func TestNormalizeIntFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as an integer`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -725,10 +735,10 @@ func TestNormalizeIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected int, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -742,14 +752,14 @@ func TestNormalizeFloat(t *testing.T) { func TestNormalizeFloatNil(t *testing.T) { var typ float64 - vin := dyn.NewValue(nil, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(nil, []dyn.Location{{File: "file", Line: 1, Column: 1}}) _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a float value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -771,10 +781,10 @@ func TestNormalizeFloatFromIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -800,10 +810,10 @@ func TestNormalizeFloatFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as a floating point number`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -813,10 +823,10 @@ func TestNormalizeFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected float, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -842,26 +852,26 @@ func TestNormalizeAnchors(t *testing.T) { func TestNormalizeBoolToAny(t *testing.T) { var typ any - vin := dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } func TestNormalizeIntToAny(t *testing.T) { var typ any - vin := dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}) + vin := dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } func TestNormalizeSliceToAny(t *testing.T) { var typ any - v1 := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1}) - v2 := dyn.NewValue(2, dyn.Location{File: "file", Line: 1, Column: 1}) - vin := dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1}) + v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + v2 := dyn.NewValue(2, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vin := dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}) vout, err := Normalize(&typ, vin) assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1}), vout) + assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 181c88cc9..839d0111a 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -9,6 +9,12 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" ) +// Populate a destination typed value from a source dynamic value. +// +// At any point while walking the destination type tree using +// reflection, if this function sees an exported field with type dyn.Value it +// will populate that field with the appropriate source dynamic value. +// see PR: https://github.com/databricks/cli/pull/1010 func ToTyped(dst any, src dyn.Value) error { dstv := reflect.ValueOf(dst) diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index d2494bc21..111da25c8 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -155,7 +155,7 @@ func (r *resolver) resolveRef(ref ref, seen []string) (dyn.Value, error) { // of where it is used. This also means that relative path resolution is done // relative to where a variable is used, not where it is defined. // - return dyn.NewValue(resolved[0].Value(), ref.value.Location()), nil + return dyn.NewValue(resolved[0].Value(), ref.value.Locations()), nil } // Not pure; perform string interpolation. @@ -178,7 +178,7 @@ func (r *resolver) resolveRef(ref ref, seen []string) (dyn.Value, error) { ref.str = strings.Replace(ref.str, ref.matches[j][0], s, 1) } - return dyn.NewValue(ref.str, ref.value.Location()), nil + return dyn.NewValue(ref.str, ref.value.Locations()), nil } func (r *resolver) resolveKey(key string, seen []string) (dyn.Value, error) { diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go index 668f57ecc..f9f2d2e97 100644 --- a/libs/dyn/mapping.go +++ b/libs/dyn/mapping.go @@ -46,7 +46,8 @@ func newMappingFromGoMap(vin map[string]Value) Mapping { return m } -// Pairs returns all the key-value pairs in the Mapping. +// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by +// their key in lexicographic order. func (m Mapping) Pairs() []Pair { return m.pairs } diff --git a/libs/dyn/merge/elements_by_key.go b/libs/dyn/merge/elements_by_key.go index da20ee849..e6e640d14 100644 --- a/libs/dyn/merge/elements_by_key.go +++ b/libs/dyn/merge/elements_by_key.go @@ -52,7 +52,7 @@ func (e elementsByKey) Map(_ dyn.Path, v dyn.Value) (dyn.Value, error) { out = append(out, nv) } - return dyn.NewValue(out, v.Location()), nil + return dyn.NewValue(out, v.Locations()), nil } // ElementsByKey returns a [dyn.MapFunc] that operates on a sequence diff --git a/libs/dyn/merge/merge.go b/libs/dyn/merge/merge.go index ffe000da3..29decd779 100644 --- a/libs/dyn/merge/merge.go +++ b/libs/dyn/merge/merge.go @@ -12,6 +12,26 @@ import ( // * Merging x with nil or nil with x always yields x. // * Merging maps a and b means entries from map b take precedence. // * Merging sequences a and b means concatenating them. +// +// Merging retains and accumulates the locations metadata associated with the values. +// This allows users of the module to track the provenance of values across merging of +// configuration trees, which is useful for reporting errors and warnings. +// +// Semantics for location metadata in the merged value are similar to the semantics +// for the values themselves: +// +// - When merging x with nil or nil with x, the location of x is retained. +// +// - When merging maps or sequences, the combined value retains the location of a and +// accumulates the location of b. The individual elements of the map or sequence retain +// their original locations, i.e., whether they were originally defined in a or b. +// +// The rationale for retaining location of a is that we would like to return +// the first location a bit of configuration showed up when reporting errors and warnings. +// +// - Merging primitive values means using the incoming value `b`. The location of the +// incoming value is retained and the location of the existing value `a` is accumulated. +// This is because the incoming value overwrites the existing value. func Merge(a, b dyn.Value) (dyn.Value, error) { return merge(a, b) } @@ -22,12 +42,12 @@ func merge(a, b dyn.Value) (dyn.Value, error) { // If a is nil, return b. if ak == dyn.KindNil { - return b, nil + return b.AppendLocationsFromValue(a), nil } // If b is nil, return a. if bk == dyn.KindNil { - return a, nil + return a.AppendLocationsFromValue(b), nil } // Call the appropriate merge function based on the kind of a and b. @@ -75,8 +95,8 @@ func mergeMap(a, b dyn.Value) (dyn.Value, error) { } } - // Preserve the location of the first value. - return dyn.NewValue(out, a.Location()), nil + // Preserve the location of the first value. Accumulate the locations of the second value. + return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil } func mergeSequence(a, b dyn.Value) (dyn.Value, error) { @@ -88,11 +108,10 @@ func mergeSequence(a, b dyn.Value) (dyn.Value, error) { copy(out[:], as) copy(out[len(as):], bs) - // Preserve the location of the first value. - return dyn.NewValue(out, a.Location()), nil + // Preserve the location of the first value. Accumulate the locations of the second value. + return dyn.NewValue(out, a.Locations()).AppendLocationsFromValue(b), nil } - func mergePrimitive(a, b dyn.Value) (dyn.Value, error) { // Merging primitive values means using the incoming value. - return b, nil + return b.AppendLocationsFromValue(a), nil } diff --git a/libs/dyn/merge/merge_test.go b/libs/dyn/merge/merge_test.go index 3706dbd77..4a4bf9e6c 100644 --- a/libs/dyn/merge/merge_test.go +++ b/libs/dyn/merge/merge_test.go @@ -8,15 +8,17 @@ import ( ) func TestMergeMaps(t *testing.T) { - v1 := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + v1 := dyn.NewValue(map[string]dyn.Value{ + "foo": dyn.NewValue("bar", []dyn.Location{l1}), + "bar": dyn.NewValue("baz", []dyn.Location{l1}), + }, []dyn.Location{l1}) - v2 := dyn.V(map[string]dyn.Value{ - "bar": dyn.V("qux"), - "qux": dyn.V("foo"), - }) + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + v2 := dyn.NewValue(map[string]dyn.Value{ + "bar": dyn.NewValue("qux", []dyn.Location{l2}), + "qux": dyn.NewValue("foo", []dyn.Location{l2}), + }, []dyn.Location{l2}) // Merge v2 into v1. { @@ -27,6 +29,23 @@ func TestMergeMaps(t *testing.T) { "bar": "qux", "qux": "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) + assert.Equal(t, []dyn.Location{l2, l1}, out.Get("bar").Locations()) + assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) + assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) + + // Location of the merged value should be the location of v1. + assert.Equal(t, l1, out.Location()) + + // Value of bar is "qux" which comes from v2. This .Location() should + // return the location of v2. + assert.Equal(t, l2, out.Get("bar").Location()) + + // Original locations of keys that were not overwritten should be preserved. + assert.Equal(t, l1, out.Get("foo").Location()) + assert.Equal(t, l2, out.Get("qux").Location()) } // Merge v1 into v2. @@ -38,30 +57,64 @@ func TestMergeMaps(t *testing.T) { "bar": "baz", "qux": "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) + assert.Equal(t, []dyn.Location{l1, l2}, out.Get("bar").Locations()) + assert.Equal(t, []dyn.Location{l1}, out.Get("foo").Locations()) + assert.Equal(t, []dyn.Location{l2}, out.Get("qux").Locations()) + + // Location of the merged value should be the location of v2. + assert.Equal(t, l2, out.Location()) + + // Value of bar is "baz" which comes from v1. This .Location() should + // return the location of v1. + assert.Equal(t, l1, out.Get("bar").Location()) + + // Original locations of keys that were not overwritten should be preserved. + assert.Equal(t, l1, out.Get("foo").Location()) + assert.Equal(t, l2, out.Get("qux").Location()) } + } func TestMergeMapsNil(t *testing.T) { - v := dyn.V(map[string]dyn.Value{ + l := dyn.Location{File: "file", Line: 1, Column: 2} + v := dyn.NewValue(map[string]dyn.Value{ "foo": dyn.V("bar"), - }) + }, []dyn.Location{l}) + + nilL := dyn.Location{File: "file", Line: 3, Column: 4} + nilV := dyn.NewValue(nil, []dyn.Location{nilL}) // Merge nil into v. { - out, err := Merge(v, dyn.NilValue) + out, err := Merge(v, nilV) assert.NoError(t, err) assert.Equal(t, map[string]any{ "foo": "bar", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) + + // Location of the non-nil value should be returned by .Location(). + assert.Equal(t, l, out.Location()) } // Merge v into nil. { - out, err := Merge(dyn.NilValue, v) + out, err := Merge(nilV, v) assert.NoError(t, err) assert.Equal(t, map[string]any{ "foo": "bar", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l, nilL}, out.Locations()) + + // Location of the non-nil value should be returned by .Location(). + assert.Equal(t, l, out.Location()) } } @@ -81,15 +134,18 @@ func TestMergeMapsError(t *testing.T) { } func TestMergeSequences(t *testing.T) { - v1 := dyn.V([]dyn.Value{ - dyn.V("bar"), - dyn.V("baz"), - }) + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + v1 := dyn.NewValue([]dyn.Value{ + dyn.NewValue("bar", []dyn.Location{l1}), + dyn.NewValue("baz", []dyn.Location{l1}), + }, []dyn.Location{l1}) - v2 := dyn.V([]dyn.Value{ - dyn.V("qux"), - dyn.V("foo"), - }) + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + l3 := dyn.Location{File: "file3", Line: 5, Column: 6} + v2 := dyn.NewValue([]dyn.Value{ + dyn.NewValue("qux", []dyn.Location{l2}), + dyn.NewValue("foo", []dyn.Location{l3}), + }, []dyn.Location{l2, l3}) // Merge v2 into v1. { @@ -101,6 +157,18 @@ func TestMergeSequences(t *testing.T) { "qux", "foo", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2, l3}, out.Locations()) + + // Location of the merged value should be the location of v1. + assert.Equal(t, l1, out.Location()) + + // Location of the individual values should be preserved. + assert.Equal(t, l1, out.Index(0).Location()) // "bar" + assert.Equal(t, l1, out.Index(1).Location()) // "baz" + assert.Equal(t, l2, out.Index(2).Location()) // "qux" + assert.Equal(t, l3, out.Index(3).Location()) // "foo" } // Merge v1 into v2. @@ -113,6 +181,18 @@ func TestMergeSequences(t *testing.T) { "bar", "baz", }, out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l3, l1}, out.Locations()) + + // Location of the merged value should be the location of v2. + assert.Equal(t, l2, out.Location()) + + // Location of the individual values should be preserved. + assert.Equal(t, l2, out.Index(0).Location()) // "qux" + assert.Equal(t, l3, out.Index(1).Location()) // "foo" + assert.Equal(t, l1, out.Index(2).Location()) // "bar" + assert.Equal(t, l1, out.Index(3).Location()) // "baz" } } @@ -156,14 +236,22 @@ func TestMergeSequencesError(t *testing.T) { } func TestMergePrimitives(t *testing.T) { - v1 := dyn.V("bar") - v2 := dyn.V("baz") + l1 := dyn.Location{File: "file1", Line: 1, Column: 2} + l2 := dyn.Location{File: "file2", Line: 3, Column: 4} + v1 := dyn.NewValue("bar", []dyn.Location{l1}) + v2 := dyn.NewValue("baz", []dyn.Location{l2}) // Merge v2 into v1. { out, err := Merge(v1, v2) assert.NoError(t, err) assert.Equal(t, "baz", out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l2, l1}, out.Locations()) + + // Location of the merged value should be the location of v2, the second value. + assert.Equal(t, l2, out.Location()) } // Merge v1 into v2. @@ -171,6 +259,12 @@ func TestMergePrimitives(t *testing.T) { out, err := Merge(v2, v1) assert.NoError(t, err) assert.Equal(t, "bar", out.AsAny()) + + // Locations of both values should be preserved. + assert.Equal(t, []dyn.Location{l1, l2}, out.Locations()) + + // Location of the merged value should be the location of v1, the second value. + assert.Equal(t, l1, out.Location()) } } diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go index 823fb1933..7a8667cd6 100644 --- a/libs/dyn/merge/override.go +++ b/libs/dyn/merge/override.go @@ -51,7 +51,7 @@ func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor Overri return dyn.InvalidValue, err } - return dyn.NewValue(merged, left.Location()), nil + return dyn.NewValue(merged, left.Locations()), nil case dyn.KindSequence: // some sequences are keyed, and we can detect which elements are added/removed/updated, @@ -62,7 +62,7 @@ func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor Overri return dyn.InvalidValue, err } - return dyn.NewValue(merged, left.Location()), nil + return dyn.NewValue(merged, left.Locations()), nil case dyn.KindString: if left.MustString() == right.MustString() { diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go index d9ca97486..9d41a526e 100644 --- a/libs/dyn/merge/override_test.go +++ b/libs/dyn/merge/override_test.go @@ -27,79 +27,79 @@ func TestOverride_Primitive(t *testing.T) { { name: "string (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue("b", rightLocation), - expected: dyn.NewValue("b", rightLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue("b", []dyn.Location{rightLocation}), + expected: dyn.NewValue("b", []dyn.Location{rightLocation}), }, { name: "string (not updated)", state: visitorState{}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue("a", rightLocation), - expected: dyn.NewValue("a", leftLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue("a", []dyn.Location{rightLocation}), + expected: dyn.NewValue("a", []dyn.Location{leftLocation}), }, { name: "bool (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(true, leftLocation), - right: dyn.NewValue(false, rightLocation), - expected: dyn.NewValue(false, rightLocation), + left: dyn.NewValue(true, []dyn.Location{leftLocation}), + right: dyn.NewValue(false, []dyn.Location{rightLocation}), + expected: dyn.NewValue(false, []dyn.Location{rightLocation}), }, { name: "bool (not updated)", state: visitorState{}, - left: dyn.NewValue(true, leftLocation), - right: dyn.NewValue(true, rightLocation), - expected: dyn.NewValue(true, leftLocation), + left: dyn.NewValue(true, []dyn.Location{leftLocation}), + right: dyn.NewValue(true, []dyn.Location{rightLocation}), + expected: dyn.NewValue(true, []dyn.Location{leftLocation}), }, { name: "int (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1, leftLocation), - right: dyn.NewValue(2, rightLocation), - expected: dyn.NewValue(2, rightLocation), + left: dyn.NewValue(1, []dyn.Location{leftLocation}), + right: dyn.NewValue(2, []dyn.Location{rightLocation}), + expected: dyn.NewValue(2, []dyn.Location{rightLocation}), }, { name: "int (not updated)", state: visitorState{}, - left: dyn.NewValue(int32(1), leftLocation), - right: dyn.NewValue(int64(1), rightLocation), - expected: dyn.NewValue(int32(1), leftLocation), + left: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), + right: dyn.NewValue(int64(1), []dyn.Location{rightLocation}), + expected: dyn.NewValue(int32(1), []dyn.Location{leftLocation}), }, { name: "float (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(1.0, leftLocation), - right: dyn.NewValue(2.0, rightLocation), - expected: dyn.NewValue(2.0, rightLocation), + left: dyn.NewValue(1.0, []dyn.Location{leftLocation}), + right: dyn.NewValue(2.0, []dyn.Location{rightLocation}), + expected: dyn.NewValue(2.0, []dyn.Location{rightLocation}), }, { name: "float (not updated)", state: visitorState{}, - left: dyn.NewValue(float32(1.0), leftLocation), - right: dyn.NewValue(float64(1.0), rightLocation), - expected: dyn.NewValue(float32(1.0), leftLocation), + left: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), + right: dyn.NewValue(float64(1.0), []dyn.Location{rightLocation}), + expected: dyn.NewValue(float32(1.0), []dyn.Location{leftLocation}), }, { name: "time (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(time.UnixMilli(10000), leftLocation), - right: dyn.NewValue(time.UnixMilli(10001), rightLocation), - expected: dyn.NewValue(time.UnixMilli(10001), rightLocation), + left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), + right: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), + expected: dyn.NewValue(time.UnixMilli(10001), []dyn.Location{rightLocation}), }, { name: "time (not updated)", state: visitorState{}, - left: dyn.NewValue(time.UnixMilli(10000), leftLocation), - right: dyn.NewValue(time.UnixMilli(10000), rightLocation), - expected: dyn.NewValue(time.UnixMilli(10000), leftLocation), + left: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), + right: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{rightLocation}), + expected: dyn.NewValue(time.UnixMilli(10000), []dyn.Location{leftLocation}), }, { name: "different types (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue("a", leftLocation), - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + left: dyn.NewValue("a", []dyn.Location{leftLocation}), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, { name: "map - remove 'a', update 'b'", @@ -109,23 +109,22 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, leftLocation), - "b": dyn.NewValue(10, leftLocation), + "a": dyn.NewValue(42, []dyn.Location{leftLocation}), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, - ), + []dyn.Location{leftLocation}), + right: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(20, rightLocation), + "b": dyn.NewValue(20, []dyn.Location{rightLocation}), }, - rightLocation, - ), + []dyn.Location{rightLocation}), + expected: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(20, rightLocation), + "b": dyn.NewValue(20, []dyn.Location{rightLocation}), }, - leftLocation, - ), + []dyn.Location{leftLocation}), }, { name: "map - add 'a'", @@ -134,24 +133,26 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, rightLocation), - "b": dyn.NewValue(10, rightLocation), + "a": dyn.NewValue(42, []dyn.Location{rightLocation}), + "b": dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, rightLocation), + "a": dyn.NewValue(42, []dyn.Location{rightLocation}), // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -161,23 +162,25 @@ func TestOverride_Primitive(t *testing.T) { }, left: dyn.NewValue( map[string]dyn.Value{ - "a": dyn.NewValue(42, leftLocation), - "b": dyn.NewValue(10, leftLocation), + "a": dyn.NewValue(42, []dyn.Location{leftLocation}), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ - "b": dyn.NewValue(10, rightLocation), + "b": dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ // location hasn't changed because value hasn't changed - "b": dyn.NewValue(10, leftLocation), + "b": dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -189,36 +192,38 @@ func TestOverride_Primitive(t *testing.T) { map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), + right: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, rightLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), }, - rightLocation, + []dyn.Location{rightLocation}, ), + expected: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -228,35 +233,35 @@ func TestOverride_Primitive(t *testing.T) { map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), - "job_1": dyn.NewValue(1337, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), + "job_1": dyn.NewValue(1337, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, rightLocation), + "job_0": dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( map[string]dyn.Value{ "jobs": dyn.NewValue( map[string]dyn.Value{ - "job_0": dyn.NewValue(42, leftLocation), + "job_0": dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -264,23 +269,23 @@ func TestOverride_Primitive(t *testing.T) { state: visitorState{added: []string{"root[1]"}}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), - dyn.NewValue(10, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), + dyn.NewValue(10, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), - dyn.NewValue(10, rightLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), + dyn.NewValue(10, []dyn.Location{rightLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { @@ -288,67 +293,67 @@ func TestOverride_Primitive(t *testing.T) { state: visitorState{removed: []string{"root[1]"}}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), - dyn.NewValue(10, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), + dyn.NewValue(10, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - // location hasn't changed because value hasn't changed - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), + // location hasn't changed because value hasn't changed }, { name: "sequence (not updated)", state: visitorState{}, left: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), right: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, rightLocation), + dyn.NewValue(42, []dyn.Location{rightLocation}), }, - rightLocation, + []dyn.Location{rightLocation}, ), expected: dyn.NewValue( []dyn.Value{ - dyn.NewValue(42, leftLocation), + dyn.NewValue(42, []dyn.Location{leftLocation}), }, - leftLocation, + []dyn.Location{leftLocation}, ), }, { name: "nil (not updated)", state: visitorState{}, - left: dyn.NilValue.WithLocation(leftLocation), - right: dyn.NilValue.WithLocation(rightLocation), - expected: dyn.NilValue.WithLocation(leftLocation), + left: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), + right: dyn.NilValue.WithLocations([]dyn.Location{rightLocation}), + expected: dyn.NilValue.WithLocations([]dyn.Location{leftLocation}), }, { name: "nil (updated)", state: visitorState{updated: []string{"root"}}, left: dyn.NilValue, - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, { name: "change kind (updated)", state: visitorState{updated: []string{"root"}}, - left: dyn.NewValue(42.0, leftLocation), - right: dyn.NewValue(42, rightLocation), - expected: dyn.NewValue(42, rightLocation), + left: dyn.NewValue(42.0, []dyn.Location{leftLocation}), + right: dyn.NewValue(42, []dyn.Location{rightLocation}), + expected: dyn.NewValue(42, []dyn.Location{rightLocation}), }, } @@ -375,7 +380,7 @@ func TestOverride_Primitive(t *testing.T) { }) t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { - expected := dyn.NewValue("return value", dyn.Location{}) + expected := dyn.V("return value") s, visitor := createVisitor(visitorOpts{returnValue: &expected}) out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) @@ -427,17 +432,17 @@ func TestOverride_PreserveMappingKeys(t *testing.T) { rightValueLocation := dyn.Location{File: "right.yml", Line: 3, Column: 1} left := dyn.NewMapping() - left.Set(dyn.NewValue("a", leftKeyLocation), dyn.NewValue(42, leftValueLocation)) + left.Set(dyn.NewValue("a", []dyn.Location{leftKeyLocation}), dyn.NewValue(42, []dyn.Location{leftValueLocation})) right := dyn.NewMapping() - right.Set(dyn.NewValue("a", rightKeyLocation), dyn.NewValue(7, rightValueLocation)) + right.Set(dyn.NewValue("a", []dyn.Location{rightKeyLocation}), dyn.NewValue(7, []dyn.Location{rightValueLocation})) state, visitor := createVisitor(visitorOpts{}) out, err := override( dyn.EmptyPath, - dyn.NewValue(left, leftLocation), - dyn.NewValue(right, rightLocation), + dyn.NewValue(left, []dyn.Location{leftLocation}), + dyn.NewValue(right, []dyn.Location{rightLocation}), visitor, ) diff --git a/libs/dyn/pattern.go b/libs/dyn/pattern.go index a265dad08..aecdc3ca6 100644 --- a/libs/dyn/pattern.go +++ b/libs/dyn/pattern.go @@ -72,7 +72,7 @@ func (c anyKeyComponent) visit(v Value, prefix Path, suffix Pattern, opts visitO m.Set(pk, nv) } - return NewValue(m, v.Location()), nil + return NewValue(m, v.Locations()), nil } type anyIndexComponent struct{} @@ -103,5 +103,5 @@ func (c anyIndexComponent) visit(v Value, prefix Path, suffix Pattern, opts visi s[i] = nv } - return NewValue(s, v.Location()), nil + return NewValue(s, v.Locations()), nil } diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 3d62ea1f5..2aed2f6cd 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -2,13 +2,18 @@ package dyn import ( "fmt" + "slices" ) type Value struct { v any k Kind - l Location + + // List of locations this value is defined at. The first location in the slice + // is the location returned by the `.Location()` method and is typically used + // for reporting errors and warnings associated with the value. + l []Location // Whether or not this value is an anchor. // If this node doesn't map to a type, we don't need to warn about it. @@ -27,11 +32,11 @@ var NilValue = Value{ // V constructs a new Value with the given value. func V(v any) Value { - return NewValue(v, Location{}) + return NewValue(v, []Location{}) } // NewValue constructs a new Value with the given value and location. -func NewValue(v any, loc Location) Value { +func NewValue(v any, loc []Location) Value { switch vin := v.(type) { case map[string]Value: v = newMappingFromGoMap(vin) @@ -40,16 +45,30 @@ func NewValue(v any, loc Location) Value { return Value{ v: v, k: kindOf(v), - l: loc, + + // create a copy of the locations, so that mutations to the original slice + // don't affect new value. + l: slices.Clone(loc), } } -// WithLocation returns a new Value with its location set to the given value. -func (v Value) WithLocation(loc Location) Value { +// WithLocations returns a new Value with its location set to the given value. +func (v Value) WithLocations(loc []Location) Value { return Value{ v: v.v, k: v.k, - l: loc, + + // create a copy of the locations, so that mutations to the original slice + // don't affect new value. + l: slices.Clone(loc), + } +} + +func (v Value) AppendLocationsFromValue(w Value) Value { + return Value{ + v: v.v, + k: v.k, + l: append(v.l, w.l...), } } @@ -61,10 +80,18 @@ func (v Value) Value() any { return v.v } -func (v Value) Location() Location { +func (v Value) Locations() []Location { return v.l } +func (v Value) Location() Location { + if len(v.l) == 0 { + return Location{} + } + + return v.l[0] +} + func (v Value) IsValid() bool { return v.k != KindInvalid } @@ -153,7 +180,10 @@ func (v Value) IsAnchor() bool { // We need a custom implementation because maps and slices // cannot be compared with the regular == operator. func (v Value) eq(w Value) bool { - if v.k != w.k || v.l != w.l { + if v.k != w.k { + return false + } + if !slices.Equal(v.l, w.l) { return false } diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go index bbdc2c96b..6a0a27b8d 100644 --- a/libs/dyn/value_test.go +++ b/libs/dyn/value_test.go @@ -25,16 +25,19 @@ func TestValueAsMap(t *testing.T) { _, ok := zeroValue.AsMap() assert.False(t, ok) - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) _, ok = intValue.AsMap() assert.False(t, ok) var mapValue = dyn.NewValue( map[string]dyn.Value{ - "key": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key": dyn.NewValue( + "value", + []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) + m, ok := mapValue.AsMap() assert.True(t, ok) assert.Equal(t, 1, m.Len()) @@ -43,6 +46,6 @@ func TestValueAsMap(t *testing.T) { func TestValueIsValid(t *testing.T) { var zeroValue dyn.Value assert.False(t, zeroValue.IsValid()) - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) assert.True(t, intValue.IsValid()) } diff --git a/libs/dyn/value_underlying_test.go b/libs/dyn/value_underlying_test.go index 83cffb772..e35cde582 100644 --- a/libs/dyn/value_underlying_test.go +++ b/libs/dyn/value_underlying_test.go @@ -11,7 +11,7 @@ import ( func TestValueUnderlyingMap(t *testing.T) { v := dyn.V( map[string]dyn.Value{ - "key": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, ) @@ -33,7 +33,7 @@ func TestValueUnderlyingMap(t *testing.T) { func TestValueUnderlyingSequence(t *testing.T) { v := dyn.V( []dyn.Value{ - dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, ) diff --git a/libs/dyn/visit_map.go b/libs/dyn/visit_map.go index 56a9cf9f3..cd2cd4831 100644 --- a/libs/dyn/visit_map.go +++ b/libs/dyn/visit_map.go @@ -27,7 +27,7 @@ func Foreach(fn MapFunc) MapFunc { } m.Set(pk, nv) } - return NewValue(m, v.Location()), nil + return NewValue(m, v.Locations()), nil case KindSequence: s := slices.Clone(v.MustSequence()) for i, value := range s { @@ -37,7 +37,7 @@ func Foreach(fn MapFunc) MapFunc { return InvalidValue, err } } - return NewValue(s, v.Location()), nil + return NewValue(s, v.Locations()), nil default: return InvalidValue, fmt.Errorf("expected a map or sequence, found %s", v.Kind()) } diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index e6a16f79e..fbb52b504 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -86,7 +86,7 @@ func (d *loader) loadSequence(node *yaml.Node, loc dyn.Location) (dyn.Value, err acc[i] = v } - return dyn.NewValue(acc, loc), nil + return dyn.NewValue(acc, []dyn.Location{loc}), nil } func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { @@ -130,7 +130,7 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro } if merge == nil { - return dyn.NewValue(acc, loc), nil + return dyn.NewValue(acc, []dyn.Location{loc}), nil } // Build location for the merge node. @@ -171,20 +171,20 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro out.Merge(m) } - return dyn.NewValue(out, loc), nil + return dyn.NewValue(out, []dyn.Location{loc}), nil } func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { st := node.ShortTag() switch st { case "!!str": - return dyn.NewValue(node.Value, loc), nil + return dyn.NewValue(node.Value, []dyn.Location{loc}), nil case "!!bool": switch strings.ToLower(node.Value) { case "true": - return dyn.NewValue(true, loc), nil + return dyn.NewValue(true, []dyn.Location{loc}), nil case "false": - return dyn.NewValue(false, loc), nil + return dyn.NewValue(false, []dyn.Location{loc}), nil default: return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) } @@ -195,17 +195,17 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error } // Use regular int type instead of int64 if possible. if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), loc), nil + return dyn.NewValue(int(i64), []dyn.Location{loc}), nil } - return dyn.NewValue(i64, loc), nil + return dyn.NewValue(i64, []dyn.Location{loc}), nil case "!!float": f64, err := strconv.ParseFloat(node.Value, 64) if err != nil { return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) } - return dyn.NewValue(f64, loc), nil + return dyn.NewValue(f64, []dyn.Location{loc}), nil case "!!null": - return dyn.NewValue(nil, loc), nil + return dyn.NewValue(nil, []dyn.Location{loc}), nil case "!!timestamp": // Try a couple of layouts for _, layout := range []string{ @@ -216,7 +216,7 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error } { t, terr := time.Parse(layout, node.Value) if terr == nil { - return dyn.NewValue(t, loc), nil + return dyn.NewValue(t, []dyn.Location{loc}), nil } } return dyn.InvalidValue, errorf(loc, "invalid timestamp value: %v", node.Value) diff --git a/libs/dyn/yamlsaver/saver_test.go b/libs/dyn/yamlsaver/saver_test.go index bdf1891cd..387090104 100644 --- a/libs/dyn/yamlsaver/saver_test.go +++ b/libs/dyn/yamlsaver/saver_test.go @@ -19,7 +19,7 @@ func TestMarshalNilValue(t *testing.T) { func TestMarshalIntValue(t *testing.T) { s := NewSaver() - var intValue = dyn.NewValue(1, dyn.Location{}) + var intValue = dyn.V(1) v, err := s.toYamlNode(intValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) @@ -28,7 +28,7 @@ func TestMarshalIntValue(t *testing.T) { func TestMarshalFloatValue(t *testing.T) { s := NewSaver() - var floatValue = dyn.NewValue(1.0, dyn.Location{}) + var floatValue = dyn.V(1.0) v, err := s.toYamlNode(floatValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) @@ -37,7 +37,7 @@ func TestMarshalFloatValue(t *testing.T) { func TestMarshalBoolValue(t *testing.T) { s := NewSaver() - var boolValue = dyn.NewValue(true, dyn.Location{}) + var boolValue = dyn.V(true) v, err := s.toYamlNode(boolValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) @@ -46,7 +46,7 @@ func TestMarshalBoolValue(t *testing.T) { func TestMarshalTimeValue(t *testing.T) { s := NewSaver() - var timeValue = dyn.NewValue(time.Unix(0, 0), dyn.Location{}) + var timeValue = dyn.V(time.Unix(0, 0)) v, err := s.toYamlNode(timeValue) assert.NoError(t, err) assert.Equal(t, "1970-01-01 00:00:00 +0000 UTC", v.Value) @@ -57,10 +57,10 @@ func TestMarshalSequenceValue(t *testing.T) { s := NewSaver() var sequenceValue = dyn.NewValue( []dyn.Value{ - dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), - dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), + dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) v, err := s.toYamlNode(sequenceValue) assert.NoError(t, err) @@ -71,7 +71,7 @@ func TestMarshalSequenceValue(t *testing.T) { func TestMarshalStringValue(t *testing.T) { s := NewSaver() - var stringValue = dyn.NewValue("value", dyn.Location{}) + var stringValue = dyn.V("value") v, err := s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "value", v.Value) @@ -82,12 +82,13 @@ func TestMarshalMapValue(t *testing.T) { s := NewSaver() var mapValue = dyn.NewValue( map[string]dyn.Value{ - "key3": dyn.NewValue("value3", dyn.Location{File: "file", Line: 3, Column: 2}), - "key2": dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), - "key1": dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), + "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 3, Column: 2}}), + "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), + "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) + v, err := s.toYamlNode(mapValue) assert.NoError(t, err) assert.Equal(t, yaml.MappingNode, v.Kind) @@ -107,12 +108,12 @@ func TestMarshalNestedValues(t *testing.T) { map[string]dyn.Value{ "key1": dyn.NewValue( map[string]dyn.Value{ - "key2": dyn.NewValue("value", dyn.Location{File: "file", Line: 1, Column: 2}), + "key2": dyn.NewValue("value", []dyn.Location{{File: "file", Line: 1, Column: 2}}), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ), }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) v, err := s.toYamlNode(mapValue) assert.NoError(t, err) @@ -125,14 +126,14 @@ func TestMarshalNestedValues(t *testing.T) { func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { s := NewSaver() - var hexValue = dyn.NewValue(0x123, dyn.Location{}) + var hexValue = dyn.V(0x123) v, err := s.toYamlNode(hexValue) assert.NoError(t, err) assert.Equal(t, "291", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0x123", dyn.Location{}) + var stringValue = dyn.V("0x123") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0x123", v.Value) @@ -142,14 +143,14 @@ func TestMarshalHexadecimalValueIsQuoted(t *testing.T) { func TestMarshalBinaryValueIsQuoted(t *testing.T) { s := NewSaver() - var binaryValue = dyn.NewValue(0b101, dyn.Location{}) + var binaryValue = dyn.V(0b101) v, err := s.toYamlNode(binaryValue) assert.NoError(t, err) assert.Equal(t, "5", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0b101", dyn.Location{}) + var stringValue = dyn.V("0b101") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0b101", v.Value) @@ -159,14 +160,14 @@ func TestMarshalBinaryValueIsQuoted(t *testing.T) { func TestMarshalOctalValueIsQuoted(t *testing.T) { s := NewSaver() - var octalValue = dyn.NewValue(0123, dyn.Location{}) + var octalValue = dyn.V(0123) v, err := s.toYamlNode(octalValue) assert.NoError(t, err) assert.Equal(t, "83", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("0123", dyn.Location{}) + var stringValue = dyn.V("0123") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "0123", v.Value) @@ -176,14 +177,14 @@ func TestMarshalOctalValueIsQuoted(t *testing.T) { func TestMarshalFloatValueIsQuoted(t *testing.T) { s := NewSaver() - var floatValue = dyn.NewValue(1.0, dyn.Location{}) + var floatValue = dyn.V(1.0) v, err := s.toYamlNode(floatValue) assert.NoError(t, err) assert.Equal(t, "1", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("1.0", dyn.Location{}) + var stringValue = dyn.V("1.0") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "1.0", v.Value) @@ -193,14 +194,14 @@ func TestMarshalFloatValueIsQuoted(t *testing.T) { func TestMarshalBoolValueIsQuoted(t *testing.T) { s := NewSaver() - var boolValue = dyn.NewValue(true, dyn.Location{}) + var boolValue = dyn.V(true) v, err := s.toYamlNode(boolValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) assert.Equal(t, yaml.Style(0), v.Style) assert.Equal(t, yaml.ScalarNode, v.Kind) - var stringValue = dyn.NewValue("true", dyn.Location{}) + var stringValue = dyn.V("true") v, err = s.toYamlNode(stringValue) assert.NoError(t, err) assert.Equal(t, "true", v.Value) @@ -215,18 +216,18 @@ func TestCustomStylingWithNestedMap(t *testing.T) { var styledMap = dyn.NewValue( map[string]dyn.Value{ - "key1": dyn.NewValue("value1", dyn.Location{File: "file", Line: 1, Column: 2}), - "key2": dyn.NewValue("value2", dyn.Location{File: "file", Line: 2, Column: 2}), + "key1": dyn.NewValue("value1", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + "key2": dyn.NewValue("value2", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: -2, Column: 2}, + []dyn.Location{{File: "file", Line: -2, Column: 2}}, ) var unstyledMap = dyn.NewValue( map[string]dyn.Value{ - "key3": dyn.NewValue("value3", dyn.Location{File: "file", Line: 1, Column: 2}), - "key4": dyn.NewValue("value4", dyn.Location{File: "file", Line: 2, Column: 2}), + "key3": dyn.NewValue("value3", []dyn.Location{{File: "file", Line: 1, Column: 2}}), + "key4": dyn.NewValue("value4", []dyn.Location{{File: "file", Line: 2, Column: 2}}), }, - dyn.Location{File: "file", Line: -1, Column: 2}, + []dyn.Location{{File: "file", Line: -1, Column: 2}}, ) var val = dyn.NewValue( @@ -234,7 +235,7 @@ func TestCustomStylingWithNestedMap(t *testing.T) { "styled": styledMap, "unstyled": unstyledMap, }, - dyn.Location{File: "file", Line: 1, Column: 2}, + []dyn.Location{{File: "file", Line: 1, Column: 2}}, ) mv, err := s.toYamlNode(val) diff --git a/libs/dyn/yamlsaver/utils.go b/libs/dyn/yamlsaver/utils.go index fa5ab08fb..a162bf31f 100644 --- a/libs/dyn/yamlsaver/utils.go +++ b/libs/dyn/yamlsaver/utils.go @@ -44,7 +44,7 @@ func skipAndOrder(mv dyn.Value, order *Order, skipFields []string, dst map[strin continue } - dst[k] = dyn.NewValue(v.Value(), dyn.Location{Line: order.Get(k)}) + dst[k] = dyn.NewValue(v.Value(), []dyn.Location{{Line: order.Get(k)}}) } return dyn.V(dst), nil diff --git a/libs/dyn/yamlsaver/utils_test.go b/libs/dyn/yamlsaver/utils_test.go index 04b4c404f..1afab601a 100644 --- a/libs/dyn/yamlsaver/utils_test.go +++ b/libs/dyn/yamlsaver/utils_test.go @@ -33,16 +33,25 @@ func TestConvertToMapValueWithOrder(t *testing.T) { assert.NoError(t, err) assert.Equal(t, dyn.V(map[string]dyn.Value{ - "list": dyn.NewValue([]dyn.Value{ - dyn.V("a"), - dyn.V("b"), - dyn.V("c"), - }, dyn.Location{Line: -3}), - "name": dyn.NewValue("test", dyn.Location{Line: -2}), - "map": dyn.NewValue(map[string]dyn.Value{ - "key1": dyn.V("value1"), - "key2": dyn.V("value2"), - }, dyn.Location{Line: -1}), - "long_name_field": dyn.NewValue("long name goes here", dyn.Location{Line: 1}), + "list": dyn.NewValue( + []dyn.Value{ + dyn.V("a"), + dyn.V("b"), + dyn.V("c"), + }, + []dyn.Location{{Line: -3}}, + ), + "name": dyn.NewValue( + "test", + []dyn.Location{{Line: -2}}, + ), + "map": dyn.NewValue( + map[string]dyn.Value{ + "key1": dyn.V("value1"), + "key2": dyn.V("value2"), + }, + []dyn.Location{{Line: -1}}, + ), + "long_name_field": dyn.NewValue("long name goes here", []dyn.Location{{Line: 1}}), }), result) } diff --git a/libs/filer/completer/completer.go b/libs/filer/completer/completer.go new file mode 100644 index 000000000..569286ca3 --- /dev/null +++ b/libs/filer/completer/completer.go @@ -0,0 +1,95 @@ +package completer + +import ( + "context" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/filer" + "github.com/spf13/cobra" +) + +type completer struct { + ctx context.Context + + // The filer to use for completing remote or local paths. + filer filer.Filer + + // CompletePath will only return directories when onlyDirs is true. + onlyDirs bool + + // Prefix to prepend to completions. + prefix string + + // Whether the path is local or remote. If the path is local we use the `filepath` + // package for path manipulation. Otherwise we use the `path` package. + isLocalPath bool +} + +// General completer that takes a filer to complete remote paths when TAB-ing through a path. +func New(ctx context.Context, filer filer.Filer, onlyDirs bool) *completer { + return &completer{ctx: ctx, filer: filer, onlyDirs: onlyDirs, prefix: "", isLocalPath: true} +} + +func (c *completer) SetPrefix(p string) { + c.prefix = p +} + +func (c *completer) SetIsLocalPath(i bool) { + c.isLocalPath = i +} + +func (c *completer) CompletePath(p string) ([]string, cobra.ShellCompDirective, error) { + trailingSeparator := "/" + joinFunc := path.Join + + // Use filepath functions if we are in a local path. + if c.isLocalPath { + joinFunc = filepath.Join + trailingSeparator = string(filepath.Separator) + } + + // If the user is TAB-ing their way through a path and the + // path ends in a trailing slash, we should list nested directories. + // If the path is incomplete, however, then we should list adjacent + // directories. + dirPath := p + if !strings.HasSuffix(p, trailingSeparator) { + dirPath = path.Dir(p) + } + + entries, err := c.filer.ReadDir(c.ctx, dirPath) + if err != nil { + return nil, cobra.ShellCompDirectiveError, err + } + + completions := []string{} + for _, entry := range entries { + if c.onlyDirs && !entry.IsDir() { + continue + } + + // Join directory path and entry name + completion := joinFunc(dirPath, entry.Name()) + + // Prepend prefix if it has been set + if c.prefix != "" { + completion = joinFunc(c.prefix, completion) + } + + // Add trailing separator for directories. + if entry.IsDir() { + completion += trailingSeparator + } + + completions = append(completions, completion) + } + + // If the path is local, we add the dbfs:/ prefix suggestion as an option + if c.isLocalPath { + completions = append(completions, "dbfs:/") + } + + return completions, cobra.ShellCompDirectiveNoSpace, err +} diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go new file mode 100644 index 000000000..c533f0b6c --- /dev/null +++ b/libs/filer/completer/completer_test.go @@ -0,0 +1,104 @@ +package completer + +import ( + "context" + "runtime" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func setupCompleter(t *testing.T, onlyDirs bool) *completer { + ctx := context.Background() + // Needed to make type context.valueCtx for mockFilerForPath + ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient) + + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + + completer := New(ctx, fakeFiler, onlyDirs) + completer.SetIsLocalPath(false) + return completer +} + +func TestFilerCompleterSetsPrefix(t *testing.T) { + completer := setupCompleter(t, true) + completer.SetPrefix("dbfs:") + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/wrong_path") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { + completer := setupCompleter(t, false) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterAddsDbfsPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterWindowsSeparator(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterNoCompletions(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("wrong_dir/wrong_dir") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveError, directive) + assert.Error(t, err) +} diff --git a/libs/filer/fake_filer.go b/libs/filer/fake_filer.go new file mode 100644 index 000000000..0e650ff60 --- /dev/null +++ b/libs/filer/fake_filer.go @@ -0,0 +1,134 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "sort" + "strings" + "time" +) + +type FakeDirEntry struct { + FakeFileInfo +} + +func (entry FakeDirEntry) Type() fs.FileMode { + typ := fs.ModePerm + if entry.FakeDir { + typ |= fs.ModeDir + } + return typ +} + +func (entry FakeDirEntry) Info() (fs.FileInfo, error) { + return entry.FakeFileInfo, nil +} + +type FakeFileInfo struct { + FakeName string + FakeSize int64 + FakeDir bool + FakeMode fs.FileMode +} + +func (info FakeFileInfo) Name() string { + return info.FakeName +} + +func (info FakeFileInfo) Size() int64 { + return info.FakeSize +} + +func (info FakeFileInfo) Mode() fs.FileMode { + return info.FakeMode +} + +func (info FakeFileInfo) ModTime() time.Time { + return time.Now() +} + +func (info FakeFileInfo) IsDir() bool { + return info.FakeDir +} + +func (info FakeFileInfo) Sys() any { + return nil +} + +type FakeFiler struct { + entries map[string]FakeFileInfo +} + +func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { + _, ok := f.entries[p] + if !ok { + return nil, fs.ErrNotExist + } + + return io.NopCloser(strings.NewReader("foo")), nil +} + +func (f *FakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { + p = strings.TrimSuffix(p, "/") + entry, ok := f.entries[p] + if !ok { + return nil, NoSuchDirectoryError{p} + } + + if !entry.FakeDir { + return nil, fs.ErrInvalid + } + + // Find all entries contained in the specified directory `p`. + var out []fs.DirEntry + for k, v := range f.entries { + if k == p || path.Dir(k) != p { + continue + } + + out = append(out, FakeDirEntry{v}) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) + return out, nil +} + +func (f *FakeFiler) Mkdir(ctx context.Context, path string) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { + entry, ok := f.entries[path] + if !ok { + return nil, fs.ErrNotExist + } + + return entry, nil +} + +func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler { + fakeFiler := &FakeFiler{ + entries: entries, + } + + for k, v := range fakeFiler.entries { + if v.FakeName != "" { + continue + } + v.FakeName = path.Base(k) + fakeFiler.entries[k] = v + } + + return fakeFiler +} diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 9fc68bd56..7ea1d0f03 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,7 +1,6 @@ package filer import ( - "bytes" "context" "errors" "fmt" @@ -179,12 +178,12 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, err } - var buf bytes.Buffer - err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &buf) + var reader io.ReadCloser + err = w.apiClient.Do(ctx, http.MethodGet, urlPath, nil, nil, &reader) // Return early on success. if err == nil { - return io.NopCloser(&buf), nil + return reader, nil } // Special handling of this error only if it is an API error. diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 03ed312b4..a74c10f0b 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -2,124 +2,14 @@ package filer import ( "context" - "fmt" "io" "io/fs" - "path" - "sort" - "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type fakeDirEntry struct { - fakeFileInfo -} - -func (entry fakeDirEntry) Type() fs.FileMode { - typ := fs.ModePerm - if entry.dir { - typ |= fs.ModeDir - } - return typ -} - -func (entry fakeDirEntry) Info() (fs.FileInfo, error) { - return entry.fakeFileInfo, nil -} - -type fakeFileInfo struct { - name string - size int64 - dir bool - mode fs.FileMode -} - -func (info fakeFileInfo) Name() string { - return info.name -} - -func (info fakeFileInfo) Size() int64 { - return info.size -} - -func (info fakeFileInfo) Mode() fs.FileMode { - return info.mode -} - -func (info fakeFileInfo) ModTime() time.Time { - return time.Now() -} - -func (info fakeFileInfo) IsDir() bool { - return info.dir -} - -func (info fakeFileInfo) Sys() any { - return nil -} - -type fakeFiler struct { - entries map[string]fakeFileInfo -} - -func (f *fakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { - _, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - return io.NopCloser(strings.NewReader("foo")), nil -} - -func (f *fakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { - entry, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - if !entry.dir { - return nil, fs.ErrInvalid - } - - // Find all entries contained in the specified directory `p`. - var out []fs.DirEntry - for k, v := range f.entries { - if k == p || path.Dir(k) != p { - continue - } - - out = append(out, fakeDirEntry{v}) - } - - sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) - return out, nil -} - -func (f *fakeFiler) Mkdir(ctx context.Context, path string) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { - entry, ok := f.entries[path] - if !ok { - return nil, fs.ErrNotExist - } - - return entry, nil -} - func TestFsImplementsFS(t *testing.T) { var _ fs.FS = &filerFS{} } @@ -145,22 +35,12 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) { } func fakeFS() fs.FS { - fakeFiler := &fakeFiler{ - entries: map[string]fakeFileInfo{ - ".": {name: "root", dir: true}, - "dirA": {dir: true}, - "dirB": {dir: true}, - "fileA": {size: 3}, - }, - } - - for k, v := range fakeFiler.entries { - if v.name != "" { - continue - } - v.name = path.Base(k) - fakeFiler.entries[k] = v - } + fakeFiler := NewFakeFiler(map[string]FakeFileInfo{ + ".": {FakeName: "root", FakeDir: true}, + "dirA": {FakeDir: true}, + "dirB": {FakeDir: true}, + "fileA": {FakeSize: 3}, + }) return NewFS(context.Background(), fakeFiler) } diff --git a/libs/filer/workspace_files_cache.go b/libs/filer/workspace_files_cache.go new file mode 100644 index 000000000..5837ad27d --- /dev/null +++ b/libs/filer/workspace_files_cache.go @@ -0,0 +1,428 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "slices" + "sync" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// This readahead cache is designed to optimize file system operations by caching the results of +// directory reads (ReadDir) and file/directory metadata reads (Stat). This cache aims to eliminate +// redundant operations and improve performance by storing the results of these operations and +// reusing them when possible. Additionally, the cache performs readahead on ReadDir calls, +// proactively caching information about files and subdirectories to speed up future access. +// +// The cache maintains two primary maps: one for ReadDir results and another for Stat results. +// When a directory read or a stat operation is requested, the cache first checks if the result +// is already available. If it is, the cached result is returned immediately. If not, the +// operation is queued for execution, and the result is stored in the cache once the operation +// completes. In cases where the result is not immediately available, the caller may need to wait +// for the cache entry to be populated. However, because the queue is processed in order by a +// fixed number of worker goroutines, we are guaranteed that the required cache entry will be +// populated and available once the queue processes the corresponding task. +// +// The cache uses a worker pool to process the queued operations concurrently. This is +// implemented using a fixed number of worker goroutines that continually pull tasks from a +// queue and execute them. The queue itself is logically unbounded in the sense that it needs to +// accommodate all the new tasks that may be generated dynamically during the execution of ReadDir +// calls. Specifically, a single ReadDir call can add an unknown number of new Stat and ReadDir +// tasks to the queue because each directory entry may represent a file or subdirectory that +// requires further processing. +// +// For practical reasons, we are not using an unbounded queue but a channel with a maximum size +// of 10,000. This helps prevent excessive memory usage and ensures that the system remains +// responsive under load. If we encounter real examples of subtrees with more than 10,000 +// elements, we can consider addressing this limitation in the future. For now, this approach +// balances the need for readahead efficiency with practical constraints. +// +// It is crucial to note that each ReadDir and Stat call is executed only once. The result of a +// Stat call can be served from the cache if the information was already returned by an earlier +// ReadDir call. This helps to avoid redundant operations and ensures that the system remains +// efficient even under high load. + +const ( + kMaxQueueSize = 10_000 + + // Number of worker goroutines to process the queue. + // These workers share the same HTTP client and therefore the same rate limiter. + // If this number is increased, the rate limiter should be modified as well. + kNumCacheWorkers = 1 +) + +// queueFullError is returned when the queue is at capacity. +type queueFullError struct { + name string +} + +// Error returns the error message. +func (e queueFullError) Error() string { + return fmt.Sprintf("queue is at capacity (%d); cannot enqueue work for %q", kMaxQueueSize, e.name) +} + +// Common type for all cacheable calls. +type cacheEntry struct { + // Channel to signal that the operation has completed. + done chan struct{} + + // The (cleaned) name of the file or directory being operated on. + name string + + // Return values of the operation. + err error +} + +// String returns the path of the file or directory being operated on. +func (e *cacheEntry) String() string { + return e.name +} + +// Mark this entry as errored. +func (e *cacheEntry) markError(err error) { + e.err = err + close(e.done) +} + +// readDirEntry is the cache entry for a [ReadDir] call. +type readDirEntry struct { + cacheEntry + + // Return values of a [ReadDir] call. + entries []fs.DirEntry +} + +// Create a new readDirEntry. +func newReadDirEntry(name string) *readDirEntry { + return &readDirEntry{cacheEntry: cacheEntry{done: make(chan struct{}), name: name}} +} + +// Execute the operation and signal completion. +func (e *readDirEntry) execute(ctx context.Context, c *cache) { + t1 := time.Now() + e.entries, e.err = c.f.ReadDir(ctx, e.name) + t2 := time.Now() + log.Tracef(ctx, "readdir for %s took %f", e.name, t2.Sub(t1).Seconds()) + + // Finalize the read call by adding all directory entries to the stat cache. + c.completeReadDir(e.name, e.entries) + + // Signal that the operation has completed. + // The return value can now be used by routines waiting on it. + close(e.done) +} + +// Wait for the operation to complete and return the result. +func (e *readDirEntry) wait(ctx context.Context) ([]fs.DirEntry, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-e.done: + // Note: return a copy of the slice to prevent the caller from modifying the cache. + // The underlying elements are values (see [wsfsDirEntry]) so a shallow copy is sufficient. + return slices.Clone(e.entries), e.err + } +} + +// statEntry is the cache entry for a [Stat] call. +type statEntry struct { + cacheEntry + + // Return values of a [Stat] call. + info fs.FileInfo +} + +// Create a new stat entry. +func newStatEntry(name string) *statEntry { + return &statEntry{cacheEntry: cacheEntry{done: make(chan struct{}), name: name}} +} + +// Execute the operation and signal completion. +func (e *statEntry) execute(ctx context.Context, c *cache) { + t1 := time.Now() + e.info, e.err = c.f.Stat(ctx, e.name) + t2 := time.Now() + log.Tracef(ctx, "stat for %s took %f", e.name, t2.Sub(t1).Seconds()) + + // Signal that the operation has completed. + // The return value can now be used by routines waiting on it. + close(e.done) +} + +// Wait for the operation to complete and return the result. +func (e *statEntry) wait(ctx context.Context) (fs.FileInfo, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-e.done: + return e.info, e.err + } +} + +// Mark the stat entry as done. +func (e *statEntry) markDone(info fs.FileInfo, err error) { + e.info = info + e.err = err + close(e.done) +} + +// executable is the interface all cacheable calls must implement. +type executable interface { + fmt.Stringer + + execute(ctx context.Context, c *cache) +} + +// cache stores all entries for cacheable Workspace File System calls. +// We care about caching only [ReadDir] and [Stat] calls. +type cache struct { + f Filer + m sync.Mutex + + readDir map[string]*readDirEntry + stat map[string]*statEntry + + // Queue of operations to execute. + queue chan executable + + // For tracking the number of active goroutines. + wg sync.WaitGroup +} + +func newWorkspaceFilesReadaheadCache(f Filer) *cache { + c := &cache{ + f: f, + + readDir: make(map[string]*readDirEntry), + stat: make(map[string]*statEntry), + + queue: make(chan executable, kMaxQueueSize), + } + + ctx := context.Background() + for range kNumCacheWorkers { + c.wg.Add(1) + go c.work(ctx) + } + + return c +} + +// work until the queue is closed. +func (c *cache) work(ctx context.Context) { + defer c.wg.Done() + + for e := range c.queue { + e.execute(ctx, c) + } +} + +// enqueue adds an operation to the queue. +// If the context is canceled, an error is returned. +// If the queue is full, an error is returned. +// +// Its caller is holding the lock so it cannot block. +func (c *cache) enqueue(ctx context.Context, e executable) error { + select { + case <-ctx.Done(): + return ctx.Err() + case c.queue <- e: + return nil + default: + return queueFullError{e.String()} + } +} + +func (c *cache) completeReadDirForDir(name string, dirEntry fs.DirEntry) { + // Add to the stat cache if not already present. + if _, ok := c.stat[name]; !ok { + e := newStatEntry(name) + e.markDone(dirEntry.Info()) + c.stat[name] = e + } + + // Queue a [ReadDir] call for the directory if not already present. + if _, ok := c.readDir[name]; !ok { + // Create a new cache entry and queue the operation. + e := newReadDirEntry(name) + err := c.enqueue(context.Background(), e) + if err != nil { + e.markError(err) + } + + // Add the entry to the cache, even if has an error. + c.readDir[name] = e + } +} + +func (c *cache) completeReadDirForFile(name string, dirEntry fs.DirEntry) { + // Skip if this entry is already in the cache. + if _, ok := c.stat[name]; ok { + return + } + + // Create a new cache entry. + e := newStatEntry(name) + + // Depending on the object type, we either have to perform a real + // stat call, or we can use the [fs.DirEntry] info directly. + switch dirEntry.(wsfsDirEntry).ObjectType { + case workspace.ObjectTypeNotebook: + // Note: once the list API returns `repos_export_format` we can avoid this additional stat call. + // This is the only (?) case where this implementation is tied to the workspace filer. + + // Queue a [Stat] call for the file. + err := c.enqueue(context.Background(), e) + if err != nil { + e.markError(err) + } + default: + // Use the [fs.DirEntry] info directly. + e.markDone(dirEntry.Info()) + } + + // Add the entry to the cache, even if has an error. + c.stat[name] = e +} + +func (c *cache) completeReadDir(dir string, entries []fs.DirEntry) { + c.m.Lock() + defer c.m.Unlock() + + for _, e := range entries { + name := path.Join(dir, e.Name()) + + if e.IsDir() { + c.completeReadDirForDir(name, e) + } else { + c.completeReadDirForFile(name, e) + } + } +} + +// Cleanup closes the queue and waits for all goroutines to exit. +func (c *cache) Cleanup() { + close(c.queue) + c.wg.Wait() +} + +// Write passes through to the underlying Filer. +func (c *cache) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + return c.f.Write(ctx, name, reader, mode...) +} + +// Read passes through to the underlying Filer. +func (c *cache) Read(ctx context.Context, name string) (io.ReadCloser, error) { + return c.f.Read(ctx, name) +} + +// Delete passes through to the underlying Filer. +func (c *cache) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + return c.f.Delete(ctx, name, mode...) +} + +// Mkdir passes through to the underlying Filer. +func (c *cache) Mkdir(ctx context.Context, name string) error { + return c.f.Mkdir(ctx, name) +} + +// ReadDir returns the entries in a directory. +// If the directory is already in the cache, the cached value is returned. +func (c *cache) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + name = path.Clean(name) + + // Lock before R/W access to the cache. + c.m.Lock() + + // If the directory is already in the cache, wait for and return the cached value. + if e, ok := c.readDir[name]; ok { + c.m.Unlock() + return e.wait(ctx) + } + + // Otherwise, create a new cache entry and queue the operation. + e := newReadDirEntry(name) + err := c.enqueue(ctx, e) + if err != nil { + c.m.Unlock() + return nil, err + } + + c.readDir[name] = e + c.m.Unlock() + + // Wait for the operation to complete. + return e.wait(ctx) +} + +// statFromReadDir returns the file info for a file or directory. +// If the file info is already in the cache, the cached value is returned. +func (c *cache) statFromReadDir(ctx context.Context, name string, entry *readDirEntry) (fs.FileInfo, error) { + _, err := entry.wait(ctx) + if err != nil { + return nil, err + } + + // Upon completion of a [ReadDir] call, all directory entries are added to the stat cache and + // enqueue a [Stat] call if necessary (entries for notebooks are incomplete and require a + // real stat call). + // + // This means that the file or directory we're trying to stat, either + // + // - is present in the stat cache + // - doesn't exist. + // + c.m.Lock() + e, ok := c.stat[name] + c.m.Unlock() + if ok { + return e.wait(ctx) + } + + return nil, FileDoesNotExistError{name} +} + +// Stat returns the file info for a file or directory. +// If the file info is already in the cache, the cached value is returned. +func (c *cache) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + name = path.Clean(name) + + // Lock before R/W access to the cache. + c.m.Lock() + + // If the file info is already in the cache, wait for and return the cached value. + if e, ok := c.stat[name]; ok { + c.m.Unlock() + return e.wait(ctx) + } + + // If the parent directory is in the cache (or queued to be read), + // wait for it to complete to avoid redundant stat calls. + dir := path.Dir(name) + if dir != name { + if e, ok := c.readDir[dir]; ok { + c.m.Unlock() + return c.statFromReadDir(ctx, name, e) + } + } + + // Otherwise, create a new cache entry and queue the operation. + e := newStatEntry(name) + err := c.enqueue(ctx, e) + if err != nil { + c.m.Unlock() + return nil, err + } + + c.stat[name] = e + c.m.Unlock() + + // Wait for the operation to complete. + return e.wait(ctx) +} diff --git a/libs/filer/workspace_files_cache_test.go b/libs/filer/workspace_files_cache_test.go new file mode 100644 index 000000000..8983c5982 --- /dev/null +++ b/libs/filer/workspace_files_cache_test.go @@ -0,0 +1,319 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "testing" + + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" +) + +var errNotImplemented = fmt.Errorf("not implemented") + +type cacheTestFiler struct { + calls int + + readDir map[string][]fs.DirEntry + stat map[string]fs.FileInfo +} + +func (m *cacheTestFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...WriteMode) error { + return errNotImplemented +} + +func (m *cacheTestFiler) Read(ctx context.Context, path string) (io.ReadCloser, error) { + return nil, errNotImplemented +} + +func (m *cacheTestFiler) Delete(ctx context.Context, path string, mode ...DeleteMode) error { + return errNotImplemented +} + +func (m *cacheTestFiler) ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error) { + m.calls++ + if fi, ok := m.readDir[path]; ok { + delete(m.readDir, path) + return fi, nil + } + return nil, fs.ErrNotExist +} + +func (m *cacheTestFiler) Mkdir(ctx context.Context, path string) error { + return errNotImplemented +} + +func (m *cacheTestFiler) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + m.calls++ + if fi, ok := m.stat[name]; ok { + delete(m.stat, name) + return fi, nil + } + return nil, fs.ErrNotExist +} + +func TestWorkspaceFilesCache_ReadDirCache(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file2", + Size: 2, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First read dir should hit the filer, second should hit the cache. + for range 2 { + fi, err := cache.ReadDir(ctx, "dir1") + assert.NoError(t, err) + if assert.Len(t, fi, 2) { + assert.Equal(t, "file1", fi[0].Name()) + assert.Equal(t, "file2", fi[1].Name()) + } + } + + // Third stat should hit the filer, fourth should hit the cache. + for range 2 { + _, err := cache.ReadDir(ctx, "dir2") + assert.ErrorIs(t, err, fs.ErrNotExist) + } + + // Assert we only called the filer twice. + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirCacheIsolation(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First read dir should hit the filer, second should hit the cache. + entries, err := cache.ReadDir(ctx, "dir") + assert.NoError(t, err) + assert.Equal(t, "file", entries[0].Name()) + + // Modify the entry to check that mutations are not reflected in the cache. + entries[0] = wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "tainted", + }, + }, + } + + // Read the directory again to check that the cache is isolated. + entries, err = cache.ReadDir(ctx, "dir") + assert.NoError(t, err) + assert.Equal(t, "file", entries[0].Name()) +} + +func TestWorkspaceFilesCache_StatCache(t *testing.T) { + f := &cacheTestFiler{ + stat: map[string]fs.FileInfo{ + "file1": &wsfsFileInfo{ObjectInfo: workspace.ObjectInfo{Path: "file1", Size: 1}}, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // First stat should hit the filer, second should hit the cache. + for range 2 { + fi, err := cache.Stat(ctx, "file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + } + + // Third stat should hit the filer, fourth should hit the cache. + for range 2 { + _, err := cache.Stat(ctx, "file2") + assert.ErrorIs(t, err, fs.ErrNotExist) + } + + // Assert we only called the filer twice. + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirPopulatesStatCache(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "dir1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file2", + Size: 2, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "notebook1", + Size: 1, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: "this should not end up in the stat cache", + }, + }, + }, + }, + stat: map[string]fs.FileInfo{ + "dir1/notebook1": wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "notebook1", + Size: 1, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: workspace.ExportFormatJupyter, + }, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // Issue read dir to populate the stat cache. + _, err := cache.ReadDir(ctx, "dir1") + assert.NoError(t, err) + + // Stat on a file in the directory should hit the cache. + fi, err := cache.Stat(ctx, "dir1/file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + + // If the containing directory has been read, absence is also inferred from the cache. + _, err = cache.Stat(ctx, "dir1/file3") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Stat on a notebook in the directory should have been performed in the background. + fi, err = cache.Stat(ctx, "dir1/notebook1") + if assert.NoError(t, err) { + assert.Equal(t, "notebook1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + assert.Equal(t, workspace.ExportFormatJupyter, fi.(wsfsFileInfo).ReposExportFormat) + } + + // Assert we called the filer twice (once for read dir, once for stat on the notebook). + assert.Equal(t, 2, f.calls) +} + +func TestWorkspaceFilesCache_ReadDirTriggersReadahead(t *testing.T) { + f := &cacheTestFiler{ + readDir: map[string][]fs.DirEntry{ + "a": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "b1", + ObjectType: workspace.ObjectTypeDirectory, + }, + }, + }, + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "b2", + ObjectType: workspace.ObjectTypeDirectory, + }, + }, + }, + }, + "a/b1": { + wsfsDirEntry{ + wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: "file1", + Size: 1, + ObjectType: workspace.ObjectTypeFile, + }, + }, + }, + }, + "a/b2": {}, + }, + } + + ctx := context.Background() + cache := newWorkspaceFilesReadaheadCache(f) + defer cache.Cleanup() + + // Issue read dir to populate the stat cache. + _, err := cache.ReadDir(ctx, "a") + assert.NoError(t, err) + + // Stat on a directory in the directory should hit the cache. + fi, err := cache.Stat(ctx, "a/b1") + if assert.NoError(t, err) { + assert.Equal(t, "b1", fi.Name()) + assert.True(t, fi.IsDir()) + } + + // Stat on a file in a nested directory should hit the cache. + fi, err = cache.Stat(ctx, "a/b1/file1") + if assert.NoError(t, err) { + assert.Equal(t, "file1", fi.Name()) + assert.Equal(t, int64(1), fi.Size()) + } + + // Stat on a non-existing file in an empty nested directory should hit the cache. + _, err = cache.Stat(ctx, "a/b2/file2") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Assert we called the filer 3 times; once for each directory. + assert.Equal(t, 3, f.calls) +} diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index 194317578..5710f354a 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -84,6 +84,10 @@ func (info wsfsFileInfo) Sys() any { return info.ObjectInfo } +func (info wsfsFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo { + return info.ObjectInfo +} + // UnmarshalJSON is a custom unmarshaller for the wsfsFileInfo struct. // It must be defined for this type because otherwise the implementation // of the embedded ObjectInfo type will be used. @@ -98,13 +102,21 @@ func (info *wsfsFileInfo) MarshalJSON() ([]byte, error) { return marshal.Marshal(info) } +// Interface for *client.DatabricksClient from the Databricks Go SDK. Abstracted +// as an interface to allow for mocking in tests. +type apiClient interface { + Do(ctx context.Context, method, path string, + headers map[string]string, request, response any, + visitors ...func(*http.Request) error) error +} + // WorkspaceFilesClient implements the files-in-workspace API. // NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled. // It can access any workspace path if files-in-workspace is enabled. -type WorkspaceFilesClient struct { +type workspaceFilesClient struct { workspaceClient *databricks.WorkspaceClient - apiClient *client.DatabricksClient + apiClient apiClient // File operations will be relative to this path. root WorkspaceRootPath @@ -116,7 +128,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, return nil, err } - return &WorkspaceFilesClient{ + return &workspaceFilesClient{ workspaceClient: w, apiClient: apiClient, @@ -124,7 +136,7 @@ func NewWorkspaceFilesClient(w *databricks.WorkspaceClient, root string) (Filer, }, nil } -func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { +func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { absPath, err := w.root.Join(name) if err != nil { return err @@ -208,7 +220,7 @@ func (w *WorkspaceFilesClient) Write(ctx context.Context, name string, reader io return err } -func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { +func (w *workspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err @@ -232,7 +244,7 @@ func (w *WorkspaceFilesClient) Read(ctx context.Context, name string) (io.ReadCl return w.workspaceClient.Workspace.Download(ctx, absPath) } -func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { +func (w *workspaceFilesClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { absPath, err := w.root.Join(name) if err != nil { return err @@ -276,7 +288,7 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string, mode ... return err } -func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { +func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err @@ -309,7 +321,7 @@ func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D return wsfsDirEntriesFromObjectInfos(objects), nil } -func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { +func (w *workspaceFilesClient) Mkdir(ctx context.Context, name string) error { dirPath, err := w.root.Join(name) if err != nil { return err @@ -319,7 +331,7 @@ func (w *WorkspaceFilesClient) Mkdir(ctx context.Context, name string) error { }) } -func (w *WorkspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { +func (w *workspaceFilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { absPath, err := w.root.Join(name) if err != nil { return nil, err diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go index a872dcc65..b24ecf7ee 100644 --- a/libs/filer/workspace_files_extensions_client.go +++ b/libs/filer/workspace_files_extensions_client.go @@ -18,8 +18,9 @@ import ( type workspaceFilesExtensionsClient struct { workspaceClient *databricks.WorkspaceClient - wsfs Filer - root string + wsfs Filer + root string + readonly bool } var extensionsToLanguages = map[string]workspace.Language{ @@ -132,23 +133,31 @@ func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx con }, nil } -type DuplicatePathError struct { +type duplicatePathError struct { oi1 workspace.ObjectInfo oi2 workspace.ObjectInfo commonName string } -func (e DuplicatePathError) Error() string { +func (e duplicatePathError) Error() string { return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName) } +type ReadOnlyError struct { + op string +} + +func (e ReadOnlyError) Error() string { + return fmt.Sprintf("failed to %s: filer is in read-only mode", e.op) +} + // This is a filer for the workspace file system that allows you to pretend the // workspace file system is a traditional file system. It allows you to list, read, write, // delete, and stat notebooks (and files in general) in the workspace, using their paths // with the extension included. // -// The ReadDir method returns a DuplicatePathError if this traditional file system view is +// The ReadDir method returns a duplicatePathError if this traditional file system view is // not possible. For example, a Python notebook called foo and a Python file called `foo.py` // would resolve to the same path `foo.py` in a tradition file system. // @@ -157,16 +166,30 @@ func (e DuplicatePathError) Error() string { // errors for namespace clashes (e.g. a file and a notebook or a directory and a notebook). // Thus users of these methods should be careful to avoid such clashes. func NewWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + return newWorkspaceFilesExtensionsClient(w, root, false) +} + +func NewReadOnlyWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + return newWorkspaceFilesExtensionsClient(w, root, true) +} + +func newWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string, readonly bool) (Filer, error) { filer, err := NewWorkspaceFilesClient(w, root) if err != nil { return nil, err } + if readonly { + // Wrap in a readahead cache to avoid making unnecessary calls to the workspace. + filer = newWorkspaceFilesReadaheadCache(filer) + } + return &workspaceFilesExtensionsClient{ workspaceClient: w, - wsfs: filer, - root: root, + wsfs: filer, + root: root, + readonly: readonly, }, nil } @@ -197,7 +220,7 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin // Error if we have seen this path before in the current directory. // If not seen before, add it to the seen paths. if _, ok := seenPaths[entries[i].Name()]; ok { - return nil, DuplicatePathError{ + return nil, duplicatePathError{ oi1: seenPaths[entries[i].Name()], oi2: sysInfo, commonName: path.Join(name, entries[i].Name()), @@ -213,6 +236,10 @@ func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name strin // (e.g. a file and a notebook or a directory and a notebook). Thus users of this // method should be careful to avoid such clashes. func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + if w.readonly { + return ReadOnlyError{"write"} + } + return w.wsfs.Write(ctx, name, reader, mode...) } @@ -246,6 +273,10 @@ func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) // Try to delete the file as a regular file. If the file is not found, try to delete it as a notebook. func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + if w.readonly { + return ReadOnlyError{"delete"} + } + err := w.wsfs.Delete(ctx, name, mode...) // If the file is not found, it might be a notebook. @@ -292,5 +323,9 @@ func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string) // (e.g. a file and a notebook or a directory and a notebook). Thus users of this // method should be careful to avoid such clashes. func (w *workspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error { + if w.readonly { + return ReadOnlyError{"mkdir"} + } + return w.wsfs.Mkdir(ctx, name) } diff --git a/libs/filer/workspace_files_extensions_client_test.go b/libs/filer/workspace_files_extensions_client_test.go new file mode 100644 index 000000000..321c43712 --- /dev/null +++ b/libs/filer/workspace_files_extensions_client_test.go @@ -0,0 +1,151 @@ +package filer + +import ( + "context" + "net/http" + "testing" + + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// Mocks client.DatabricksClient from the databricks-sdk-go package. +type mockApiClient struct { + mock.Mock +} + +func (m *mockApiClient) Do(ctx context.Context, method, path string, + headers map[string]string, request any, response any, + visitors ...func(*http.Request) error) error { + args := m.Called(ctx, method, path, headers, request, response, visitors) + + // Set the http response from a value provided in the mock call. + p := response.(*wsfsFileInfo) + *p = args.Get(1).(wsfsFileInfo) + return args.Error(0) +} + +func TestFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) { + for _, tc := range []struct { + name string + language workspace.Language + notebookExportFormat workspace.ExportFormat + notebookPath string + filePath string + expectedError string + }{ + { + name: "python source notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.py", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.py resolve to the same name /foo.py. Changing the name of one of these objects will resolve this issue", + }, + { + name: "python jupyter notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatJupyter, + notebookPath: "/dir/foo", + filePath: "/dir/foo.py", + // Jupyter notebooks would correspond to foo.ipynb so an error is not expected. + expectedError: "", + }, + { + name: "scala source notebook and file", + language: workspace.LanguageScala, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.scala", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.scala resolve to the same name /foo.scala. Changing the name of one of these objects will resolve this issue", + }, + { + name: "r source notebook and file", + language: workspace.LanguageR, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.r", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.r resolve to the same name /foo.r. Changing the name of one of these objects will resolve this issue", + }, + { + name: "sql source notebook and file", + language: workspace.LanguageSql, + notebookExportFormat: workspace.ExportFormatSource, + notebookPath: "/dir/foo", + filePath: "/dir/foo.sql", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.sql resolve to the same name /foo.sql. Changing the name of one of these objects will resolve this issue", + }, + { + name: "python jupyter notebook and file", + language: workspace.LanguagePython, + notebookExportFormat: workspace.ExportFormatJupyter, + notebookPath: "/dir/foo", + filePath: "/dir/foo.ipynb", + expectedError: "failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at /dir/foo and FILE at /dir/foo.ipynb resolve to the same name /foo.ipynb. Changing the name of one of these objects will resolve this issue", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mockedWorkspaceClient := mocks.NewMockWorkspaceClient(t) + mockedApiClient := mockApiClient{} + + // Mock the workspace API's ListAll method. + workspaceApi := mockedWorkspaceClient.GetMockWorkspaceAPI() + workspaceApi.EXPECT().ListAll(mock.Anything, workspace.ListWorkspaceRequest{ + Path: "/dir", + }).Return([]workspace.ObjectInfo{ + { + Path: tc.filePath, + Language: tc.language, + ObjectType: workspace.ObjectTypeFile, + }, + { + Path: tc.notebookPath, + Language: tc.language, + ObjectType: workspace.ObjectTypeNotebook, + }, + }, nil) + + // Mock bespoke API calls to /api/2.0/workspace/get-status, that are + // used to figure out the right file extension for the notebook. + statNotebook := wsfsFileInfo{ + ObjectInfo: workspace.ObjectInfo{ + Path: tc.notebookPath, + Language: tc.language, + ObjectType: workspace.ObjectTypeNotebook, + }, + ReposExportFormat: tc.notebookExportFormat, + } + + mockedApiClient.On("Do", mock.Anything, http.MethodGet, "/api/2.0/workspace/get-status", map[string]string(nil), map[string]string{ + "path": tc.notebookPath, + "return_export_info": "true", + }, mock.AnythingOfType("*filer.wsfsFileInfo"), []func(*http.Request) error(nil)).Return(nil, statNotebook) + + workspaceFilesClient := workspaceFilesClient{ + workspaceClient: mockedWorkspaceClient.WorkspaceClient, + apiClient: &mockedApiClient, + root: NewWorkspaceRootPath("/dir"), + } + + workspaceFilesExtensionsClient := workspaceFilesExtensionsClient{ + workspaceClient: mockedWorkspaceClient.WorkspaceClient, + wsfs: &workspaceFilesClient, + } + + _, err := workspaceFilesExtensionsClient.ReadDir(context.Background(), "/") + + if tc.expectedError == "" { + assert.NoError(t, err) + } else { + assert.ErrorAs(t, err, &duplicatePathError{}) + assert.EqualError(t, err, tc.expectedError) + } + + // assert the mocked methods were actually called, as a sanity check. + workspaceApi.AssertNumberOfCalls(t, "ListAll", 1) + mockedApiClient.AssertNumberOfCalls(t, "Do", 1) + }) + } +} diff --git a/libs/fileset/fileset.go b/libs/fileset/fileset.go index d0f00f97a..00c6dcfa4 100644 --- a/libs/fileset/fileset.go +++ b/libs/fileset/fileset.go @@ -3,25 +3,56 @@ package fileset import ( "fmt" "io/fs" - "os" + pathlib "path" + "path/filepath" + "slices" "github.com/databricks/cli/libs/vfs" ) -// FileSet facilitates fast recursive file listing of a path. +// FileSet facilitates recursive file listing for paths rooted at a given directory. // It optionally takes into account ignore rules through the [Ignorer] interface. type FileSet struct { // Root path of the fileset. root vfs.Path + // Paths to include in the fileset. + // Files are included as-is (if not ignored) and directories are traversed recursively. + // Defaults to []string{"."} if not specified. + paths []string + // Ignorer interface to check if a file or directory should be ignored. ignore Ignorer } // New returns a [FileSet] for the given root path. -func New(root vfs.Path) *FileSet { +// It optionally accepts a list of paths relative to the root to include in the fileset. +// If not specified, it defaults to including all files in the root path. +func New(root vfs.Path, args ...[]string) *FileSet { + // Default to including all files in the root path. + if len(args) == 0 { + args = [][]string{{"."}} + } + + // Collect list of normalized and cleaned paths. + var paths []string + for _, arg := range args { + for _, path := range arg { + path = filepath.ToSlash(path) + path = pathlib.Clean(path) + + // Skip path if it's already in the list. + if slices.Contains(paths, path) { + continue + } + + paths = append(paths, path) + } + } + return &FileSet{ root: root, + paths: paths, ignore: nopIgnorer{}, } } @@ -36,30 +67,38 @@ func (w *FileSet) SetIgnorer(ignore Ignorer) { w.ignore = ignore } -// Return all tracked files for Repo -func (w *FileSet) All() ([]File, error) { - return w.recursiveListFiles() +// Files returns performs recursive listing on all configured paths and returns +// the collection of files it finds (and are not ignored). +// The returned slice does not contain duplicates. +// The order of files in the slice is stable. +func (w *FileSet) Files() (out []File, err error) { + seen := make(map[string]struct{}) + for _, p := range w.paths { + files, err := w.recursiveListFiles(p, seen) + if err != nil { + return nil, err + } + out = append(out, files...) + } + return out, nil } // Recursively traverses dir in a depth first manner and returns a list of all files // that are being tracked in the FileSet (ie not being ignored for matching one of the // patterns in w.ignore) -func (w *FileSet) recursiveListFiles() (fileList []File, err error) { - err = fs.WalkDir(w.root, ".", func(name string, d fs.DirEntry, err error) error { +func (w *FileSet) recursiveListFiles(path string, seen map[string]struct{}) (out []File, err error) { + err = fs.WalkDir(w.root, path, func(name string, d fs.DirEntry, err error) error { if err != nil { return err } - // skip symlinks info, err := d.Info() if err != nil { return err } - if info.Mode()&os.ModeSymlink != 0 { - return nil - } - if d.IsDir() { + switch { + case info.Mode().IsDir(): ign, err := w.ignore.IgnoreDirectory(name) if err != nil { return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) @@ -67,18 +106,28 @@ func (w *FileSet) recursiveListFiles() (fileList []File, err error) { if ign { return fs.SkipDir } - return nil + + case info.Mode().IsRegular(): + ign, err := w.ignore.IgnoreFile(name) + if err != nil { + return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) + } + if ign { + return nil + } + + // Skip duplicates + if _, ok := seen[name]; ok { + return nil + } + + seen[name] = struct{}{} + out = append(out, NewFile(w.root, d, name)) + + default: + // Skip non-regular files (e.g. symlinks). } - ign, err := w.ignore.IgnoreFile(name) - if err != nil { - return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) - } - if ign { - return nil - } - - fileList = append(fileList, NewFile(w.root, d, name)) return nil }) return diff --git a/libs/fileset/fileset_test.go b/libs/fileset/fileset_test.go new file mode 100644 index 000000000..be27b6b6f --- /dev/null +++ b/libs/fileset/fileset_test.go @@ -0,0 +1,144 @@ +package fileset + +import ( + "errors" + "testing" + + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" +) + +func TestFileSet_NoPaths(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 4) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) + assert.Equal(t, "dir2/a", files[2].Relative) + assert.Equal(t, "dir2/b", files[3].Relative) +} + +func TestFileSet_ParentPath(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{".."}) + _, err := fs.Files() + + // It is impossible to escape the root directory. + assert.Error(t, err) +} + +func TestFileSet_DuplicatePaths(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{"dir1", "dir1"}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) +} + +func TestFileSet_OverlappingPaths(t *testing.T) { + fs := New(vfs.MustNew("testdata"), []string{"dir1", "dir1/a"}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir1/a", files[0].Relative) + assert.Equal(t, "dir1/b", files[1].Relative) +} + +func TestFileSet_IgnoreDirError(t *testing.T) { + testError := errors.New("test error") + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{dirErr: testError}) + _, err := fs.Files() + assert.ErrorIs(t, err, testError) +} + +func TestFileSet_IgnoreDir(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{dir: []string{"dir1"}}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 2) + assert.Equal(t, "dir2/a", files[0].Relative) + assert.Equal(t, "dir2/b", files[1].Relative) +} + +func TestFileSet_IgnoreFileError(t *testing.T) { + testError := errors.New("test error") + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{fileErr: testError}) + _, err := fs.Files() + assert.ErrorIs(t, err, testError) +} + +func TestFileSet_IgnoreFile(t *testing.T) { + fs := New(vfs.MustNew("testdata")) + fs.SetIgnorer(testIgnorer{file: []string{"dir1/a"}}) + files, err := fs.Files() + if !assert.NoError(t, err) { + return + } + + assert.Len(t, files, 3) + assert.Equal(t, "dir1/b", files[0].Relative) + assert.Equal(t, "dir2/a", files[1].Relative) + assert.Equal(t, "dir2/b", files[2].Relative) +} + +type testIgnorer struct { + // dir is a list of directories to ignore. Strings are compared verbatim. + dir []string + + // dirErr is an error to return when IgnoreDirectory is called. + dirErr error + + // file is a list of files to ignore. Strings are compared verbatim. + file []string + + // fileErr is an error to return when IgnoreFile is called. + fileErr error +} + +// IgnoreDirectory returns true if the path is in the dir list. +// If dirErr is set, it returns dirErr. +func (t testIgnorer) IgnoreDirectory(path string) (bool, error) { + if t.dirErr != nil { + return false, t.dirErr + } + + for _, d := range t.dir { + if d == path { + return true, nil + } + } + + return false, nil +} + +// IgnoreFile returns true if the path is in the file list. +// If fileErr is set, it returns fileErr. +func (t testIgnorer) IgnoreFile(path string) (bool, error) { + if t.fileErr != nil { + return false, t.fileErr + } + + for _, f := range t.file { + if f == path { + return true, nil + } + } + + return false, nil +} diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 70b9c444b..9eb786db9 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -20,16 +20,21 @@ func collectRelativePaths(files []File) []string { } func TestGlobFileset(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("./") entries, err := root.ReadDir(".") require.NoError(t, err) + // Remove testdata folder from entries + entries = slices.DeleteFunc(entries, func(de fs.DirEntry) bool { + return de.Name() == "testdata" + }) + g, err := NewGlobSet(root, []string{ "./*.go", }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.Equal(t, len(files), len(entries)) @@ -45,13 +50,13 @@ func TestGlobFileset(t *testing.T) { }) require.NoError(t, err) - files, err = g.All() + files, err = g.Files() require.NoError(t, err) require.Equal(t, len(files), 0) } func TestGlobFilesetWithRelativeRoot(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("../set") entries, err := root.ReadDir(".") require.NoError(t, err) @@ -60,7 +65,7 @@ func TestGlobFilesetWithRelativeRoot(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.Equal(t, len(files), len(entries)) } @@ -81,7 +86,7 @@ func TestGlobFilesetRecursively(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } @@ -102,7 +107,7 @@ func TestGlobFilesetDir(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } @@ -123,7 +128,7 @@ func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) { }) require.NoError(t, err) - files, err := g.All() + files, err := g.Files() require.NoError(t, err) require.ElementsMatch(t, entries, collectRelativePaths(files)) } diff --git a/libs/fileset/testdata/dir1/a b/libs/fileset/testdata/dir1/a new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir1/b b/libs/fileset/testdata/dir1/b new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir2/a b/libs/fileset/testdata/dir2/a new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir2/b b/libs/fileset/testdata/dir2/b new file mode 100644 index 000000000..e69de29bb diff --git a/libs/fileset/testdata/dir3/a b/libs/fileset/testdata/dir3/a new file mode 120000 index 000000000..5ac5651e9 --- /dev/null +++ b/libs/fileset/testdata/dir3/a @@ -0,0 +1 @@ +../dir1/a \ No newline at end of file diff --git a/libs/git/fileset.go b/libs/git/fileset.go index f1986aa20..bb1cd4692 100644 --- a/libs/git/fileset.go +++ b/libs/git/fileset.go @@ -7,15 +7,15 @@ import ( // FileSet is Git repository aware implementation of [fileset.FileSet]. // It forces checking if gitignore files have been modified every -// time a call to [FileSet.All] is made. +// time a call to [FileSet.Files] is made. type FileSet struct { fileset *fileset.FileSet view *View } // NewFileSet returns [FileSet] for the Git repository located at `root`. -func NewFileSet(root vfs.Path) (*FileSet, error) { - fs := fileset.New(root) +func NewFileSet(root vfs.Path, paths ...[]string) (*FileSet, error) { + fs := fileset.New(root, paths...) v, err := NewView(root) if err != nil { return nil, err @@ -35,9 +35,9 @@ func (f *FileSet) IgnoreDirectory(dir string) (bool, error) { return f.view.IgnoreDirectory(dir) } -func (f *FileSet) All() ([]fileset.File, error) { +func (f *FileSet) Files() ([]fileset.File, error) { f.view.repo.taintIgnoreRules() - return f.fileset.All() + return f.fileset.Files() } func (f *FileSet) EnsureValidGitIgnoreExists() error { diff --git a/libs/git/fileset_test.go b/libs/git/fileset_test.go index 4e6172bfd..37f3611d1 100644 --- a/libs/git/fileset_test.go +++ b/libs/git/fileset_test.go @@ -15,7 +15,7 @@ import ( func testFileSetAll(t *testing.T, root string) { fileSet, err := NewFileSet(vfs.MustNew(root)) require.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) require.Len(t, files, 3) assert.Equal(t, path.Join("a", "b", "world.txt"), files[0].Relative) @@ -37,7 +37,7 @@ func TestFileSetNonCleanRoot(t *testing.T) { // This should yield the same result as above test. fileSet, err := NewFileSet(vfs.MustNew("./testdata/../testdata")) require.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) assert.Len(t, files, 3) } diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index 0b7c04d6d..582a88479 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -12,27 +12,69 @@ import ( "github.com/databricks/databricks-sdk-go/service/workspace" ) +// FileInfoWithWorkspaceObjectInfo is an interface implemented by [fs.FileInfo] values that +// contain a file's underlying [workspace.ObjectInfo]. +// +// This may be the case when working with a [filer.Filer] backed by the workspace API. +// For these files we do not need to read a file's header to know if it is a notebook; +// we can use the [workspace.ObjectInfo] value directly. +type FileInfoWithWorkspaceObjectInfo interface { + WorkspaceObjectInfo() workspace.ObjectInfo +} + // Maximum length in bytes of the notebook header. const headerLength = 32 -// readHeader reads the first N bytes from a file. -func readHeader(fsys fs.FS, name string) ([]byte, error) { +// file wraps an fs.File and implements a few helper methods such that +// they don't need to be inlined in the [DetectWithFS] function below. +type file struct { + f fs.File +} + +func openFile(fsys fs.FS, name string) (*file, error) { f, err := fsys.Open(name) if err != nil { return nil, err } - defer f.Close() + return &file{f: f}, nil +} +func (f file) close() error { + return f.f.Close() +} + +func (f file) readHeader() (string, error) { // Scan header line with some padding. var buf = make([]byte, headerLength) - n, err := f.Read([]byte(buf)) + n, err := f.f.Read([]byte(buf)) if err != nil && err != io.EOF { - return nil, err + return "", err } // Trim buffer to actual read bytes. - return buf[:n], nil + buf = buf[:n] + + // Read the first line from the buffer. + scanner := bufio.NewScanner(bytes.NewReader(buf)) + scanner.Scan() + return scanner.Text(), nil +} + +// getObjectInfo returns the [workspace.ObjectInfo] for the file if it is +// part of the [fs.FileInfo] value returned by the [fs.Stat] call. +func (f file) getObjectInfo() (oi workspace.ObjectInfo, ok bool, err error) { + stat, err := f.f.Stat() + if err != nil { + return workspace.ObjectInfo{}, false, err + } + + // Use object info if available. + if i, ok := stat.(FileInfoWithWorkspaceObjectInfo); ok { + return i.WorkspaceObjectInfo(), true, nil + } + + return workspace.ObjectInfo{}, false, nil } // Detect returns whether the file at path is a Databricks notebook. @@ -40,13 +82,27 @@ func readHeader(fsys fs.FS, name string) ([]byte, error) { func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { header := "" - buf, err := readHeader(fsys, name) + f, err := openFile(fsys, name) + if err != nil { + return false, "", err + } + + defer f.close() + + // Use object info if available. + oi, ok, err := f.getObjectInfo() + if err != nil { + return false, "", err + } + if ok { + return oi.ObjectType == workspace.ObjectTypeNotebook, oi.Language, nil + } + + // Read the first line of the file. + fileHeader, err := f.readHeader() if err != nil { return false, "", err } - scanner := bufio.NewScanner(bytes.NewReader(buf)) - scanner.Scan() - fileHeader := scanner.Text() // Determine which header to expect based on filename extension. ext := strings.ToLower(filepath.Ext(name)) diff --git a/libs/notebook/detect_test.go b/libs/notebook/detect_test.go index fd3337579..ad89d6dd5 100644 --- a/libs/notebook/detect_test.go +++ b/libs/notebook/detect_test.go @@ -99,3 +99,21 @@ func TestDetectFileWithLongHeader(t *testing.T) { require.NoError(t, err) assert.False(t, nb) } + +func TestDetectWithObjectInfo(t *testing.T) { + fakeFS := &fakeFS{ + fakeFile{ + fakeFileInfo{ + workspace.ObjectInfo{ + ObjectType: workspace.ObjectTypeNotebook, + Language: workspace.LanguagePython, + }, + }, + }, + } + + nb, lang, err := DetectWithFS(fakeFS, "doesntmatter") + require.NoError(t, err) + assert.True(t, nb) + assert.Equal(t, workspace.LanguagePython, lang) +} diff --git a/libs/notebook/fakefs_test.go b/libs/notebook/fakefs_test.go new file mode 100644 index 000000000..4ac135dd4 --- /dev/null +++ b/libs/notebook/fakefs_test.go @@ -0,0 +1,77 @@ +package notebook + +import ( + "fmt" + "io/fs" + "time" + + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type fakeFS struct { + fakeFile +} + +type fakeFile struct { + fakeFileInfo +} + +func (f fakeFile) Close() error { + return nil +} + +func (f fakeFile) Read(p []byte) (n int, err error) { + return 0, fmt.Errorf("not implemented") +} + +func (f fakeFile) Stat() (fs.FileInfo, error) { + return f.fakeFileInfo, nil +} + +type fakeFileInfo struct { + oi workspace.ObjectInfo +} + +func (f fakeFileInfo) WorkspaceObjectInfo() workspace.ObjectInfo { + return f.oi +} + +func (f fakeFileInfo) Name() string { + return "" +} + +func (f fakeFileInfo) Size() int64 { + return 0 +} + +func (f fakeFileInfo) Mode() fs.FileMode { + return 0 +} + +func (f fakeFileInfo) ModTime() time.Time { + return time.Time{} +} + +func (f fakeFileInfo) IsDir() bool { + return false +} + +func (f fakeFileInfo) Sys() any { + return nil +} + +func (f fakeFS) Open(name string) (fs.File, error) { + return f.fakeFile, nil +} + +func (f fakeFS) Stat(name string) (fs.FileInfo, error) { + panic("not implemented") +} + +func (f fakeFS) ReadDir(name string) ([]fs.DirEntry, error) { + panic("not implemented") +} + +func (f fakeFS) ReadFile(name string) ([]byte, error) { + panic("not implemented") +} diff --git a/libs/python/detect.go b/libs/python/detect.go index b0c1475c0..8fcc7cd9c 100644 --- a/libs/python/detect.go +++ b/libs/python/detect.go @@ -3,9 +3,23 @@ package python import ( "context" "errors" + "fmt" + "io/fs" + "os" "os/exec" + "path/filepath" + "runtime" ) +// DetectExecutable looks up the path to the python3 executable from the PATH +// environment variable. +// +// If virtualenv is activated, executable from the virtualenv is returned, +// because activating virtualenv adds python3 executable on a PATH. +// +// If python3 executable is not found on the PATH, the interpreter with the +// least version that satisfies minimal 3.8 version is returned, e.g. +// python3.10. func DetectExecutable(ctx context.Context) (string, error) { // TODO: add a shortcut if .python-version file is detected somewhere in // the parent directory tree. @@ -32,3 +46,35 @@ func DetectExecutable(ctx context.Context) (string, error) { } return interpreter.Path, nil } + +// DetectVEnvExecutable returns the path to the python3 executable inside venvPath, +// that is not necessarily activated. +// +// If virtualenv is not created, or executable doesn't exist, the error is returned. +func DetectVEnvExecutable(venvPath string) (string, error) { + interpreterPath := filepath.Join(venvPath, "bin", "python3") + if runtime.GOOS == "windows" { + interpreterPath = filepath.Join(venvPath, "Scripts", "python3.exe") + } + + if _, err := os.Stat(interpreterPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("can't find %q, check if virtualenv is created", interpreterPath) + } else { + return "", fmt.Errorf("can't find %q: %w", interpreterPath, err) + } + } + + // pyvenv.cfg must be always present in correctly configured virtualenv, + // read more in https://snarky.ca/how-virtual-environments-work/ + pyvenvPath := filepath.Join(venvPath, "pyvenv.cfg") + if _, err := os.Stat(pyvenvPath); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("expected %q to be virtualenv, but pyvenv.cfg is missing", venvPath) + } else { + return "", fmt.Errorf("can't find %q: %w", pyvenvPath, err) + } + } + + return interpreterPath, nil +} diff --git a/libs/python/detect_test.go b/libs/python/detect_test.go new file mode 100644 index 000000000..78c7067f7 --- /dev/null +++ b/libs/python/detect_test.go @@ -0,0 +1,46 @@ +package python + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetectVEnvExecutable(t *testing.T) { + dir := t.TempDir() + interpreterPath := interpreterPath(dir) + + err := os.Mkdir(filepath.Dir(interpreterPath), 0755) + require.NoError(t, err) + + err = os.WriteFile(interpreterPath, []byte(""), 0755) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(dir, "pyvenv.cfg"), []byte(""), 0755) + require.NoError(t, err) + + executable, err := DetectVEnvExecutable(dir) + + assert.NoError(t, err) + assert.Equal(t, interpreterPath, executable) +} + +func TestDetectVEnvExecutable_badLayout(t *testing.T) { + dir := t.TempDir() + + _, err := DetectVEnvExecutable(dir) + + assert.Errorf(t, err, "can't find %q, check if virtualenv is created", interpreterPath(dir)) +} + +func interpreterPath(venvPath string) string { + if runtime.GOOS == "windows" { + return filepath.Join(venvPath, "Scripts", "python3.exe") + } else { + return filepath.Join(venvPath, "bin", "python3") + } +} diff --git a/libs/sync/snapshot_state_test.go b/libs/sync/snapshot_state_test.go index 92c14e8e0..248e5832c 100644 --- a/libs/sync/snapshot_state_test.go +++ b/libs/sync/snapshot_state_test.go @@ -13,7 +13,7 @@ import ( func TestSnapshotState(t *testing.T) { fileSet := fileset.New(vfs.MustNew("./testdata/sync-fileset")) - files, err := fileSet.All() + files, err := fileSet.Files() require.NoError(t, err) // Assert initial contents of the fileset diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go index 050b5d965..b7830406d 100644 --- a/libs/sync/snapshot_test.go +++ b/libs/sync/snapshot_test.go @@ -47,7 +47,7 @@ func TestDiff(t *testing.T) { defer f2.Close(t) // New files are put - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -62,7 +62,7 @@ func TestDiff(t *testing.T) { // world.txt is editted f2.Overwrite(t, "bunnies are cute.") assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -77,7 +77,7 @@ func TestDiff(t *testing.T) { // hello.txt is deleted f1.Remove(t) assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -113,7 +113,7 @@ func TestSymlinkDiff(t *testing.T) { err = os.Symlink(filepath.Join(projectDir, "foo"), filepath.Join(projectDir, "bar")) assert.NoError(t, err) - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -141,7 +141,7 @@ func TestFolderDiff(t *testing.T) { defer f1.Close(t) f1.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -153,7 +153,7 @@ func TestFolderDiff(t *testing.T) { assert.Contains(t, change.put, "foo/bar.py") f1.Remove(t) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -184,7 +184,7 @@ func TestPythonNotebookDiff(t *testing.T) { defer foo.Close(t) // Case 1: notebook foo.py is uploaded - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) foo.Overwrite(t, "# Databricks notebook source\nprint(\"abc\")") change, err := state.diff(ctx, files) @@ -199,7 +199,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 2: notebook foo.py is converted to python script by removing // magic keyword foo.Overwrite(t, "print(\"abc\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -213,7 +213,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 3: Python script foo.py is converted to a databricks notebook foo.Overwrite(t, "# Databricks notebook source\nprint(\"def\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -228,7 +228,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Case 4: Python notebook foo.py is deleted, and its remote name is used in change.delete foo.Remove(t) assert.NoError(t, err) - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) @@ -260,7 +260,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { defer pythonFoo.Close(t) vanillaFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo")) defer vanillaFoo.Close(t) - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -271,7 +271,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { // errors out because they point to the same destination pythonFoo.Overwrite(t, "# Databricks notebook source\nprint(\"def\")") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.ErrorContains(t, err, "both foo and foo.py point to the same remote file location foo. Please remove one of them from your local project") @@ -296,7 +296,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { pythonFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.py")) defer pythonFoo.Close(t) pythonFoo.Overwrite(t, "# Databricks notebook source\n") - files, err := fileSet.All() + files, err := fileSet.Files() assert.NoError(t, err) change, err := state.diff(ctx, files) assert.NoError(t, err) @@ -308,7 +308,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { sqlFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.sql")) defer sqlFoo.Close(t) sqlFoo.Overwrite(t, "-- Databricks notebook source\n") - files, err = fileSet.All() + files, err = fileSet.Files() assert.NoError(t, err) change, err = state.diff(ctx, files) assert.NoError(t, err) diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 3d5bc61ec..9eaebf2ad 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -16,10 +16,12 @@ import ( ) type SyncOptions struct { - LocalPath vfs.Path + LocalRoot vfs.Path + Paths []string + Include []string + Exclude []string + RemotePath string - Include []string - Exclude []string Full bool @@ -51,7 +53,7 @@ type Sync struct { // New initializes and returns a new [Sync] instance. func New(ctx context.Context, opts SyncOptions) (*Sync, error) { - fileSet, err := git.NewFileSet(opts.LocalPath) + fileSet, err := git.NewFileSet(opts.LocalRoot, opts.Paths) if err != nil { return nil, err } @@ -61,12 +63,12 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } - includeFileSet, err := fileset.NewGlobSet(opts.LocalPath, opts.Include) + includeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Include) if err != nil { return nil, err } - excludeFileSet, err := fileset.NewGlobSet(opts.LocalPath, opts.Exclude) + excludeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Exclude) if err != nil { return nil, err } @@ -195,14 +197,14 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) { all := set.NewSetF(func(f fileset.File) string { return f.Relative }) - gitFiles, err := s.fileSet.All() + gitFiles, err := s.fileSet.Files() if err != nil { log.Errorf(ctx, "cannot list files: %s", err) return nil, err } all.Add(gitFiles...) - include, err := s.includeFileSet.All() + include, err := s.includeFileSet.Files() if err != nil { log.Errorf(ctx, "cannot list include files: %s", err) return nil, err @@ -210,7 +212,7 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) { all.Add(include...) - exclude, err := s.excludeFileSet.All() + exclude, err := s.excludeFileSet.Files() if err != nil { log.Errorf(ctx, "cannot list exclude files: %s", err) return nil, err diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index 292586e8d..2d800f466 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -2,70 +2,32 @@ package sync import ( "context" - "os" - "path/filepath" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) -func createFile(dir string, name string) error { - f, err := os.Create(filepath.Join(dir, name)) - if err != nil { - return err - } - - return f.Close() -} - func setupFiles(t *testing.T) string { dir := t.TempDir() - err := createFile(dir, "a.go") - require.NoError(t, err) - - err = createFile(dir, "b.go") - require.NoError(t, err) - - err = createFile(dir, "ab.go") - require.NoError(t, err) - - err = createFile(dir, "abc.go") - require.NoError(t, err) - - err = createFile(dir, "c.go") - require.NoError(t, err) - - err = createFile(dir, "d.go") - require.NoError(t, err) - - dbDir := filepath.Join(dir, ".databricks") - err = os.Mkdir(dbDir, 0755) - require.NoError(t, err) - - err = createFile(dbDir, "e.go") - require.NoError(t, err) - - testDir := filepath.Join(dir, "test") - err = os.Mkdir(testDir, 0755) - require.NoError(t, err) - - sub1 := filepath.Join(testDir, "sub1") - err = os.Mkdir(sub1, 0755) - require.NoError(t, err) - - err = createFile(sub1, "f.go") - require.NoError(t, err) - - sub2 := filepath.Join(sub1, "sub2") - err = os.Mkdir(sub2, 0755) - require.NoError(t, err) - - err = createFile(sub2, "g.go") - require.NoError(t, err) + for _, f := range []([]string){ + []string{dir, "a.go"}, + []string{dir, "b.go"}, + []string{dir, "ab.go"}, + []string{dir, "abc.go"}, + []string{dir, "c.go"}, + []string{dir, "d.go"}, + []string{dir, ".databricks", "e.go"}, + []string{dir, "test", "sub1", "f.go"}, + []string{dir, "test", "sub1", "sub2", "g.go"}, + []string{dir, "test", "sub1", "sub2", "h.txt"}, + } { + testutil.Touch(t, f...) + } return dir } @@ -97,7 +59,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 9) + require.Equal(t, len(fileList), 10) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -115,9 +77,9 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 1) + require.Equal(t, len(fileList), 2) - inc, err = fileset.NewGlobSet(root, []string{".databricks/*"}) + inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) excl, err = fileset.NewGlobSet(root, []string{}) @@ -133,7 +95,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 10) + require.Equal(t, len(fileList), 11) } func TestRecursiveExclude(t *testing.T) { @@ -165,3 +127,34 @@ func TestRecursiveExclude(t *testing.T) { require.NoError(t, err) require.Equal(t, len(fileList), 7) } + +func TestNegateExclude(t *testing.T) { + ctx := context.Background() + + dir := setupFiles(t) + root := vfs.MustNew(dir) + fileSet, err := git.NewFileSet(root) + require.NoError(t, err) + + err = fileSet.EnsureValidGitIgnoreExists() + require.NoError(t, err) + + inc, err := fileset.NewGlobSet(root, []string{}) + require.NoError(t, err) + + excl, err := fileset.NewGlobSet(root, []string{"./*", "!*.txt"}) + require.NoError(t, err) + + s := &Sync{ + SyncOptions: &SyncOptions{}, + + fileSet: fileSet, + includeFileSet: inc, + excludeFileSet: excl, + } + + fileList, err := s.GetFileList(ctx) + require.NoError(t, err) + require.Equal(t, len(fileList), 1) + require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt") +} diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index ca7ec46e9..cc2ca83c5 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -57,7 +57,7 @@ func (s *Sync) applyMkdir(ctx context.Context, localName string) error { func (s *Sync) applyPut(ctx context.Context, localName string) error { s.notifyProgress(ctx, EventActionPut, localName, 0.0) - localFile, err := s.LocalPath.Open(localName) + localFile, err := s.LocalRoot.Open(localName) if err != nil { return err } diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 1af2e5f5a..73b47f289 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -3,59 +3,70 @@ package template import ( "context" "fmt" + "path/filepath" "testing" "text/template" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testConfig(t *testing.T) *config { - c, err := newConfig(context.Background(), "./testdata/config-test-schema/test-schema.json") - require.NoError(t, err) - return c -} - func TestTemplateConfigAssignValuesFromFile(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file" - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "hello", c.values["string_val"]) -} - -func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { - c := testConfig(t) - - err := c.assignValuesFromFile("./testdata/config-assign-from-file-invalid-int/config.json") - assert.EqualError(t, err, "failed to load config from file ./testdata/config-assign-from-file-invalid-int/config.json: failed to parse property int_val: cannot convert \"abc\" to an integer") + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } } func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + c.values = map[string]any{ "string_val": "this-is-not-overwritten", } - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) + } +} - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) +func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { + testDir := "./testdata/config-assign-from-file-invalid-int" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + assert.EqualError(t, err, fmt.Sprintf("failed to load config from file %s: failed to parse property int_val: cannot convert \"abc\" to an integer", filepath.Join(testDir, "config.json"))) } func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file-unknown-property" - err := c.assignValuesFromFile("./testdata/config-assign-from-file-unknown-property/config.json") + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) assert.NoError(t, err) // assert only the known property is loaded @@ -63,37 +74,66 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te assert.Equal(t, "i am a known property", c.values["string_val"]) } -func TestTemplateConfigAssignDefaultValues(t *testing.T) { - c := testConfig(t) +func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-default-value" ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", t.TempDir()) + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir()) require.NoError(t, err) err = c.assignDefaultValues(r) - assert.NoError(t, err) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } +} - assert.Len(t, c.values, 2) - assert.Equal(t, "my_file", c.values["string_val"]) - assert.Equal(t, int64(123), c.values["int_val"]) +func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-templated-default-value" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir()) + require.NoError(t, err) + + // Note: only the string value is templated. + // The JSON schema package doesn't allow using a string default for integer types. + err = c.assignDefaultValues(r) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "world", c.values["string_val"]) + } } func TestTemplateConfigValidateValuesDefined(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.0, "bool_val": false, } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. no value provided for required property string_val") } func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.1, @@ -101,12 +141,15 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.NoError(t, err) } func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "unknown_prop": 1, "int_val": 1, @@ -115,12 +158,15 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. property unknown_prop is not defined in the schema") } func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": "this-should-be-an-int", "float_val": 1.1, @@ -128,7 +174,7 @@ func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. incorrect type for property int_val: expected type integer, but value is \"this-should-be-an-int\"") } @@ -224,19 +270,6 @@ func TestTemplateEnumValidation(t *testing.T) { assert.NoError(t, c.validate()) } -func TestAssignDefaultValuesWithTemplatedDefaults(t *testing.T) { - c := testConfig(t) - ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/templated-defaults/template", "./testdata/templated-defaults/library", t.TempDir()) - require.NoError(t, err) - - err = c.assignDefaultValues(r) - assert.NoError(t, err) - assert.Equal(t, "my_file", c.values["string_val"]) -} - func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) { _, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json") assert.EqualError(t, err, "template property property-without-description is missing a description") diff --git a/libs/template/helpers.go b/libs/template/helpers.go index b3dea329e..1dfe74d73 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -14,6 +14,8 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/iam" + + "github.com/google/uuid" ) type ErrFail struct { @@ -51,6 +53,10 @@ func loadHelpers(ctx context.Context) template.FuncMap { "random_int": func(n int) int { return rand.Intn(n) }, + // Alias for https://pkg.go.dev/github.com/google/uuid#New. Returns, as a string, a UUID which is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC 4122. + "uuid": func() string { + return uuid.New().String() + }, // A key value pair. This is used with the map function to generate maps // to use inside a template "pair": func(k string, v any) pair { diff --git a/libs/template/helpers_test.go b/libs/template/helpers_test.go index c0848c8d0..8cc7b928e 100644 --- a/libs/template/helpers_test.go +++ b/libs/template/helpers_test.go @@ -69,6 +69,23 @@ func TestTemplateRandIntFunction(t *testing.T) { assert.Empty(t, err) } +func TestTemplateUuidFunction(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + + ctx = root.SetWorkspaceClient(ctx, nil) + helpers := loadHelpers(ctx) + r, err := newRenderer(ctx, nil, helpers, "./testdata/uuid/template", "./testdata/uuid/library", tmpDir) + require.NoError(t, err) + + err = r.walk() + assert.NoError(t, err) + + assert.Len(t, r.files, 1) + uuid := strings.TrimSpace(string(r.files[0].(*inMemoryFile).content)) + assert.Regexp(t, "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", uuid) +} + func TestTemplateUrlFunction(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index a8678a525..92133c5fe 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -16,6 +16,7 @@ import ( bundleConfig "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go" @@ -655,15 +656,27 @@ func TestRendererFileTreeRendering(t *testing.T) { func TestRendererSubTemplateInPath(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", tmpDir) + // Copy the template directory to a temporary directory where we can safely include a templated file path. + // These paths include characters that are forbidden in Go modules, so we can't use the testdata directory. + // Also see https://github.com/databricks/cli/pull/1671. + templateDir := t.TempDir() + testutil.CopyDirectory(t, "./testdata/template-in-path", templateDir) + + // Use a backtick-quoted string; double quotes are a reserved character for Windows paths: + // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file. + testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}")) + + tmpDir := t.TempDir() + r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir) require.NoError(t, err) err = r.walk() require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), r.files[0].DstPath().absPath()) - assert.Equal(t, "my_directory/my_file", r.files[0].DstPath().relPath) + if assert.Len(t, r.files, 2) { + f := r.files[1] + assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), f.DstPath().absPath()) + assert.Equal(t, "my_directory/my_file", f.DstPath().relPath) + } } diff --git a/libs/template/templates/default-sql/databricks_template_schema.json b/libs/template/templates/default-sql/databricks_template_schema.json index 329f91962..113cbef64 100644 --- a/libs/template/templates/default-sql/databricks_template_schema.json +++ b/libs/template/templates/default-sql/databricks_template_schema.json @@ -13,7 +13,7 @@ "type": "string", "pattern": "^/sql/.\\../warehouses/[a-z0-9]+$", "pattern_match_failure_message": "Path must be of the form /sql/1.0/warehouses/", - "description": "\nPlease provide the HTTP Path of the SQL warehouse you would like to use with dbt during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]", + "description": "\nPlease provide the HTTP Path of the SQL warehouse you would like to use during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]", "order": 2 }, "default_catalog": { diff --git a/libs/template/testdata/config-assign-from-default-value/schema.json b/libs/template/testdata/config-assign-from-default-value/schema.json new file mode 100644 index 000000000..259bb9a7f --- /dev/null +++ b/libs/template/testdata/config-assign-from-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "hello" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-invalid-int/schema.json b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-unknown-property/schema.json b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file/schema.json b/libs/template/testdata/config-assign-from-file/schema.json new file mode 100644 index 000000000..80c44d6d9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-file/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/schema.json b/libs/template/testdata/config-assign-from-templated-default-value/schema.json new file mode 100644 index 000000000..fe664430b --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "{{ template \"string_val\" }}" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl new file mode 100644 index 000000000..41c50d7e5 --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl @@ -0,0 +1,3 @@ +{{define "string_val" -}} +world +{{- end}} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep b/libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/config-test-schema/test-schema.json b/libs/template/testdata/config-test-schema/test-schema.json index 10f8652f4..80c44d6d9 100644 --- a/libs/template/testdata/config-test-schema/test-schema.json +++ b/libs/template/testdata/config-test-schema/test-schema.json @@ -2,8 +2,7 @@ "properties": { "int_val": { "type": "integer", - "description": "This is an integer value", - "default": 123 + "description": "This is an integer value" }, "float_val": { "type": "number", @@ -15,8 +14,7 @@ }, "string_val": { "type": "string", - "description": "This is a string value", - "default": "{{template \"file_name\"}}" + "description": "This is a string value" } } } diff --git a/libs/template/testdata/empty/library/.gitkeep b/libs/template/testdata/empty/library/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/empty/template/.gitkeep b/libs/template/testdata/empty/template/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/template-in-path/template/.gitkeep b/libs/template/testdata/template-in-path/template/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl b/libs/template/testdata/templated-defaults/library/my_funcs.tmpl deleted file mode 100644 index 3415ad774..000000000 --- a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -{{define "dir_name" -}} -my_directory -{{- end}} - -{{define "file_name" -}} -my_file -{{- end}} diff --git a/libs/template/testdata/uuid/template/hello.tmpl b/libs/template/testdata/uuid/template/hello.tmpl new file mode 100644 index 000000000..178c2a9c4 --- /dev/null +++ b/libs/template/testdata/uuid/template/hello.tmpl @@ -0,0 +1 @@ +{{print (uuid)}} diff --git a/libs/terraform/plan.go b/libs/terraform/plan.go index 22fea6206..36383cc24 100644 --- a/libs/terraform/plan.go +++ b/libs/terraform/plan.go @@ -1,13 +1,44 @@ package terraform +import "strings" + type Plan struct { // Path to the plan Path string - // Holds whether the user can consented to destruction. Either by interactive - // confirmation or by passing a command line flag - ConfirmApply bool - // If true, the plan is empty and applying it will not do anything IsEmpty bool } + +type Action struct { + // Type and name of the resource + ResourceType string `json:"resource_type"` + ResourceName string `json:"resource_name"` + + Action ActionType `json:"action"` +} + +func (a Action) String() string { + // terraform resources have the databricks_ prefix, which is not needed. + rtype := strings.TrimPrefix(a.ResourceType, "databricks_") + return strings.Join([]string{" ", string(a.Action), rtype, a.ResourceName}, " ") +} + +func (c Action) IsInplaceSupported() bool { + return false +} + +// These enum values correspond to action types defined in the tfjson library. +// "recreate" maps to the tfjson.Actions.Replace() function. +// "update" maps to tfjson.Actions.Update() and so on. source: +// https://github.com/hashicorp/terraform-json/blob/0104004301ca8e7046d089cdc2e2db2179d225be/action.go#L14 +type ActionType string + +const ( + ActionTypeCreate ActionType = "create" + ActionTypeDelete ActionType = "delete" + ActionTypeUpdate ActionType = "update" + ActionTypeNoOp ActionType = "no-op" + ActionTypeRead ActionType = "read" + ActionTypeRecreate ActionType = "recreate" +) diff --git a/libs/textutil/textutil.go b/libs/textutil/textutil.go index a5d17d55f..ee9b0f0f1 100644 --- a/libs/textutil/textutil.go +++ b/libs/textutil/textutil.go @@ -1,6 +1,7 @@ package textutil import ( + "regexp" "strings" "unicode" ) @@ -9,7 +10,14 @@ import ( // including spaces and dots, which are not supported in e.g. experiment names or YAML keys. func NormalizeString(name string) string { name = strings.ToLower(name) - return strings.Map(replaceNonAlphanumeric, name) + s := strings.Map(replaceNonAlphanumeric, name) + + // replacing multiple underscores with a single one + re := regexp.MustCompile(`_+`) + s = re.ReplaceAllString(s, "_") + + // removing leading and trailing underscores + return strings.Trim(s, "_") } func replaceNonAlphanumeric(r rune) rune { diff --git a/libs/textutil/textutil_test.go b/libs/textutil/textutil_test.go index fb8bf0b60..f6834a1ef 100644 --- a/libs/textutil/textutil_test.go +++ b/libs/textutil/textutil_test.go @@ -46,6 +46,10 @@ func TestNormalizeString(t *testing.T) { { input: "TestTestTest", expected: "testtesttest", + }, + { + input: ".test//test..test", + expected: "test_test_test", }} for _, c := range cases { diff --git a/main.go b/main.go index 8c8516d9d..c568e6adb 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,16 @@ package main import ( "context" + "os" "github.com/databricks/cli/cmd" "github.com/databricks/cli/cmd/root" ) func main() { - root.Execute(cmd.New(context.Background())) + ctx := context.Background() + err := root.Execute(ctx, cmd.New(ctx)) + if err != nil { + os.Exit(1) + } } diff --git a/main_test.go b/main_test.go index 34ecdca0f..dea82e9b9 100644 --- a/main_test.go +++ b/main_test.go @@ -2,11 +2,14 @@ package main import ( "context" + "io/fs" + "path/filepath" "testing" "github.com/databricks/cli/cmd" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" ) func TestCommandsDontUseUnderscoreInName(t *testing.T) { @@ -23,3 +26,25 @@ func TestCommandsDontUseUnderscoreInName(t *testing.T) { queue = append(queue[1:], cmd.Commands()...) } } + +func TestFilePath(t *testing.T) { + // To import this repository as a library, all files must match the + // file path constraints made by Go. This test ensures that all files + // in the repository have a valid file path. + // + // See https://github.com/databricks/cli/issues/1629 + // + err := filepath.WalkDir(".", func(path string, _ fs.DirEntry, err error) error { + switch path { + case ".": + return nil + case ".git": + return filepath.SkipDir + } + if assert.NoError(t, err) { + assert.NoError(t, module.CheckFilePath(filepath.ToSlash(path))) + } + return nil + }) + assert.NoError(t, err) +}