diff --git a/.codegen.json b/.codegen.json index 077d072bb..4524ab55d 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,6 +11,7 @@ "toolchain": { "required": ["go"], "post_generate": [ + "go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build", "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index ffd6f58dd..2d9cb6d86 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -6f6b1371e640f2dfeba72d365ac566368656f6b6 \ No newline at end of file +cf9c61453990df0f9453670f2fe68e1b128647a2 \ No newline at end of file diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index 281dfd6eb..b489a0b0a 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -5,6 +5,7 @@ package {{(.TrimPrefix "account").SnakeName}} import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/cmd/root" "github.com/databricks/databricks-sdk-go/service/{{.Package.Name}}" "github.com/spf13/cobra" @@ -231,9 +232,15 @@ func new{{.PascalName}}() *cobra.Command { {{- if .Request }} {{ if .CanUseJson }} if cmd.Flags().Changed("json") { - err = {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req) - if err != nil { - return err + diags := {{.CamelName}}Json.Unmarshal(&{{.CamelName}}Req) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } }{{end}}{{ if .MustUseJson }}else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/.gitattributes b/.gitattributes index 2470eb33d..ae10198bb 100755 --- a/.gitattributes +++ b/.gitattributes @@ -54,6 +54,7 @@ cmd/workspace/dashboards/dashboards.go linguist-generated=true cmd/workspace/data-sources/data-sources.go linguist-generated=true cmd/workspace/default-namespace/default-namespace.go linguist-generated=true cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true +cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go linguist-generated=true cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true diff --git a/.github/workflows/external-message.yml b/.github/workflows/external-message.yml new file mode 100644 index 000000000..a8596e246 --- /dev/null +++ b/.github/workflows/external-message.yml @@ -0,0 +1,114 @@ +name: PR Comment + +# WARNING: +# THIS WORKFLOW ALWAYS RUNS FOR EXTERNAL CONTRIBUTORS WITHOUT ANY APPROVAL. +# THIS WORKFLOW RUNS FROM MAIN BRANCH, NOT FROM THE PR BRANCH. +# DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR. + +on: + pull_request_target: + types: [opened, reopened, synchronize] + branches: + - main + + +jobs: + comment-on-pr: + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + # NOTE: The following checks may not be accurate depending on Org or Repo settings. + - name: Check user and potential secret access + id: check-secrets-access + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + USER_LOGIN="${{ github.event.pull_request.user.login }}" + REPO_OWNER="${{ github.repository_owner }}" + REPO_NAME="${{ github.event.repository.name }}" + + echo "Pull request opened by: $USER_LOGIN" + + # Check if PR is from a fork + IS_FORK=$([[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]] && echo "true" || echo "false") + + HAS_ACCESS="false" + + # Check user's permission level on the repository + USER_PERMISSION=$(gh api repos/$REPO_OWNER/$REPO_NAME/collaborators/$USER_LOGIN/permission --jq '.permission') + + if [[ "$USER_PERMISSION" == "admin" || "$USER_PERMISSION" == "write" ]]; then + HAS_ACCESS="true" + elif [[ "$USER_PERMISSION" == "read" ]]; then + # For read access, we need to check if the user has been explicitly granted secret access + # This information is not directly available via API, so we'll make an assumption + # that read access does not imply secret access + HAS_ACCESS="false" + fi + + # Check if repo owner is an organization + IS_ORG=$(gh api users/$REPO_OWNER --jq '.type == "Organization"') + + if [[ "$IS_ORG" == "true" && "$HAS_ACCESS" == "false" ]]; then + # Check if user is a member of any team with write or admin access to the repo + TEAMS_WITH_ACCESS=$(gh api repos/$REPO_OWNER/$REPO_NAME/teams --jq '.[] | select(.permission == "push" or .permission == "admin") | .slug') + for team in $TEAMS_WITH_ACCESS; do + IS_TEAM_MEMBER=$(gh api orgs/$REPO_OWNER/teams/$team/memberships/$USER_LOGIN --silent && echo "true" || echo "false") + if [[ "$IS_TEAM_MEMBER" == "true" ]]; then + HAS_ACCESS="true" + break + fi + done + fi + + # If it's a fork, set HAS_ACCESS to false regardless of other checks + if [[ "$IS_FORK" == "true" ]]; then + HAS_ACCESS="false" + fi + + echo "has_secrets_access=$HAS_ACCESS" >> $GITHUB_OUTPUT + if [[ "$HAS_ACCESS" == "true" ]]; then + echo "User $USER_LOGIN likely has access to secrets" + else + echo "User $USER_LOGIN likely does not have access to secrets" + fi + + + - uses: actions/checkout@v4 + + - name: Delete old comments + if: steps.check-secrets-access.outputs.has_secrets_access != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Delete previous comment if it exists + previous_comment_ids=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ + --jq '.[] | select(.body | startswith("")) | .id') + echo "Previous comment IDs: $previous_comment_ids" + # Iterate over each comment ID and delete the comment + if [ ! -z "$previous_comment_ids" ]; then + echo "$previous_comment_ids" | while read -r comment_id; do + echo "Deleting comment with ID: $comment_id" + gh api "repos/${{ github.repository }}/issues/comments/$comment_id" -X DELETE + done + fi + + - name: Comment on PR + if: steps.check-secrets-access.outputs.has_secrets_access != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: | + gh pr comment ${{ github.event.pull_request.number }} --body \ + " + Run integration tests manually: + [go/deco-tests-run/cli](https://go/deco-tests-run/cli) + + Inputs: + * PR number: ${{github.event.pull_request.number}} + * Commit SHA: \`${{ env.COMMIT_SHA }}\` + + Checks will be approved automatically on success. + " diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..a958a97c2 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,77 @@ +name: integration + +on: + + pull_request: + types: [opened, synchronize] + + merge_group: + + +jobs: + check-token: + runs-on: ubuntu-latest + outputs: + has_token: ${{ steps.set-token-status.outputs.has_token }} + steps: + - name: Check if GITHUB_TOKEN is set + id: set-token-status + run: | + if [ -z "${{ secrets.GITHUB_TOKEN }}" ]; then + echo "GITHUB_TOKEN is empty. User has no access to tokens." + echo "::set-output name=has_token::false" + else + echo "GITHUB_TOKEN is set. User has no access to tokens." + echo "::set-output name=has_token::true" + fi + + trigger-tests: + runs-on: ubuntu-latest + needs: check-token + if: github.event_name == 'pull_request' && needs.check-token.outputs.has_token == 'true' + environment: "test-trigger-is" + + steps: + - uses: actions/checkout@v4 + + - name: Generate GitHub App Token + id: generate-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }} + private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }} + owner: ${{ secrets.ORG_NAME }} + repositories: ${{secrets.REPO_NAME}} + + - name: Trigger Workflow in Another Repo + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{secrets.REPO_NAME}} \ + --ref main \ + -f pull_request_number=${{ github.event.pull_request.number }} \ + -f commit_sha=${{ github.event.pull_request.head.sha }} + + + + # Statuses and checks apply to specific commits (by hash). + # Enforcement of required checks is done both at the PR level and the merge queue level. + # In case of multiple commits in a single PR, the hash of the squashed commit + # will not match the one for the latest (approved) commit in the PR. + # We auto approve the check for the merge queue for two reasons: + # * Queue times out due to duration of tests. + # * Avoid running integration tests twice, since it was already run at the tip of the branch before squashing. + auto-approve: + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + steps: + - name: Mark Check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + gh api -X POST -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ github.repository }}/statuses/${{ github.sha }} \ + -f 'state=success' \ + -f 'context=Integration Tests Check' diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5f68ac6..863474934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Version changelog +## [Release] Release v0.231.0 + +CLI: + * Added JSON input validation for CLI commands ([#1771](https://github.com/databricks/cli/pull/1771)). + * Support Git worktrees for `sync` ([#1831](https://github.com/databricks/cli/pull/1831)). + +Bundles: + * Add `bundle summary` to display URLs for deployed resources ([#1731](https://github.com/databricks/cli/pull/1731)). + * Added a warning when incorrect permissions used for `/Workspace/Shared` bundle root ([#1821](https://github.com/databricks/cli/pull/1821)). + * Show actionable errors for collaborative deployment scenarios ([#1386](https://github.com/databricks/cli/pull/1386)). + * Fix path to repository-wide exclude file ([#1837](https://github.com/databricks/cli/pull/1837)). + * Fixed typo in converting cluster permissions ([#1826](https://github.com/databricks/cli/pull/1826)). + * Ignore metastore permission error during template generation ([#1819](https://github.com/databricks/cli/pull/1819)). + * Handle normalization of `dyn.KindTime` into an any type ([#1836](https://github.com/databricks/cli/pull/1836)). + * Added support for pip options in environment dependencies ([#1842](https://github.com/databricks/cli/pull/1842)). + * Fix race condition when restarting continuous jobs ([#1849](https://github.com/databricks/cli/pull/1849)). + * Fix pipeline in default-python template not working for certain workspaces ([#1854](https://github.com/databricks/cli/pull/1854)). + * Add "output" flag to the bundle sync command ([#1853](https://github.com/databricks/cli/pull/1853)). + +Internal: + * Move utility functions dealing with IAM to libs/iamutil ([#1820](https://github.com/databricks/cli/pull/1820)). + * Remove unused `IS_OWNER` constant ([#1823](https://github.com/databricks/cli/pull/1823)). + * Assert SDK version is consistent in the CLI generation process ([#1814](https://github.com/databricks/cli/pull/1814)). + * Fixed unmarshalling json input into `interface{}` type ([#1832](https://github.com/databricks/cli/pull/1832)). + * Fix `TestAccFsMkdirWhenFileExistsAtPath` in isolated Azure environments ([#1833](https://github.com/databricks/cli/pull/1833)). + * Add behavioral tests for examples from the YAML spec ([#1835](https://github.com/databricks/cli/pull/1835)). + * Remove Terraform conversion function that's no longer used ([#1840](https://github.com/databricks/cli/pull/1840)). + * Encode assumptions about the dashboards API in a test ([#1839](https://github.com/databricks/cli/pull/1839)). + * Add script to make testing of code on branches easier ([#1844](https://github.com/databricks/cli/pull/1844)). + +API Changes: + * Added `databricks disable-legacy-dbfs` command group. + +OpenAPI commit cf9c61453990df0f9453670f2fe68e1b128647a2 (2024-10-14) +Dependency updates: + * Upgrade TF provider to 1.54.0 ([#1852](https://github.com/databricks/cli/pull/1852)). + * Bump github.com/databricks/databricks-sdk-go from 0.48.0 to 0.49.0 ([#1843](https://github.com/databricks/cli/pull/1843)). + +## [Release] Release v0.230.0 + +Notable changes for Databricks Asset Bundles: + +Workspace paths are automatically prefixed with `/Workspace`. In addition, all usage of path strings such as `/Workspace/${workspace.root_path}/...` in bundle configuration is automatically replaced with `${workspace.root_path}/...` and generates a warning as part of bundle validate. + +More details can be found here: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths + +Bundles: + * Add an error if state files grow bigger than the export limit ([#1795](https://github.com/databricks/cli/pull/1795)). + * Always prepend bundle remote paths with /Workspace ([#1724](https://github.com/databricks/cli/pull/1724)). + * Add resource path field to bundle workspace configuration ([#1800](https://github.com/databricks/cli/pull/1800)). + * Add validation for files with a `.(resource-name).yml` extension ([#1780](https://github.com/databricks/cli/pull/1780)). + +Internal: + * Remove deprecated or readonly fields from the bundle schema ([#1809](https://github.com/databricks/cli/pull/1809)). + +API Changes: + * Changed `databricks git-credentials create`, `databricks git-credentials delete`, `databricks git-credentials get`, `databricks git-credentials list`, `databricks git-credentials update` commands . + * Changed `databricks repos create`, `databricks repos delete`, `databricks repos get`, `databricks repos update` command . + +OpenAPI commit 0c86ea6dbd9a730c24ff0d4e509603e476955ac5 (2024-10-02) +Dependency updates: + * Upgrade TF provider to 1.53.0 ([#1815](https://github.com/databricks/cli/pull/1815)). + * Bump golang.org/x/term from 0.24.0 to 0.25.0 ([#1811](https://github.com/databricks/cli/pull/1811)). + * Bump golang.org/x/text from 0.18.0 to 0.19.0 ([#1812](https://github.com/databricks/cli/pull/1812)). + * Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 ([#1810](https://github.com/databricks/cli/pull/1810)). + ## [Release] Release v0.229.0 Bundles: diff --git a/README.md b/README.md index 51780d0f9..3c238702c 100644 --- a/README.md +++ b/README.md @@ -35,3 +35,6 @@ docker run -e DATABRICKS_HOST=$YOUR_HOST_URL -e DATABRICKS_TOKEN=$YOUR_TOKEN ghc This CLI follows the Databricks Unified Authentication principles. You can find a detailed description at https://github.com/databricks/databricks-sdk-go#authentication. + +## Privacy Notice +Databricks CLI use is subject to the [Databricks License](https://github.com/databricks/cli/blob/main/LICENSE) and [Databricks Privacy Notice](https://www.databricks.com/legal/privacynotice), including any Usage Data provisions. diff --git a/bundle/config/generate/dashboard.go b/bundle/config/generate/dashboard.go new file mode 100644 index 000000000..460140806 --- /dev/null +++ b/bundle/config/generate/dashboard.go @@ -0,0 +1,18 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (dyn.Value, error) { + // The majority of fields of the dashboard struct are read-only. + // We copy the relevant fields manually. + dv := map[string]dyn.Value{ + "display_name": dyn.NewValue(dashboard.DisplayName, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(dashboard.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + return dyn.V(dv), nil +} diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index 406b9b67c..0723c056c 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) { func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", } diags := bundle.Apply(context.Background(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 7cf9a17d7..f82f5db1e 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -3,12 +3,135 @@ package loader import ( "context" "fmt" + "slices" + "sort" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) +func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { + for _, resourceDescription := range config.SupportedResources() { + singularName := resourceDescription.SingularName + + for _, yamlExt := range []string{"yml", "yaml"} { + ext := fmt.Sprintf(".%s.%s", singularName, yamlExt) + if strings.HasSuffix(filePath, ext) { + return validateSingleResourceDefined(configRoot, ext, singularName) + } + } + } + + return nil +} + +func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { + type resource struct { + path dyn.Path + value dyn.Value + typ string + key string + } + + resources := []resource{} + supportedResources := config.SupportedResources() + + // Gather all resources defined in the resources block. + _, err := dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[2].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[1].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + // Gather all resources defined in a target block. + _, err = dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[4].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[3].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + typeMatch := true + seenKeys := map[string]struct{}{} + for _, rr := range resources { + // case: The resource is not of the correct type. + if rr.typ != typ { + typeMatch = false + break + } + + seenKeys[rr.key] = struct{}{} + } + + // Format matches. There's at most one resource defined in the file. + // The resource is also of the correct type. + if typeMatch && len(seenKeys) <= 1 { + return nil + } + + detail := strings.Builder{} + detail.WriteString("The following resources are defined or configured in this file:\n") + lines := []string{} + for _, r := range resources { + lines = append(lines, fmt.Sprintf(" - %s (%s)\n", r.key, r.typ)) + } + // Sort the lines to print to make the output deterministic. + sort.Strings(lines) + // Compact the lines before writing them to the message to remove any duplicate lines. + // This is needed because we do not dedup earlier when gathering the resources + // and it's valid to define the same resource in both the resources and targets block. + lines = slices.Compact(lines) + for _, l := range lines { + detail.WriteString(l) + } + + locations := []dyn.Location{} + paths := []dyn.Path{} + for _, rr := range resources { + locations = append(locations, rr.value.Locations()...) + paths = append(paths, rr.path) + } + // Sort the locations and paths to make the output deterministic. + sort.Slice(locations, func(i, j int) bool { + return locations[i].String() < locations[j].String() + }) + sort.Slice(paths, func(i, j int) bool { + return paths[i].String() < paths[j].String() + }) + + return diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext), + Detail: detail.String(), + Locations: locations, + Paths: paths, + }, + } +} + type processInclude struct { fullPath string relPath string @@ -31,6 +154,13 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos if diags.HasError() { return diags } + + // Add any diagnostics associated with the file format. + diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + if diags.HasError() { + return diags + } + err := b.Config.Merge(this) if err != nil { diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index 2ccd84b31..66c695e17 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -8,13 +8,15 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -33,3 +35,184 @@ func TestProcessInclude(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Workspace.Host) } + +func TestProcessIncludeFormatMatch(t *testing.T) { + for _, fileName := range []string{ + "one_job.job.yml", + "one_pipeline.pipeline.yaml", + "two_job.yml", + "job_and_pipeline.yml", + "multiple_resources.yml", + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + assert.Empty(t, diags) + }) + } +} + +func TestProcessIncludeFormatNotMatch(t *testing.T) { + for fileName, expectedDiags := range map[string]diag.Diagnostics{ + "single_job.pipeline.yaml": { + { + Severity: diag.Recommendation, + Summary: "define a single pipeline in a file with the .pipeline.yaml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.experiment.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single experiment in a file with the .experiment.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "two_jobs.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 7, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + }, + }, + }, + "second_job_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "two_jobs_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 6, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 8, Column: 11}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "multiple_resources.model_serving_endpoint.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single model serving endpoint in a file with the .model_serving_endpoint.yml extension.", + Detail: `The following resources are defined or configured in this file: + - experiment1 (experiment) + - job1 (job) + - job2 (job) + - job3 (job) + - model1 (model) + - model_serving_endpoint1 (model_serving_endpoint) + - pipeline1 (pipeline) + - pipeline2 (pipeline) + - quality_monitor1 (quality_monitor) + - registered_model1 (registered_model) + - schema1 (schema) +`, + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 12, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 14, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 18, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 22, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 24, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 28, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 35, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 39, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 43, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.experiments.experiment1"), + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"), + dyn.MustPathFromString("resources.models.model1"), + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("resources.pipelines.pipeline2"), + dyn.MustPathFromString("resources.schemas.schema1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job3"), + dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"), + dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"), + }, + }, + }, + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_not_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + require.Len(t, diags, 1) + assert.Equal(t, expectedDiags, diags) + }) + } +} diff --git a/bundle/config/loader/testdata/databricks.yml b/bundle/config/loader/testdata/basic/databricks.yml similarity index 100% rename from bundle/config/loader/testdata/databricks.yml rename to bundle/config/loader/testdata/basic/databricks.yml diff --git a/bundle/config/loader/testdata/host.yml b/bundle/config/loader/testdata/basic/host.yml similarity index 100% rename from bundle/config/loader/testdata/host.yml rename to bundle/config/loader/testdata/basic/host.yml diff --git a/bundle/config/loader/testdata/format_match/job_and_pipeline.yml b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_match/multiple_resources.yml b/bundle/config/loader/testdata/format_match/multiple_resources.yml new file mode 100644 index 000000000..dc8e837c6 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/multiple_resources.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_match/one_job.job.yml b/bundle/config/loader/testdata/format_match/one_job.job.yml new file mode 100644 index 000000000..91af87cdc --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_job.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml new file mode 100644 index 000000000..85cb0d7fc --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml @@ -0,0 +1,4 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 diff --git a/bundle/config/loader/testdata/format_match/two_job.yml b/bundle/config/loader/testdata/format_match/two_job.yml new file mode 100644 index 000000000..81ff90a75 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/two_job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml new file mode 100644 index 000000000..dc8e837c6 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml new file mode 100644 index 000000000..628b9879f --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml new file mode 100644 index 000000000..91af87cdc --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml new file mode 100644 index 000000000..81ff90a75 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml new file mode 100644 index 000000000..3b489c1f7 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml @@ -0,0 +1,8 @@ +targets: + target1: + resources: + jobs: + job1: + description: job1 + job2: + description: job2 diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 1fd49206f..d2a1d0c7d 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -212,6 +212,15 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Dashboards: Prefix + for key, dashboard := range r.Dashboards { + if dashboard == nil || dashboard.CreateDashboardRequest == nil { + diags = diags.Extend(diag.Errorf("dashboard %s s is not defined", key)) + continue + } + dashboard.DisplayName = prefix + dashboard.DisplayName + } + return diags } diff --git a/bundle/config/mutator/configure_dashboard_defaults.go b/bundle/config/mutator/configure_dashboard_defaults.go new file mode 100644 index 000000000..36ec279de --- /dev/null +++ b/bundle/config/mutator/configure_dashboard_defaults.go @@ -0,0 +1,70 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type configureDashboardDefaults struct{} + +func ConfigureDashboardDefaults() bundle.Mutator { + return &configureDashboardDefaults{} +} + +func (m *configureDashboardDefaults) Name() string { + return "ConfigureDashboardDefaults" +} + +func (m *configureDashboardDefaults) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("dashboards"), + dyn.AnyKey(), + ) + + // Configure defaults for all dashboards. + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + var err error + v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("parent_path")), dyn.V(b.Config.Workspace.ResourcePath)) + if err != nil { + return dyn.InvalidValue, err + } + v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("embed_credentials")), dyn.V(false)) + if err != nil { + return dyn.InvalidValue, err + } + return v, nil + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} + +func setIfNotExists(v dyn.Value, path dyn.Path, defaultValue dyn.Value) (dyn.Value, error) { + // Get the field at the specified path (if set). + _, err := dyn.GetByPath(v, path) + switch { + case dyn.IsNoSuchKeyError(err): + // OK, we'll set the default value. + break + case dyn.IsCannotTraverseNilError(err): + // Cannot traverse the value, skip it. + return v, nil + case err == nil: + // The field is set, skip it. + return v, nil + default: + // Return the error. + return v, err + } + + // Set the field at the specified path. + return dyn.SetByPath(v, path, defaultValue) +} diff --git a/bundle/config/mutator/configure_dashboard_defaults_test.go b/bundle/config/mutator/configure_dashboard_defaults_test.go new file mode 100644 index 000000000..4804b7159 --- /dev/null +++ b/bundle/config/mutator/configure_dashboard_defaults_test.go @@ -0,0 +1,130 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureDashboardDefaultsParentPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + ResourcePath: "/foo/bar", + }, + Resources: config.Resources{ + Dashboards: map[string]*resources.Dashboard{ + "d1": { + // Empty string is skipped. + // See below for how it is set. + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + ParentPath: "", + }, + }, + "d2": { + // Non-empty string is skipped. + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + ParentPath: "already-set", + }, + }, + "d3": { + // No parent path set. + }, + "d4": nil, + }, + }, + }, + } + + // We can't set an empty string in the typed configuration. + // Do it on the dyn.Value directly. + bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources.dashboards.d1.parent_path", dyn.V("")) + }) + + diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults()) + require.NoError(t, diags.Error()) + + var v dyn.Value + var err error + + // Set to empty string; unchanged. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.parent_path") + if assert.NoError(t, err) { + assert.Equal(t, "", v.MustString()) + } + + // Set to "already-set"; unchanged. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.parent_path") + if assert.NoError(t, err) { + assert.Equal(t, "already-set", v.MustString()) + } + + // Not set; now set to the workspace resource path. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.parent_path") + if assert.NoError(t, err) { + assert.Equal(t, "/foo/bar", v.MustString()) + } + + // No valid dashboard; no change. + _, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.parent_path") + assert.True(t, dyn.IsCannotTraverseNilError(err)) +} + +func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Dashboards: map[string]*resources.Dashboard{ + "d1": { + EmbedCredentials: true, + }, + "d2": { + EmbedCredentials: false, + }, + "d3": { + // No parent path set. + }, + "d4": nil, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults()) + require.NoError(t, diags.Error()) + + var v dyn.Value + var err error + + // Set to true; still true. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials") + if assert.NoError(t, err) { + assert.Equal(t, true, v.MustBool()) + } + + // Set to false; still false. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials") + if assert.NoError(t, err) { + assert.Equal(t, false, v.MustBool()) + } + + // Not set; now false. + v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials") + if assert.NoError(t, err) { + assert.Equal(t, false, v.MustBool()) + } + + // No valid dashboard; no change. + _, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.embed_credentials") + assert.True(t, dyn.IsCannotTraverseNilError(err)) +} diff --git a/bundle/config/mutator/initialize_urls.go b/bundle/config/mutator/initialize_urls.go new file mode 100644 index 000000000..319305912 --- /dev/null +++ b/bundle/config/mutator/initialize_urls.go @@ -0,0 +1,65 @@ +package mutator + +import ( + "context" + "net/url" + "strconv" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type initializeURLs struct { +} + +// InitializeURLs makes sure the URL field of each resource is configured. +// NOTE: since this depends on an extra API call, this mutator adds some extra +// latency. As such, it should only be used when needed. +// This URL field is used for the output of the 'bundle summary' CLI command. +func InitializeURLs() bundle.Mutator { + return &initializeURLs{} +} + +func (m *initializeURLs) Name() string { + return "InitializeURLs" +} + +func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + workspaceId, err := b.WorkspaceClient().CurrentWorkspaceID(ctx) + if err != nil { + return diag.FromErr(err) + } + orgId := strconv.FormatInt(workspaceId, 10) + host := b.WorkspaceClient().Config.CanonicalHostName() + initializeForWorkspace(b, orgId, host) + return nil +} + +func initializeForWorkspace(b *bundle.Bundle, orgId string, host string) error { + baseURL, err := url.Parse(host) + if err != nil { + return err + } + + // Add ?o= only if wasn't in the subdomain already. + // The ?o= is needed when vanity URLs / legacy workspace URLs are used. + // If it's not needed we prefer to leave it out since these URLs are rather + // long for most terminals. + // + // See https://docs.databricks.com/en/workspace/workspace-details.html for + // further reading about the '?o=' suffix. + if !strings.Contains(baseURL.Hostname(), orgId) { + values := baseURL.Query() + values.Add("o", orgId) + baseURL.RawQuery = values.Encode() + } + + for _, group := range b.Config.Resources.AllResources() { + for _, r := range group.Resources { + r.InitializeURL(*baseURL) + } + } + + return nil +} diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go new file mode 100644 index 000000000..61103de80 --- /dev/null +++ b/bundle/config/mutator/initialize_urls_test.go @@ -0,0 +1,140 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/require" +) + +func TestInitializeURLs(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + Host: "https://mycompany.databricks.com/", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + ID: "1", + JobSettings: &jobs.JobSettings{Name: "job1"}, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline1": { + ID: "3", + PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"}, + }, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "experiment1": { + ID: "4", + Experiment: &ml.Experiment{Name: "experiment1"}, + }, + }, + Models: map[string]*resources.MlflowModel{ + "model1": { + ID: "a model uses its name for identifier", + Model: &ml.Model{Name: "a model uses its name for identifier"}, + }, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "servingendpoint1": { + ID: "my_serving_endpoint", + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "my_serving_endpoint", + }, + }, + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "registeredmodel1": { + ID: "8", + CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{ + Name: "my_registered_model", + }, + }, + }, + QualityMonitors: map[string]*resources.QualityMonitor{ + "qualityMonitor1": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "catalog.schema.qualityMonitor1", + }, + }, + }, + Schemas: map[string]*resources.Schema{ + "schema1": { + ID: "catalog.schema", + CreateSchema: &catalog.CreateSchema{ + Name: "schema", + }, + }, + }, + Clusters: map[string]*resources.Cluster{ + "cluster1": { + ID: "1017-103929-vlr7jzcf", + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "cluster1", + }, + }, + }, + Dashboards: map[string]*resources.Dashboard{ + "dashboard1": { + ID: "01ef8d56871e1d50ae30ce7375e42478", + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "My special dashboard", + }, + }, + }, + }, + }, + } + + expectedURLs := map[string]string{ + "job1": "https://mycompany.databricks.com/jobs/1?o=123456", + "pipeline1": "https://mycompany.databricks.com/pipelines/3?o=123456", + "experiment1": "https://mycompany.databricks.com/ml/experiments/4?o=123456", + "model1": "https://mycompany.databricks.com/ml/models/a%20model%20uses%20its%20name%20for%20identifier?o=123456", + "servingendpoint1": "https://mycompany.databricks.com/ml/endpoints/my_serving_endpoint?o=123456", + "registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456", + "qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456", + "schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456", + "cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456", + "dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456", + } + + initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/") + + for _, group := range b.Config.Resources.AllResources() { + for key, r := range group.Resources { + require.Equal(t, expectedURLs[key], r.GetURL(), "Unexpected URL for "+key) + } + } +} + +func TestInitializeURLsWithoutOrgId(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + ID: "1", + JobSettings: &jobs.JobSettings{Name: "job1"}, + }, + }, + }, + }, + } + + initializeForWorkspace(b, "123456", "https://adb-123456.azuredatabricks.net/") + + require.Equal(t, "https://adb-123456.azuredatabricks.net/jobs/1", b.Config.Resources.Jobs["job1"].URL) +} diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 1e99b327c..cab5db1ba 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -5,8 +5,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" ) @@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } b.Config.Workspace.CurrentUser = &config.User{ - ShortName: auth.GetShortUserName(me), + ShortName: iamutil.GetShortUserName(me), User: me, } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 9944f6ffd..44b53681d 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -6,9 +6,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/log" ) @@ -174,7 +174,7 @@ func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Di transformDevelopmentMode(ctx, b) return diags case config.Production: - isPrincipal := auth.IsServicePrincipal(b.Config.Workspace.CurrentUser.UserName) + isPrincipal := iamutil.IsServicePrincipal(b.Config.Workspace.CurrentUser.User) return validateProductionMode(ctx, b, isPrincipal) case "": // No action diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index b0eb57ee1..4346e88fe 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -14,6 +14,7 @@ import ( sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" @@ -123,6 +124,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, + Dashboards: map[string]*resources.Dashboard{ + "dashboard1": { + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "dashboard1", + }, + }, + }, }, }, // Use AWS implementation for testing. @@ -184,6 +192,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Clusters assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) + + // Dashboards + assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index 423bc38e2..0ca71e28e 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -30,50 +30,44 @@ func (m *setRunAs) Name() string { return "SetRunAs" } -type errUnsupportedResourceTypeForRunAs struct { - resourceType string - resourceLocation dyn.Location - currentUser string - runAsUser string +func reportRunAsNotSupported(resourceType string, location dyn.Location, currentUser string, runAsUser string) diag.Diagnostics { + return diag.Diagnostics{{ + Summary: fmt.Sprintf("%s do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: %s. Run as identity: %s.\n"+ + "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", resourceType, currentUser, runAsUser), + Locations: []dyn.Location{location}, + Severity: diag.Error, + }} } -func (e errUnsupportedResourceTypeForRunAs) Error() string { - return fmt.Sprintf("%s are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s. Current identity: %s. Run as identity: %s", e.resourceType, e.resourceLocation, e.currentUser, e.runAsUser) -} +func validateRunAs(b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} -type errBothSpAndUserSpecified struct { - spName string - spLoc dyn.Location - userName string - userLoc dyn.Location -} + neitherSpecifiedErr := diag.Diagnostics{{ + Summary: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, + }} -func (e errBothSpAndUserSpecified) Error() string { - return fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name %q is specified at %s. A user_name %q is defined at %s", e.spName, e.spLoc, e.userName, e.userLoc) -} - -func validateRunAs(b *bundle.Bundle) error { - neitherSpecifiedErr := fmt.Errorf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s", b.Config.GetLocation("run_as")) - // Error if neither service_principal_name nor user_name are specified, but the + // Fail fast if neither service_principal_name nor user_name are specified, but the // run_as section is present. if b.Config.Value().Get("run_as").Kind() == dyn.KindNil { return neitherSpecifiedErr } - // Error if one or both of service_principal_name and user_name are specified, + + // Fail fast if one or both of service_principal_name and user_name are specified, // but with empty values. - if b.Config.RunAs.ServicePrincipalName == "" && b.Config.RunAs.UserName == "" { + runAs := b.Config.RunAs + if runAs.ServicePrincipalName == "" && runAs.UserName == "" { return neitherSpecifiedErr } - // Error if both service_principal_name and user_name are specified - runAs := b.Config.RunAs if runAs.UserName != "" && runAs.ServicePrincipalName != "" { - return errBothSpAndUserSpecified{ - spName: runAs.ServicePrincipalName, - userName: runAs.UserName, - spLoc: b.Config.GetLocation("run_as.service_principal_name"), - userLoc: b.Config.GetLocation("run_as.user_name"), - } + diags = diags.Extend(diag.Diagnostics{{ + Summary: "run_as section cannot specify both user_name and service_principal_name", + Locations: []dyn.Location{b.Config.GetLocation("run_as")}, + Severity: diag.Error, + }}) } identity := runAs.ServicePrincipalName @@ -83,40 +77,50 @@ func validateRunAs(b *bundle.Bundle) error { // All resources are supported if the run_as identity is the same as the current deployment identity. if identity == b.Config.Workspace.CurrentUser.UserName { - return nil + return diags } // DLT pipelines do not support run_as in the API. if len(b.Config.Resources.Pipelines) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "pipelines", - resourceLocation: b.Config.GetLocation("resources.pipelines"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "pipelines", + b.Config.GetLocation("resources.pipelines"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } // Model serving endpoints do not support run_as in the API. if len(b.Config.Resources.ModelServingEndpoints) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "model_serving_endpoints", - resourceLocation: b.Config.GetLocation("resources.model_serving_endpoints"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "model_serving_endpoints", + b.Config.GetLocation("resources.model_serving_endpoints"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } // Monitors do not support run_as in the API. if len(b.Config.Resources.QualityMonitors) > 0 { - return errUnsupportedResourceTypeForRunAs{ - resourceType: "quality_monitors", - resourceLocation: b.Config.GetLocation("resources.quality_monitors"), - currentUser: b.Config.Workspace.CurrentUser.UserName, - runAsUser: identity, - } + diags = diags.Extend(reportRunAsNotSupported( + "quality_monitors", + b.Config.GetLocation("resources.quality_monitors"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) } - return nil + // Dashboards do not support run_as in the API. + if len(b.Config.Resources.Dashboards) > 0 { + diags = diags.Extend(reportRunAsNotSupported( + "dashboards", + b.Config.GetLocation("resources.dashboards"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) + } + + return diags } func setRunAsForJobs(b *bundle.Bundle) { @@ -187,8 +191,9 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { } // Assert the run_as configuration is valid in the context of the bundle - if err := validateRunAs(b); err != nil { - return diag.FromErr(err) + diags := validateRunAs(b) + if diags.HasError() { + return diags } setRunAsForJobs(b) diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index abeea45d0..acb6c3a43 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -33,6 +33,7 @@ func allResourceTypes(t *testing.T) []string { // also update this check when adding a new resource require.Equal(t, []string{ "clusters", + "dashboards", "experiments", "jobs", "model_serving_endpoints", @@ -188,11 +189,9 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { Config: *r, } diags := bundle.Apply(context.Background(), b, SetRunAs()) - assert.Equal(t, diags.Error().Error(), errUnsupportedResourceTypeForRunAs{ - resourceType: rt, - resourceLocation: dyn.Location{}, - currentUser: "alice", - runAsUser: "bob", - }.Error(), "expected run_as with a different identity than the current deployment user to not supported for resources of type: %s", rt) + require.Error(t, diags.Error()) + assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: alice. Run as identity: bob.\n"+ + "See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt) } } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 5f22570e7..82b0b3caa 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -162,6 +162,20 @@ func (t *translateContext) translateNoOp(literal, localFullPath, localRelPath, r return localRelPath, nil } +func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) { + info, err := t.b.SyncRoot.Stat(localRelPath) + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("file %s not found", literal) + } + if err != nil { + return "", fmt.Errorf("unable to determine if %s is a file: %w", localFullPath, err) + } + if info.IsDir() { + return "", fmt.Errorf("expected %s to be a file but found a directory", literal) + } + return localFullPath, nil +} + func (t *translateContext) translateNoOpWithPrefix(literal, localFullPath, localRelPath, remotePath string) (string, error) { if !strings.HasPrefix(localRelPath, ".") { localRelPath = "." + string(filepath.Separator) + localRelPath @@ -215,6 +229,7 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos t.applyJobTranslations, t.applyPipelineTranslations, t.applyArtifactTranslations, + t.applyDashboardTranslations, } { v, err = fn(v) if err != nil { diff --git a/bundle/config/mutator/translate_paths_dashboards.go b/bundle/config/mutator/translate_paths_dashboards.go new file mode 100644 index 000000000..93822a599 --- /dev/null +++ b/bundle/config/mutator/translate_paths_dashboards.go @@ -0,0 +1,28 @@ +package mutator + +import ( + "fmt" + + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, error) { + // Convert the `file_path` field to a local absolute path. + // We load the file at this path and use its contents for the dashboard contents. + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("dashboards"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + key := p[2].Key() + dir, err := v.Location().Directory() + if err != nil { + return dyn.InvalidValue, fmt.Errorf("unable to determine directory for dashboard %s: %w", key, err) + } + + return t.rewriteRelativeTo(p, v, t.retainLocalAbsoluteFilePath, dir, "") + }) +} diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index c03cee73e..9d655b27b 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -699,6 +699,9 @@ func TestTranslatePathJobEnvironments(t *testing.T) { "../dist/env2.whl", "simplejson", "/Workspace/Users/foo@bar.com/test.whl", + "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", + "foobar --extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple", + "https://foo@bar.com/packages/pypi/simple", }, }, }, @@ -719,6 +722,9 @@ func TestTranslatePathJobEnvironments(t *testing.T) { assert.Equal(t, strings.Join([]string{".", "dist", "env2.whl"}, string(os.PathSeparator)), b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) assert.Equal(t, "simplejson", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[2]) assert.Equal(t, "/Workspace/Users/foo@bar.com/test.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[3]) + assert.Equal(t, "--extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple foobar", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[4]) + assert.Equal(t, "foobar --extra-index-url https://name:token@gitlab.com/api/v4/projects/9876/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[5]) + assert.Equal(t, "https://foo@bar.com/packages/pypi/simple", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[6]) } func TestTranslatePathWithComplexVariables(t *testing.T) { diff --git a/bundle/config/resources.go b/bundle/config/resources.go index a3afb7fc3..0affb6ef0 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "net/url" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -20,6 +21,7 @@ type Resources struct { QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` Schemas map[string]*resources.Schema `json:"schemas,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` + Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` } type ConfigResource interface { @@ -30,6 +32,54 @@ type ConfigResource interface { // Terraform equivalent name of the resource. For example "databricks_job" // for jobs and "databricks_pipeline" for pipelines. TerraformResourceName() string + + // GetName returns the in-product name of the resource. + GetName() string + + // GetURL returns the URL of the resource. + GetURL() string + + // InitializeURL initializes the URL field of the resource. + InitializeURL(baseURL url.URL) +} + +// ResourceGroup represents a group of resources of the same type. +// It includes a description of the resource type and a map of resources. +type ResourceGroup struct { + Description ResourceDescription + Resources map[string]ConfigResource +} + +// collectResourceMap collects resources of a specific type into a ResourceGroup. +func collectResourceMap[T ConfigResource]( + description ResourceDescription, + input map[string]T, +) ResourceGroup { + resources := make(map[string]ConfigResource) + for key, resource := range input { + resources[key] = resource + } + return ResourceGroup{ + Description: description, + Resources: resources, + } +} + +// AllResources returns all resources in the bundle grouped by their resource type. +func (r *Resources) AllResources() []ResourceGroup { + descriptions := SupportedResources() + return []ResourceGroup{ + collectResourceMap(descriptions["jobs"], r.Jobs), + collectResourceMap(descriptions["pipelines"], r.Pipelines), + collectResourceMap(descriptions["models"], r.Models), + collectResourceMap(descriptions["experiments"], r.Experiments), + collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints), + collectResourceMap(descriptions["registered_models"], r.RegisteredModels), + collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors), + collectResourceMap(descriptions["schemas"], r.Schemas), + collectResourceMap(descriptions["clusters"], r.Clusters), + collectResourceMap(descriptions["dashboards"], r.Dashboards), + } } func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) { @@ -59,3 +109,79 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) return found[0], nil } + +type ResourceDescription struct { + // Singular and plural name when used to refer to the configuration. + SingularName string + PluralName string + + // Singular and plural title when used in summaries / terminal UI. + SingularTitle string + PluralTitle string +} + +// The keys of the map corresponds to the resource key in the bundle configuration. +func SupportedResources() map[string]ResourceDescription { + return map[string]ResourceDescription{ + "jobs": { + SingularName: "job", + PluralName: "jobs", + SingularTitle: "Job", + PluralTitle: "Jobs", + }, + "pipelines": { + SingularName: "pipeline", + PluralName: "pipelines", + SingularTitle: "Pipeline", + PluralTitle: "Pipelines", + }, + "models": { + SingularName: "model", + PluralName: "models", + SingularTitle: "Model", + PluralTitle: "Models", + }, + "experiments": { + SingularName: "experiment", + PluralName: "experiments", + SingularTitle: "Experiment", + PluralTitle: "Experiments", + }, + "model_serving_endpoints": { + SingularName: "model_serving_endpoint", + PluralName: "model_serving_endpoints", + SingularTitle: "Model Serving Endpoint", + PluralTitle: "Model Serving Endpoints", + }, + "registered_models": { + SingularName: "registered_model", + PluralName: "registered_models", + SingularTitle: "Registered Model", + PluralTitle: "Registered Models", + }, + "quality_monitors": { + SingularName: "quality_monitor", + PluralName: "quality_monitors", + SingularTitle: "Quality Monitor", + PluralTitle: "Quality Monitors", + }, + "schemas": { + SingularName: "schema", + PluralName: "schemas", + SingularTitle: "Schema", + PluralTitle: "Schemas", + }, + "clusters": { + SingularName: "cluster", + PluralName: "clusters", + SingularTitle: "Cluster", + PluralTitle: "Clusters", + }, + "dashboards": { + SingularName: "dashboard", + PluralName: "dashboards", + SingularTitle: "Dashboard", + PluralTitle: "Dashboards", + }, + } +} diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index 632345666..eb0247c6e 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -13,6 +15,7 @@ type Cluster struct { ID string `json:"id,omitempty" bundle:"readonly"` Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` *compute.ClusterSpec } @@ -37,3 +40,19 @@ func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (s *Cluster) TerraformResourceName() string { return "databricks_cluster" } + +func (s *Cluster) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("compute/clusters/%s", s.ID) + s.URL = baseURL.String() +} + +func (s *Cluster) GetName() string { + return s.ClusterName +} + +func (s *Cluster) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/dashboard.go b/bundle/config/resources/dashboard.go new file mode 100644 index 000000000..462dbc564 --- /dev/null +++ b/bundle/config/resources/dashboard.go @@ -0,0 +1,81 @@ +package resources + +import ( + "context" + "fmt" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type Dashboard struct { + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` + + *dashboards.CreateDashboardRequest + + // ========================= + // === Additional fields === + // ========================= + + // SerializedDashboard holds the contents of the dashboard in serialized JSON form. + // We override the field's type from the SDK struct here to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedDashboard any `json:"serialized_dashboard,omitempty"` + + // EmbedCredentials is a flag to indicate if the publisher's credentials should + // be embedded in the published dashboard. These embedded credentials will be used + // to execute the published dashboard's queries. + // + // Defaults to false if not set. + EmbedCredentials bool `json:"embed_credentials,omitempty"` + + // FilePath points to the local `.lvdash.json` file containing the dashboard definition. + FilePath string `json:"file_path,omitempty"` +} + +func (r *Dashboard) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) +} + +func (r Dashboard) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) +} + +func (*Dashboard) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Lakeview.Get(ctx, dashboards.GetDashboardRequest{ + DashboardId: id, + }) + if err != nil { + log.Debugf(ctx, "dashboard %s does not exist", id) + return false, err + } + return true, nil +} + +func (*Dashboard) TerraformResourceName() string { + return "databricks_dashboard" +} + +func (r *Dashboard) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + baseURL.Path = fmt.Sprintf("dashboardsv3/%s/published", r.ID) + r.URL = baseURL.String() +} + +func (r *Dashboard) GetName() string { + return r.DisplayName +} + +func (r *Dashboard) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index d8f97a2db..98db1ec5d 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "strconv" "github.com/databricks/cli/libs/log" @@ -14,6 +16,7 @@ type Job struct { ID string `json:"id,omitempty" bundle:"readonly"` Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` *jobs.JobSettings } @@ -44,3 +47,19 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri func (j *Job) TerraformResourceName() string { return "databricks_job" } + +func (j *Job) InitializeURL(baseURL url.URL) { + if j.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("jobs/%s", j.ID) + j.URL = baseURL.String() +} + +func (j *Job) GetName() string { + return j.Name +} + +func (j *Job) GetURL() string { + return j.URL +} diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index 0ab486436..a5871468f 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -13,6 +15,7 @@ type MlflowExperiment struct { ID string `json:"id,omitempty" bundle:"readonly"` Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` *ml.Experiment } @@ -39,3 +42,19 @@ func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceCl func (s *MlflowExperiment) TerraformResourceName() string { return "databricks_mlflow_experiment" } + +func (s *MlflowExperiment) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("ml/experiments/%s", s.ID) + s.URL = baseURL.String() +} + +func (s *MlflowExperiment) GetName() string { + return s.Name +} + +func (s *MlflowExperiment) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 300474e35..9ead254d8 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -13,6 +15,7 @@ type MlflowModel struct { ID string `json:"id,omitempty" bundle:"readonly"` Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` *ml.Model } @@ -39,3 +42,19 @@ func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, func (s *MlflowModel) TerraformResourceName() string { return "databricks_mlflow_model" } + +func (s *MlflowModel) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("ml/models/%s", s.ID) + s.URL = baseURL.String() +} + +func (s *MlflowModel) GetName() string { + return s.Name +} + +func (s *MlflowModel) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 5efb7ea26..7f3ae00c8 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -23,6 +25,7 @@ type ModelServingEndpoint struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` } func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error { @@ -47,3 +50,19 @@ func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.Workspa func (s *ModelServingEndpoint) TerraformResourceName() string { return "databricks_model_serving" } + +func (s *ModelServingEndpoint) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("ml/endpoints/%s", s.ID) + s.URL = baseURL.String() +} + +func (s *ModelServingEndpoint) GetName() string { + return s.Name +} + +func (s *ModelServingEndpoint) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/permission.go b/bundle/config/resources/permission.go index fa2d8796c..62e18a09e 100644 --- a/bundle/config/resources/permission.go +++ b/bundle/config/resources/permission.go @@ -1,5 +1,7 @@ package resources +import "fmt" + // Permission holds the permission level setting for a single principal. // Multiple of these can be defined on any resource. type Permission struct { @@ -9,3 +11,19 @@ type Permission struct { ServicePrincipalName string `json:"service_principal_name,omitempty"` GroupName string `json:"group_name,omitempty"` } + +func (p Permission) String() string { + if p.UserName != "" { + return fmt.Sprintf("level: %s, user_name: %s", p.Level, p.UserName) + } + + if p.ServicePrincipalName != "" { + return fmt.Sprintf("level: %s, service_principal_name: %s", p.Level, p.ServicePrincipalName) + } + + if p.GroupName != "" { + return fmt.Sprintf("level: %s, group_name: %s", p.Level, p.GroupName) + } + + return fmt.Sprintf("level: %s", p.Level) +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 55270be65..b3311d8e2 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -2,6 +2,8 @@ package resources import ( "context" + "fmt" + "net/url" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -13,6 +15,7 @@ type Pipeline struct { ID string `json:"id,omitempty" bundle:"readonly"` Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` *pipelines.PipelineSpec } @@ -39,3 +42,19 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (p *Pipeline) TerraformResourceName() string { return "databricks_pipeline" } + +func (p *Pipeline) InitializeURL(baseURL url.URL) { + if p.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("pipelines/%s", p.ID) + p.URL = baseURL.String() +} + +func (p *Pipeline) GetName() string { + return p.Name +} + +func (s *Pipeline) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go index 9160782cd..3c823e625 100644 --- a/bundle/config/resources/quality_monitor.go +++ b/bundle/config/resources/quality_monitor.go @@ -2,6 +2,9 @@ package resources import ( "context" + "fmt" + "net/url" + "strings" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -20,6 +23,7 @@ type QualityMonitor struct { ID string `json:"id,omitempty" bundle:"readonly"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` } func (s *QualityMonitor) UnmarshalJSON(b []byte) error { @@ -44,3 +48,19 @@ func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClie func (s *QualityMonitor) TerraformResourceName() string { return "databricks_quality_monitor" } + +func (s *QualityMonitor) InitializeURL(baseURL url.URL) { + if s.TableName == "" { + return + } + baseURL.Path = fmt.Sprintf("explore/data/%s", strings.ReplaceAll(s.TableName, ".", "/")) + s.URL = baseURL.String() +} + +func (s *QualityMonitor) GetName() string { + return s.TableName +} + +func (s *QualityMonitor) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index 6033ffdf2..c44526d09 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -2,6 +2,9 @@ package resources import ( "context" + "fmt" + "net/url" + "strings" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" @@ -24,6 +27,7 @@ type RegisteredModel struct { *catalog.CreateRegisteredModelRequest ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` } func (s *RegisteredModel) UnmarshalJSON(b []byte) error { @@ -48,3 +52,19 @@ func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceCli func (s *RegisteredModel) TerraformResourceName() string { return "databricks_registered_model" } + +func (s *RegisteredModel) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("explore/data/models/%s", strings.ReplaceAll(s.ID, ".", "/")) + s.URL = baseURL.String() +} + +func (s *RegisteredModel) GetName() string { + return s.Name +} + +func (s *RegisteredModel) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go index 7ab00495a..a9f905cf1 100644 --- a/bundle/config/resources/schema.go +++ b/bundle/config/resources/schema.go @@ -1,6 +1,12 @@ package resources import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -16,6 +22,31 @@ type Schema struct { *catalog.CreateSchema ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + URL string `json:"url,omitempty" bundle:"internal"` +} + +func (s *Schema) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + return false, fmt.Errorf("schema.Exists() is not supported") +} + +func (s *Schema) TerraformResourceName() string { + return "databricks_schema" +} + +func (s *Schema) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + baseURL.Path = fmt.Sprintf("explore/data/%s", strings.ReplaceAll(s.ID, ".", "/")) + s.URL = baseURL.String() +} + +func (s *Schema) GetURL() string { + return s.URL +} + +func (s *Schema) GetName() string { + return s.Name } func (s *Schema) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 6860d73da..9ae73b22a 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -61,3 +62,38 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { }, "Resource %s does not have a custom unmarshaller", field.Name) } } + +func TestResourcesAllResourcesCompleteness(t *testing.T) { + r := Resources{} + rt := reflect.TypeOf(r) + + // Collect set of includes resource types + var types []string + for _, group := range r.AllResources() { + types = append(types, group.Description.PluralName) + } + + for i := 0; i < rt.NumField(); i++ { + field := rt.Field(i) + jsonTag := field.Tag.Get("json") + + if idx := strings.Index(jsonTag, ","); idx != -1 { + jsonTag = jsonTag[:idx] + } + + assert.Contains(t, types, jsonTag, "Field %s is missing in AllResources", field.Name) + } +} + +func TestSupportedResources(t *testing.T) { + // Please add your resource to the SupportedResources() function in resources.go if you add a new resource. + actual := SupportedResources() + + typ := reflect.TypeOf(Resources{}) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + jsonTags := strings.Split(field.Tag.Get("json"), ",") + pluralName := jsonTags[0] + assert.Equal(t, actual[pluralName].PluralName, pluralName) + } +} diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go new file mode 100644 index 000000000..88502ec8f --- /dev/null +++ b/bundle/config/validate/folder_permissions.go @@ -0,0 +1,100 @@ +package validate + +import ( + "context" + "fmt" + "path" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/paths" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/workspace" + "golang.org/x/sync/errgroup" +) + +type folderPermissions struct { +} + +// Apply implements bundle.ReadOnlyMutator. +func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle) diag.Diagnostics { + if len(b.Config().Permissions) == 0 { + return nil + } + + bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config().Workspace) + + var diags diag.Diagnostics + g, ctx := errgroup.WithContext(ctx) + results := make([]diag.Diagnostics, len(bundlePaths)) + for i, p := range bundlePaths { + g.Go(func() error { + results[i] = checkFolderPermission(ctx, b, p) + return nil + }) + } + + if err := g.Wait(); err != nil { + return diag.FromErr(err) + } + + for _, r := range results { + diags = diags.Extend(r) + } + + return diags +} + +func checkFolderPermission(ctx context.Context, b bundle.ReadOnlyBundle, folderPath string) diag.Diagnostics { + w := b.WorkspaceClient().Workspace + obj, err := getClosestExistingObject(ctx, w, folderPath) + if err != nil { + return diag.FromErr(err) + } + + objPermissions, err := w.GetPermissions(ctx, workspace.GetWorkspaceObjectPermissionsRequest{ + WorkspaceObjectId: fmt.Sprint(obj.ObjectId), + WorkspaceObjectType: "directories", + }) + if err != nil { + return diag.FromErr(err) + } + + p := permissions.ObjectAclToResourcePermissions(folderPath, objPermissions.AccessControlList) + return p.Compare(b.Config().Permissions) +} + +func getClosestExistingObject(ctx context.Context, w workspace.WorkspaceInterface, folderPath string) (*workspace.ObjectInfo, error) { + for { + obj, err := w.GetStatusByPath(ctx, folderPath) + if err == nil { + return obj, nil + } + + if !apierr.IsMissing(err) { + return nil, err + } + + parent := path.Dir(folderPath) + // If the parent is the same as the current folder, then we have reached the root + if folderPath == parent { + break + } + + folderPath = parent + } + + return nil, fmt.Errorf("folder %s and its parent folders do not exist", folderPath) +} + +// Name implements bundle.ReadOnlyMutator. +func (f *folderPermissions) Name() string { + return "validate:folder_permissions" +} + +// ValidateFolderPermissions validates that permissions for the folders in Workspace file system matches +// the permissions in the top-level permissions section of the bundle. +func ValidateFolderPermissions() bundle.ReadOnlyMutator { + return &folderPermissions{} +} diff --git a/bundle/config/validate/folder_permissions_test.go b/bundle/config/validate/folder_permissions_test.go new file mode 100644 index 000000000..8e68c9fbf --- /dev/null +++ b/bundle/config/validate/folder_permissions_test.go @@ -0,0 +1,208 @@ +package validate + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestFolderPermissionsInheritedWhenRootPathDoesNotExist(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/foo@bar.com", + ArtifactPath: "/Workspace/Users/otherfoo@bar.com/artifacts", + FilePath: "/Workspace/Users/foo@bar.com/files", + StatePath: "/Workspace/Users/foo@bar.com/state", + ResourcePath: "/Workspace/Users/foo@bar.com/resources", + }, + Permissions: []resources.Permission{ + {Level: permissions.CAN_MANAGE, UserName: "foo@bar.com"}, + }, + }, + } + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockWorkspaceAPI() + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users/otherfoo@bar.com/artifacts").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users/otherfoo@bar.com").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users/foo@bar.com").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace").Return(&workspace.ObjectInfo{ + ObjectId: 1234, + }, nil) + + api.EXPECT().GetPermissions(mock.Anything, workspace.GetWorkspaceObjectPermissionsRequest{ + WorkspaceObjectId: "1234", + WorkspaceObjectType: "directories", + }).Return(&workspace.WorkspaceObjectPermissions{ + ObjectId: "1234", + AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + }, nil) + + b.SetWorkpaceClient(m.WorkspaceClient) + rb := bundle.ReadOnly(b) + + diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + require.Empty(t, diags) +} + +func TestValidateFolderPermissionsFailsOnMissingBundlePermission(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/foo@bar.com", + ArtifactPath: "/Workspace/Users/foo@bar.com/artifacts", + FilePath: "/Workspace/Users/foo@bar.com/files", + StatePath: "/Workspace/Users/foo@bar.com/state", + ResourcePath: "/Workspace/Users/foo@bar.com/resources", + }, + Permissions: []resources.Permission{ + {Level: permissions.CAN_MANAGE, UserName: "foo@bar.com"}, + }, + }, + } + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockWorkspaceAPI() + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users/foo@bar.com").Return(&workspace.ObjectInfo{ + ObjectId: 1234, + }, nil) + + api.EXPECT().GetPermissions(mock.Anything, workspace.GetWorkspaceObjectPermissionsRequest{ + WorkspaceObjectId: "1234", + WorkspaceObjectType: "directories", + }).Return(&workspace.WorkspaceObjectPermissions{ + ObjectId: "1234", + AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + { + UserName: "foo2@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + }, nil) + + b.SetWorkpaceClient(m.WorkspaceClient) + rb := bundle.ReadOnly(b) + + diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + require.Len(t, diags, 1) + require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary) + require.Equal(t, diag.Warning, diags[0].Severity) + require.Equal(t, "The following permissions apply to the workspace folder at \"/Workspace/Users/foo@bar.com\" but are not configured in the bundle:\n- level: CAN_MANAGE, user_name: foo2@bar.com\n", diags[0].Detail) +} + +func TestValidateFolderPermissionsFailsOnPermissionMismatch(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/foo@bar.com", + ArtifactPath: "/Workspace/Users/foo@bar.com/artifacts", + FilePath: "/Workspace/Users/foo@bar.com/files", + StatePath: "/Workspace/Users/foo@bar.com/state", + ResourcePath: "/Workspace/Users/foo@bar.com/resources", + }, + Permissions: []resources.Permission{ + {Level: permissions.CAN_MANAGE, UserName: "foo@bar.com"}, + }, + }, + } + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockWorkspaceAPI() + api.EXPECT().GetStatusByPath(mock.Anything, "/Workspace/Users/foo@bar.com").Return(&workspace.ObjectInfo{ + ObjectId: 1234, + }, nil) + + api.EXPECT().GetPermissions(mock.Anything, workspace.GetWorkspaceObjectPermissionsRequest{ + WorkspaceObjectId: "1234", + WorkspaceObjectType: "directories", + }).Return(&workspace.WorkspaceObjectPermissions{ + ObjectId: "1234", + AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo2@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + }, nil) + + b.SetWorkpaceClient(m.WorkspaceClient) + rb := bundle.ReadOnly(b) + + diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + require.Len(t, diags, 1) + require.Equal(t, "untracked permissions apply to target workspace path", diags[0].Summary) + require.Equal(t, diag.Warning, diags[0].Severity) +} + +func TestValidateFolderPermissionsFailsOnNoRootFolder(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/NotExisting", + ArtifactPath: "/NotExisting/artifacts", + FilePath: "/NotExisting/files", + StatePath: "/NotExisting/state", + ResourcePath: "/NotExisting/resources", + }, + Permissions: []resources.Permission{ + {Level: permissions.CAN_MANAGE, UserName: "foo@bar.com"}, + }, + }, + } + m := mocks.NewMockWorkspaceClient(t) + api := m.GetMockWorkspaceAPI() + api.EXPECT().GetStatusByPath(mock.Anything, "/NotExisting").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + api.EXPECT().GetStatusByPath(mock.Anything, "/").Return(nil, &apierr.APIError{ + StatusCode: 404, + ErrorCode: "RESOURCE_DOES_NOT_EXIST", + }) + + b.SetWorkpaceClient(m.WorkspaceClient) + rb := bundle.ReadOnly(b) + + diags := bundle.ApplyReadOnly(context.Background(), rb, ValidateFolderPermissions()) + require.Len(t, diags, 1) + require.Equal(t, "folder / and its parent folders do not exist", diags[0].Summary) + require.Equal(t, diag.Error, diags[0].Severity) +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index 79f42bd23..440477e65 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -35,6 +35,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics FilesToSync(), ValidateSyncPatterns(), JobTaskClusterSpec(), + ValidateFolderPermissions(), )) } diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index 77b83611b..bab4e176c 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -2,9 +2,12 @@ package files import ( "context" + "errors" "fmt" + "io/fs" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -35,6 +38,9 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { b.Files, err = sync.RunOnce(ctx) if err != nil { + if errors.Is(err, fs.ErrPermission) { + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.FilePath) + } return diag.FromErr(err) } diff --git a/bundle/deploy/lock/acquire.go b/bundle/deploy/lock/acquire.go index 7d3d0eca8..ab1f1cba1 100644 --- a/bundle/deploy/lock/acquire.go +++ b/bundle/deploy/lock/acquire.go @@ -3,8 +3,10 @@ package lock import ( "context" "errors" + "io/fs" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/locker" @@ -51,12 +53,17 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics if err != nil { log.Errorf(ctx, "Failed to acquire deployment lock: %v", err) + if errors.Is(err, fs.ErrPermission) { + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath) + } + notExistsError := filer.NoSuchDirectoryError{} if errors.As(err, ¬ExistsError) { // If we get a "doesn't exist" error from the API this indicates // we either don't have permissions or the path is invalid. - return diag.Errorf("cannot write to deployment root (this can indicate a previous deploy was done with a different identity): %s", b.Config.Workspace.RootPath) + return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath) } + return diag.FromErr(err) } diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index e52d0ca8f..5ea2effa0 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -4,6 +4,7 @@ import ( "context" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" @@ -34,6 +35,10 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Apply terraform according to the computed plan err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { + diags := permissions.TryExtendTerraformPermissionError(ctx, b, err) + if diags != nil { + return diags + } return diag.Errorf("terraform apply: %v", err) } diff --git a/bundle/deploy/terraform/check_dashboards_modified_remotely.go b/bundle/deploy/terraform/check_dashboards_modified_remotely.go new file mode 100644 index 000000000..c884bcb9b --- /dev/null +++ b/bundle/deploy/terraform/check_dashboards_modified_remotely.go @@ -0,0 +1,117 @@ +package terraform + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + tfjson "github.com/hashicorp/terraform-json" +) + +type dashboardState struct { + Name string + ID string + ETag string +} + +func collectDashboardsFromState(ctx context.Context, b *bundle.Bundle) ([]dashboardState, error) { + state, err := ParseResourcesState(ctx, b) + if err != nil && state == nil { + return nil, err + } + + var dashboards []dashboardState + for _, resource := range state.Resources { + if resource.Mode != tfjson.ManagedResourceMode { + continue + } + for _, instance := range resource.Instances { + switch resource.Type { + case "databricks_dashboard": + dashboards = append(dashboards, dashboardState{ + Name: resource.Name, + ID: instance.Attributes.ID, + ETag: instance.Attributes.ETag, + }) + } + } + } + + return dashboards, nil +} + +type checkDashboardsModifiedRemotely struct { +} + +func (l *checkDashboardsModifiedRemotely) Name() string { + return "CheckDashboardsModifiedRemotely" +} + +func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // This mutator is relevant only if the bundle includes dashboards. + if len(b.Config.Resources.Dashboards) == 0 { + return nil + } + + // If the user has forced the deployment, skip this check. + if b.Config.Bundle.Force { + return nil + } + + dashboards, err := collectDashboardsFromState(ctx, b) + if err != nil { + return diag.FromErr(err) + } + + var diags diag.Diagnostics + for _, dashboard := range dashboards { + // Skip dashboards that are not defined in the bundle. + // These will be destroyed upon deployment. + if _, ok := b.Config.Resources.Dashboards[dashboard.Name]; !ok { + continue + } + + path := dyn.MustPathFromString(fmt.Sprintf("resources.dashboards.%s", dashboard.Name)) + loc := b.Config.GetLocation(path.String()) + actual, err := b.WorkspaceClient().Lakeview.GetByDashboardId(ctx, dashboard.ID) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name), + Detail: err.Error(), + Paths: []dyn.Path{path}, + Locations: []dyn.Location{loc}, + }) + continue + } + + // If the ETag is the same, the dashboard has not been modified. + if actual.Etag == dashboard.ETag { + continue + } + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("dashboard %q has been modified remotely", dashboard.Name), + Detail: "" + + "This dashboard has been modified remotely since the last bundle deployment.\n" + + "These modifications are untracked and will be overwritten on deploy.\n" + + "\n" + + "Make sure that the local dashboard definition matches what you intend to deploy\n" + + "before proceeding with the deployment.\n" + + "\n" + + "Run `databricks bundle deploy --force` to bypass this error." + + "", + Paths: []dyn.Path{path}, + Locations: []dyn.Location{loc}, + }) + } + + return diags +} + +func CheckDashboardsModifiedRemotely() *checkDashboardsModifiedRemotely { + return &checkDashboardsModifiedRemotely{} +} diff --git a/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go b/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go new file mode 100644 index 000000000..c13f800f7 --- /dev/null +++ b/bundle/deploy/terraform/check_dashboards_modified_remotely_test.go @@ -0,0 +1,191 @@ +package terraform + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func mockDashboardBundle(t *testing.T) *bundle.Bundle { + dir := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: dir, + Config: config.Root{ + Bundle: config.Bundle{ + Target: "test", + }, + Resources: config.Resources{ + Dashboards: map[string]*resources.Dashboard{ + "dash1": { + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "My Special Dashboard", + }, + }, + }, + }, + }, + } + return b +} + +func TestCheckDashboardsModifiedRemotely_NoDashboards(t *testing.T) { + dir := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: dir, + Config: config.Root{ + Bundle: config.Bundle{ + Target: "test", + }, + Resources: config.Resources{}, + }, + } + + diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely()) + assert.Empty(t, diags) +} + +func TestCheckDashboardsModifiedRemotely_FirstDeployment(t *testing.T) { + b := mockDashboardBundle(t) + diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely()) + assert.Empty(t, diags) +} + +func TestCheckDashboardsModifiedRemotely_ExistingStateNoChange(t *testing.T) { + ctx := context.Background() + + b := mockDashboardBundle(t) + writeFakeDashboardState(t, ctx, b) + + // Mock the call to the API. + m := mocks.NewMockWorkspaceClient(t) + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT(). + GetByDashboardId(mock.Anything, "id1"). + Return(&dashboards.Dashboard{ + DisplayName: "My Special Dashboard", + Etag: "1000", + }, nil). + Once() + b.SetWorkpaceClient(m.WorkspaceClient) + + // No changes, so no diags. + diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely()) + assert.Empty(t, diags) +} + +func TestCheckDashboardsModifiedRemotely_ExistingStateChange(t *testing.T) { + ctx := context.Background() + + b := mockDashboardBundle(t) + writeFakeDashboardState(t, ctx, b) + + // Mock the call to the API. + m := mocks.NewMockWorkspaceClient(t) + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT(). + GetByDashboardId(mock.Anything, "id1"). + Return(&dashboards.Dashboard{ + DisplayName: "My Special Dashboard", + Etag: "1234", + }, nil). + Once() + b.SetWorkpaceClient(m.WorkspaceClient) + + // The dashboard has changed, so expect an error. + diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely()) + if assert.Len(t, diags, 1) { + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, `dashboard "dash1" has been modified remotely`, diags[0].Summary) + } +} + +func TestCheckDashboardsModifiedRemotely_ExistingStateFailureToGet(t *testing.T) { + ctx := context.Background() + + b := mockDashboardBundle(t) + writeFakeDashboardState(t, ctx, b) + + // Mock the call to the API. + m := mocks.NewMockWorkspaceClient(t) + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT(). + GetByDashboardId(mock.Anything, "id1"). + Return(nil, fmt.Errorf("failure")). + Once() + b.SetWorkpaceClient(m.WorkspaceClient) + + // Unable to get the dashboard, so expect an error. + diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely()) + if assert.Len(t, diags, 1) { + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, `failed to get dashboard "dash1"`, diags[0].Summary) + } +} + +func writeFakeDashboardState(t *testing.T, ctx context.Context, b *bundle.Bundle) { + tfDir, err := Dir(ctx, b) + require.NoError(t, err) + + // Write fake state file. + testutil.WriteFile(t, ` + { + "version": 4, + "terraform_version": "1.5.5", + "resources": [ + { + "mode": "managed", + "type": "databricks_dashboard", + "name": "dash1", + "instances": [ + { + "schema_version": 0, + "attributes": { + "etag": "1000", + "id": "id1" + } + } + ] + }, + { + "mode": "managed", + "type": "databricks_job", + "name": "job", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "1234" + } + } + ] + }, + { + "mode": "managed", + "type": "databricks_dashboard", + "name": "dash2", + "instances": [ + { + "schema_version": 0, + "attributes": { + "etag": "1001", + "id": "id2" + } + } + ] + } + ] + } + `, filepath.Join(tfDir, TerraformStateFileName)) +} diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index b8993c031..0ace7c66e 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -2,9 +2,7 @@ package terraform import ( "context" - "encoding/json" "fmt" - "sort" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" @@ -14,244 +12,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" ) -func conv(from any, to any) { - buf, _ := json.Marshal(from) - json.Unmarshal(buf, &to) -} - -func convPermissions(acl []resources.Permission) *schema.ResourcePermissions { - if len(acl) == 0 { - return nil - } - - resource := schema.ResourcePermissions{} - for _, ac := range acl { - resource.AccessControl = append(resource.AccessControl, convPermission(ac)) - } - - return &resource -} - -func convPermission(ac resources.Permission) schema.ResourcePermissionsAccessControl { - dst := schema.ResourcePermissionsAccessControl{ - PermissionLevel: ac.Level, - } - if ac.UserName != "" { - dst.UserName = ac.UserName - } - if ac.GroupName != "" { - dst.GroupName = ac.GroupName - } - if ac.ServicePrincipalName != "" { - dst.ServicePrincipalName = ac.ServicePrincipalName - } - return dst -} - -func convGrants(acl []resources.Grant) *schema.ResourceGrants { - if len(acl) == 0 { - return nil - } - - resource := schema.ResourceGrants{} - for _, ac := range acl { - resource.Grant = append(resource.Grant, schema.ResourceGrantsGrant{ - Privileges: ac.Privileges, - Principal: ac.Principal, - }) - } - - return &resource -} - -// BundleToTerraform converts resources in a bundle configuration -// to the equivalent Terraform JSON representation. -// -// Note: This function is an older implementation of the conversion logic. It is -// no longer used in any code paths. It is kept around to be used in tests. -// New resources do not need to modify this function and can instead can define -// the conversion login in the tfdyn package. -func BundleToTerraform(config *config.Root) *schema.Root { - tfroot := schema.NewRoot() - tfroot.Provider = schema.NewProviders() - tfroot.Resource = schema.NewResources() - noResources := true - - for k, src := range config.Resources.Jobs { - noResources = false - var dst schema.ResourceJob - conv(src, &dst) - - if src.JobSettings != nil { - sort.Slice(src.JobSettings.Tasks, func(i, j int) bool { - return src.JobSettings.Tasks[i].TaskKey < src.JobSettings.Tasks[j].TaskKey - }) - - for _, v := range src.Tasks { - var t schema.ResourceJobTask - conv(v, &t) - - for _, v_ := range v.Libraries { - var l schema.ResourceJobTaskLibrary - conv(v_, &l) - t.Library = append(t.Library, l) - } - - // Convert for_each_task libraries - if v.ForEachTask != nil { - for _, v_ := range v.ForEachTask.Task.Libraries { - var l schema.ResourceJobTaskForEachTaskTaskLibrary - conv(v_, &l) - t.ForEachTask.Task.Library = append(t.ForEachTask.Task.Library, l) - } - - } - - dst.Task = append(dst.Task, t) - } - - for _, v := range src.JobClusters { - var t schema.ResourceJobJobCluster - conv(v, &t) - dst.JobCluster = append(dst.JobCluster, t) - } - - // Unblock downstream work. To be addressed more generally later. - if git := src.GitSource; git != nil { - dst.GitSource = &schema.ResourceJobGitSource{ - Url: git.GitUrl, - Branch: git.GitBranch, - Commit: git.GitCommit, - Provider: string(git.GitProvider), - Tag: git.GitTag, - } - } - - for _, v := range src.Parameters { - var t schema.ResourceJobParameter - conv(v, &t) - dst.Parameter = append(dst.Parameter, t) - } - } - - tfroot.Resource.Job[k] = &dst - - // Configure permissions for this resource. - if rp := convPermissions(src.Permissions); rp != nil { - rp.JobId = fmt.Sprintf("${databricks_job.%s.id}", k) - tfroot.Resource.Permissions["job_"+k] = rp - } - } - - for k, src := range config.Resources.Pipelines { - noResources = false - var dst schema.ResourcePipeline - conv(src, &dst) - - if src.PipelineSpec != nil { - for _, v := range src.Libraries { - var l schema.ResourcePipelineLibrary - conv(v, &l) - dst.Library = append(dst.Library, l) - } - - for _, v := range src.Clusters { - var l schema.ResourcePipelineCluster - conv(v, &l) - dst.Cluster = append(dst.Cluster, l) - } - - for _, v := range src.Notifications { - var l schema.ResourcePipelineNotification - conv(v, &l) - dst.Notification = append(dst.Notification, l) - } - } - - tfroot.Resource.Pipeline[k] = &dst - - // Configure permissions for this resource. - if rp := convPermissions(src.Permissions); rp != nil { - rp.PipelineId = fmt.Sprintf("${databricks_pipeline.%s.id}", k) - tfroot.Resource.Permissions["pipeline_"+k] = rp - } - } - - for k, src := range config.Resources.Models { - noResources = false - var dst schema.ResourceMlflowModel - conv(src, &dst) - tfroot.Resource.MlflowModel[k] = &dst - - // Configure permissions for this resource. - if rp := convPermissions(src.Permissions); rp != nil { - rp.RegisteredModelId = fmt.Sprintf("${databricks_mlflow_model.%s.registered_model_id}", k) - tfroot.Resource.Permissions["mlflow_model_"+k] = rp - } - } - - for k, src := range config.Resources.Experiments { - noResources = false - var dst schema.ResourceMlflowExperiment - conv(src, &dst) - tfroot.Resource.MlflowExperiment[k] = &dst - - // Configure permissions for this resource. - if rp := convPermissions(src.Permissions); rp != nil { - rp.ExperimentId = fmt.Sprintf("${databricks_mlflow_experiment.%s.id}", k) - tfroot.Resource.Permissions["mlflow_experiment_"+k] = rp - } - } - - for k, src := range config.Resources.ModelServingEndpoints { - noResources = false - var dst schema.ResourceModelServing - conv(src, &dst) - tfroot.Resource.ModelServing[k] = &dst - - // Configure permissions for this resource. - if rp := convPermissions(src.Permissions); rp != nil { - rp.ServingEndpointId = fmt.Sprintf("${databricks_model_serving.%s.serving_endpoint_id}", k) - tfroot.Resource.Permissions["model_serving_"+k] = rp - } - } - - for k, src := range config.Resources.RegisteredModels { - noResources = false - var dst schema.ResourceRegisteredModel - conv(src, &dst) - tfroot.Resource.RegisteredModel[k] = &dst - - // Configure permissions for this resource. - if rp := convGrants(src.Grants); rp != nil { - rp.Function = fmt.Sprintf("${databricks_registered_model.%s.id}", k) - tfroot.Resource.Grants["registered_model_"+k] = rp - } - } - - for k, src := range config.Resources.QualityMonitors { - noResources = false - var dst schema.ResourceQualityMonitor - conv(src, &dst) - tfroot.Resource.QualityMonitor[k] = &dst - } - - for k, src := range config.Resources.Clusters { - noResources = false - var dst schema.ResourceCluster - conv(src, &dst) - tfroot.Resource.Cluster[k] = &dst - } - - // We explicitly set "resource" to nil to omit it from a JSON encoding. - // This is required because the terraform CLI requires >= 1 resources defined - // if the "resource" property is used in a .tf.json file. - if noResources { - tfroot.Resource = nil - } - return tfroot -} - // BundleToTerraformWithDynValue converts resources in a bundle configuration // to the equivalent Terraform JSON representation. func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema.Root, error) { @@ -416,6 +176,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.Clusters[resource.Name] = cur + case "databricks_dashboard": + if config.Resources.Dashboards == nil { + config.Resources.Dashboards = make(map[string]*resources.Dashboard) + } + cur := config.Resources.Dashboards[resource.Name] + if cur == nil { + cur = &resources.Dashboard{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Dashboards[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -470,6 +240,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Dashboards { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 4c6866d9d..3f69bbed4 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -2,7 +2,6 @@ package terraform import ( "context" - "encoding/json" "reflect" "testing" @@ -13,6 +12,7 @@ import ( "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -21,6 +21,27 @@ import ( "github.com/stretchr/testify/require" ) +func produceTerraformConfiguration(t *testing.T, config config.Root) *schema.Root { + vin, err := convert.FromTyped(config, dyn.NilValue) + require.NoError(t, err) + out, err := BundleToTerraformWithDynValue(context.Background(), vin) + require.NoError(t, err) + return out +} + +func convertToResourceStruct[T any](t *testing.T, resource *T, data any) { + require.NotNil(t, resource) + require.NotNil(t, data) + + // Convert data to a dyn.Value. + vin, err := convert.FromTyped(data, dyn.NilValue) + require.NoError(t, err) + + // Convert the dyn.Value to a struct. + err = convert.ToTyped(resource, vin) + require.NoError(t, err) +} + func TestBundleToTerraformJob(t *testing.T) { var src = resources.Job{ JobSettings: &jobs.JobSettings{ @@ -58,8 +79,9 @@ func TestBundleToTerraformJob(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Job["my_job"].(*schema.ResourceJob) + var resource schema.ResourceJob + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Job["my_job"]) assert.Equal(t, "my job", resource.Name) assert.Len(t, resource.JobCluster, 1) @@ -68,8 +90,6 @@ func TestBundleToTerraformJob(t *testing.T) { assert.Equal(t, "param1", resource.Parameter[0].Name) assert.Equal(t, "param2", resource.Parameter[1].Name) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformJobPermissions(t *testing.T) { @@ -90,15 +110,14 @@ func TestBundleToTerraformJobPermissions(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Permissions["job_my_job"].(*schema.ResourcePermissions) + var resource schema.ResourcePermissions + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Permissions["job_my_job"]) assert.NotEmpty(t, resource.JobId) assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformJobTaskLibraries(t *testing.T) { @@ -128,15 +147,14 @@ func TestBundleToTerraformJobTaskLibraries(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Job["my_job"].(*schema.ResourceJob) + var resource schema.ResourceJob + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Job["my_job"]) assert.Equal(t, "my job", resource.Name) require.Len(t, resource.Task, 1) require.Len(t, resource.Task[0].Library, 1) assert.Equal(t, "mlflow", resource.Task[0].Library[0].Pypi.Package) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformForEachTaskLibraries(t *testing.T) { @@ -172,15 +190,14 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Job["my_job"].(*schema.ResourceJob) + var resource schema.ResourceJob + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Job["my_job"]) assert.Equal(t, "my job", resource.Name) require.Len(t, resource.Task, 1) require.Len(t, resource.Task[0].ForEachTask.Task.Library, 1) assert.Equal(t, "mlflow", resource.Task[0].ForEachTask.Task.Library[0].Pypi.Package) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformPipeline(t *testing.T) { @@ -230,8 +247,9 @@ func TestBundleToTerraformPipeline(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Pipeline["my_pipeline"].(*schema.ResourcePipeline) + var resource schema.ResourcePipeline + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Pipeline["my_pipeline"]) assert.Equal(t, "my pipeline", resource.Name) assert.Len(t, resource.Library, 2) @@ -241,8 +259,6 @@ func TestBundleToTerraformPipeline(t *testing.T) { assert.Equal(t, resource.Notification[1].Alerts, []string{"on-update-failure", "on-flow-failure"}) assert.Equal(t, resource.Notification[1].EmailRecipients, []string{"jane@doe.com", "john@doe.com"}) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformPipelinePermissions(t *testing.T) { @@ -263,15 +279,14 @@ func TestBundleToTerraformPipelinePermissions(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Permissions["pipeline_my_pipeline"].(*schema.ResourcePermissions) + var resource schema.ResourcePermissions + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Permissions["pipeline_my_pipeline"]) assert.NotEmpty(t, resource.PipelineId) assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModel(t *testing.T) { @@ -300,8 +315,9 @@ func TestBundleToTerraformModel(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.MlflowModel["my_model"].(*schema.ResourceMlflowModel) + var resource schema.ResourceMlflowModel + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.MlflowModel["my_model"]) assert.Equal(t, "name", resource.Name) assert.Equal(t, "description", resource.Description) @@ -311,8 +327,6 @@ func TestBundleToTerraformModel(t *testing.T) { assert.Equal(t, "k2", resource.Tags[1].Key) assert.Equal(t, "v2", resource.Tags[1].Value) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelPermissions(t *testing.T) { @@ -336,15 +350,14 @@ func TestBundleToTerraformModelPermissions(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Permissions["mlflow_model_my_model"].(*schema.ResourcePermissions) + var resource schema.ResourcePermissions + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Permissions["mlflow_model_my_model"]) assert.NotEmpty(t, resource.RegisteredModelId) assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_READ", resource.AccessControl[0].PermissionLevel) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformExperiment(t *testing.T) { @@ -362,13 +375,12 @@ func TestBundleToTerraformExperiment(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.MlflowExperiment["my_experiment"].(*schema.ResourceMlflowExperiment) + var resource schema.ResourceMlflowExperiment + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.MlflowExperiment["my_experiment"]) assert.Equal(t, "name", resource.Name) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformExperimentPermissions(t *testing.T) { @@ -392,15 +404,14 @@ func TestBundleToTerraformExperimentPermissions(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Permissions["mlflow_experiment_my_experiment"].(*schema.ResourcePermissions) + var resource schema.ResourcePermissions + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Permissions["mlflow_experiment_my_experiment"]) assert.NotEmpty(t, resource.ExperimentId) assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_READ", resource.AccessControl[0].PermissionLevel) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelServing(t *testing.T) { @@ -436,8 +447,9 @@ func TestBundleToTerraformModelServing(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.ModelServing["my_model_serving_endpoint"].(*schema.ResourceModelServing) + var resource schema.ResourceModelServing + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.ModelServing["my_model_serving_endpoint"]) assert.Equal(t, "name", resource.Name) assert.Equal(t, "model_name", resource.Config.ServedModels[0].ModelName) @@ -447,8 +459,6 @@ func TestBundleToTerraformModelServing(t *testing.T) { assert.Equal(t, "model_name-1", resource.Config.TrafficConfig.Routes[0].ServedModelName) assert.Equal(t, 100, resource.Config.TrafficConfig.Routes[0].TrafficPercentage) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformModelServingPermissions(t *testing.T) { @@ -490,15 +500,14 @@ func TestBundleToTerraformModelServingPermissions(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Permissions["model_serving_my_model_serving_endpoint"].(*schema.ResourcePermissions) + var resource schema.ResourcePermissions + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Permissions["model_serving_my_model_serving_endpoint"]) assert.NotEmpty(t, resource.ServingEndpointId) assert.Len(t, resource.AccessControl, 1) assert.Equal(t, "jane@doe.com", resource.AccessControl[0].UserName) assert.Equal(t, "CAN_VIEW", resource.AccessControl[0].PermissionLevel) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformRegisteredModel(t *testing.T) { @@ -519,16 +528,15 @@ func TestBundleToTerraformRegisteredModel(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.RegisteredModel["my_registered_model"].(*schema.ResourceRegisteredModel) + var resource schema.ResourceRegisteredModel + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.RegisteredModel["my_registered_model"]) assert.Equal(t, "name", resource.Name) assert.Equal(t, "catalog", resource.CatalogName) assert.Equal(t, "schema", resource.SchemaName) assert.Equal(t, "comment", resource.Comment) assert.Nil(t, out.Data) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { @@ -554,15 +562,14 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { }, } - out := BundleToTerraform(&config) - resource := out.Resource.Grants["registered_model_my_registered_model"].(*schema.ResourceGrants) + var resource schema.ResourceGrants + out := produceTerraformConfiguration(t, config) + convertToResourceStruct(t, &resource, out.Resource.Grants["registered_model_my_registered_model"]) assert.NotEmpty(t, resource.Function) assert.Len(t, resource.Grant, 1) assert.Equal(t, "jane@doe.com", resource.Grant[0].Principal) assert.Equal(t, "EXECUTE", resource.Grant[0].Privileges[0]) - - bundleToTerraformEquivalenceTest(t, &config) } func TestBundleToTerraformDeletedResources(t *testing.T) { @@ -671,6 +678,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_dashboard", + Mode: "managed", + Name: "test_dashboard", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -703,6 +718,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -772,6 +790,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Dashboards: map[string]*resources.Dashboard{ + "test_dashboard": { + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "test_dashboard", + }, + }, + }, }, } var tfState = resourcesState{ @@ -807,6 +832,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -921,6 +949,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Dashboards: map[string]*resources.Dashboard{ + "test_dashboard": { + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "test_dashboard", + }, + }, + "test_dashboard_new": { + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "test_dashboard_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -1069,6 +1109,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "2"}}, }, }, + { + Type: "databricks_dashboard", + Mode: "managed", + Name: "test_dashboard", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_dashboard", + Mode: "managed", + Name: "test_dashboard_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1137,6 +1193,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) + assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Dashboards["test_dashboard_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -1154,25 +1217,3 @@ func AssertFullResourceCoverage(t *testing.T, config *config.Root) { } } } - -func assertEqualTerraformRoot(t *testing.T, a, b *schema.Root) { - ba, err := json.Marshal(a) - require.NoError(t, err) - bb, err := json.Marshal(b) - require.NoError(t, err) - assert.JSONEq(t, string(ba), string(bb)) -} - -func bundleToTerraformEquivalenceTest(t *testing.T, config *config.Root) { - t.Run("dyn equivalence", func(t *testing.T) { - tf1 := BundleToTerraform(config) - - vin, err := convert.FromTyped(config, dyn.NilValue) - require.NoError(t, err) - tf2, err := BundleToTerraformWithDynValue(context.Background(), vin) - require.NoError(t, err) - - // Compare roots - assertEqualTerraformRoot(t, tf1, tf2) - }) -} diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 12894c684..eb15c63ec 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -60,6 +60,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) case dyn.Key("clusters"): path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...) + case dyn.Key("dashboards"): + path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 630a904ac..b26ef928d 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -32,6 +32,7 @@ func TestInterpolate(t *testing.T) { "other_registered_model": "${resources.registered_models.other_registered_model.id}", "other_schema": "${resources.schemas.other_schema.id}", "other_cluster": "${resources.clusters.other_cluster.id}", + "other_dashboard": "${resources.dashboards.other_dashboard.id}", }, Tasks: []jobs.Task{ { @@ -69,6 +70,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"]) + assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index f25f09ea8..18819c00f 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -40,7 +40,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, // Configure permissions for this resource. if permissions := convertPermissionsResource(ctx, vin); permissions != nil { - permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key) + permissions.ClusterId = fmt.Sprintf("${databricks_cluster.%s.id}", key) out.Permissions["cluster_"+key] = permissions } diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go index e7d2542fd..e6d2620c6 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go @@ -81,7 +81,7 @@ func TestConvertCluster(t *testing.T) { // Assert equality on the permissions assert.Equal(t, &schema.ResourcePermissions{ - JobId: "${databricks_cluster.my_cluster.id}", + ClusterId: "${databricks_cluster.my_cluster.id}", AccessControl: []schema.ResourcePermissionsAccessControl{ { PermissionLevel: "CAN_RUN", diff --git a/bundle/deploy/terraform/tfdyn/convert_dashboard.go b/bundle/deploy/terraform/tfdyn/convert_dashboard.go new file mode 100644 index 000000000..3ba7e19a2 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_dashboard.go @@ -0,0 +1,109 @@ +package tfdyn + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +const ( + filePathFieldName = "file_path" + serializedDashboardFieldName = "serialized_dashboard" +) + +// Marshal "serialized_dashboard" as JSON if it is set in the input but not in the output. +func marshalSerializedDashboard(vin dyn.Value, vout dyn.Value) (dyn.Value, error) { + // Skip if the "serialized_dashboard" field is already set. + if v := vout.Get(serializedDashboardFieldName); v.IsValid() { + return vout, nil + } + + // Skip if the "serialized_dashboard" field on the input is not set. + v := vin.Get(serializedDashboardFieldName) + if !v.IsValid() { + return vout, nil + } + + // Marshal the "serialized_dashboard" field as JSON. + data, err := json.Marshal(v.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal serialized_dashboard: %w", err) + } + + // Set the "serialized_dashboard" field on the output. + return dyn.Set(vout, serializedDashboardFieldName, dyn.V(string(data))) +} + +func convertDashboardResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + var err error + + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceDashboard{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "dashboard normalization diagnostic: %s", diag.Summary) + } + + // Include "serialized_dashboard" field if "file_path" is set. + // Note: the Terraform resource supports "file_path" natively, but its + // change detection mechanism doesn't work as expected at the time of writing (Sep 30). + if path, ok := vout.Get(filePathFieldName).AsString(); ok { + vout, err = dyn.Set(vout, serializedDashboardFieldName, dyn.V(fmt.Sprintf("${file(%q)}", path))) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to set serialized_dashboard: %w", err) + } + // Drop the "file_path" field. It is mutually exclusive with "serialized_dashboard". + vout, err = dyn.Walk(vout, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + switch len(p) { + case 0: + return v, nil + case 1: + if p[0] == dyn.Key(filePathFieldName) { + return v, dyn.ErrDrop + } + } + + // Skip everything else. + return v, dyn.ErrSkip + }) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to drop file_path: %w", err) + } + } + + // Marshal "serialized_dashboard" as JSON if it is set in the input but not in the output. + vout, err = marshalSerializedDashboard(vin, vout) + if err != nil { + return dyn.InvalidValue, err + } + + return vout, nil +} + +type dashboardConverter struct{} + +func (dashboardConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertDashboardResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Dashboard[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.DashboardId = fmt.Sprintf("${databricks_dashboard.%s.id}", key) + out.Permissions["dashboard_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("dashboards", dashboardConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go b/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go new file mode 100644 index 000000000..9cefbc10e --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_dashboard_test.go @@ -0,0 +1,153 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertDashboard(t *testing.T) { + var src = resources.Dashboard{ + CreateDashboardRequest: &dashboards.CreateDashboardRequest{ + DisplayName: "my dashboard", + WarehouseId: "f00dcafe", + ParentPath: "/some/path", + }, + + EmbedCredentials: true, + + Permissions: []resources.Permission{ + { + Level: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out) + require.NoError(t, err) + + // Assert equality on the job + assert.Equal(t, map[string]any{ + "display_name": "my dashboard", + "warehouse_id": "f00dcafe", + "parent_path": "/some/path", + "embed_credentials": true, + }, out.Dashboard["my_dashboard"]) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + DashboardId: "${databricks_dashboard.my_dashboard.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_VIEW", + UserName: "jane@doe.com", + }, + }, + }, out.Permissions["dashboard_my_dashboard"]) +} + +func TestConvertDashboardFilePath(t *testing.T) { + var src = resources.Dashboard{ + FilePath: "some/path", + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out) + require.NoError(t, err) + + // Assert that the "serialized_dashboard" is included. + assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{ + "serialized_dashboard": "${file(\"some/path\")}", + }) + + // Assert that the "file_path" doesn't carry over. + assert.NotSubset(t, out.Dashboard["my_dashboard"], map[string]any{ + "file_path": "some/path", + }) +} + +func TestConvertDashboardFilePathQuoted(t *testing.T) { + var src = resources.Dashboard{ + FilePath: `C:\foo\bar\baz\dashboard.lvdash.json`, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out) + require.NoError(t, err) + + // Assert that the "serialized_dashboard" is included. + assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{ + "serialized_dashboard": `${file("C:\\foo\\bar\\baz\\dashboard.lvdash.json")}`, + }) + + // Assert that the "file_path" doesn't carry over. + assert.NotSubset(t, out.Dashboard["my_dashboard"], map[string]any{ + "file_path": `C:\foo\bar\baz\dashboard.lvdash.json`, + }) +} + +func TestConvertDashboardSerializedDashboardString(t *testing.T) { + var src = resources.Dashboard{ + SerializedDashboard: `{ "json": true }`, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out) + require.NoError(t, err) + + // Assert that the "serialized_dashboard" is included. + assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{ + "serialized_dashboard": `{ "json": true }`, + }) +} + +func TestConvertDashboardSerializedDashboardAny(t *testing.T) { + var src = resources.Dashboard{ + SerializedDashboard: map[string]any{ + "pages": []map[string]any{ + { + "displayName": "New Page", + "layout": []map[string]any{}, + }, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out) + require.NoError(t, err) + + // Assert that the "serialized_dashboard" is included. + assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{ + "serialized_dashboard": `{"pages":[{"displayName":"New Page","layout":[]}]}`, + }) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 64d667b5f..4da015c23 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -13,7 +13,7 @@ import ( // Partial representation of the Terraform state file format. // We are only interested global version and serial numbers, -// plus resource types, names, modes, and ids. +// plus resource types, names, modes, IDs, and ETags (for dashboards). type resourcesState struct { Version int `json:"version"` Resources []stateResource `json:"resources"` @@ -33,7 +33,8 @@ type stateResourceInstance struct { } type stateInstanceAttributes struct { - ID string `json:"id"` + ID string `json:"id"` + ETag string `json:"etag,omitempty"` } func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) { diff --git a/bundle/internal/bundletest/mutate.go b/bundle/internal/bundletest/mutate.go new file mode 100644 index 000000000..c0ac630ce --- /dev/null +++ b/bundle/internal/bundletest/mutate.go @@ -0,0 +1,20 @@ +package bundletest + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/require" +) + +func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) { + diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(f) + require.NoError(t, err) + return nil + }) + require.NoError(t, diags.Error()) +} diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 4a2371472..ddeffe2fd 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -8,8 +8,10 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/databricks-sdk-go/service/jobs" ) func interpolationPattern(s string) string { @@ -66,6 +68,31 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } +func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Job{}): + // This field has been deprecated in jobs API v2.1 and is always set to + // "MULTI_TASK" in the backend. We should not expose it to the user. + delete(s.Properties, "format") + + // These fields are only meant to be set by the DABs client (ie the CLI) + // and thus should not be exposed to the user. These are used to annotate + // jobs that were created by DABs. + delete(s.Properties, "deployment") + delete(s.Properties, "edit_mode") + + case reflect.TypeOf(jobs.GitSource{}): + // These fields are readonly and are not meant to be set by the user. + delete(s.Properties, "job_source") + delete(s.Properties, "git_snapshot") + + default: + // Do nothing + } + + return s +} + func main() { if len(os.Args) != 2 { fmt.Println("Usage: go run main.go ") @@ -90,6 +117,7 @@ func main() { s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ p.addDescriptions, p.addEnums, + removeJobsFields, addInterpolationPatterns, }) if err != nil { diff --git a/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml new file mode 100644 index 000000000..62e490b0c --- /dev/null +++ b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + format: SINGLE_TASK diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml new file mode 100644 index 000000000..705ce9516 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml @@ -0,0 +1,6 @@ +resources: + jobs: + foo: + deployment: + kind: BUNDLE + metadata_file_path: /a/b/c diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml new file mode 100644 index 000000000..9cbe95f03 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml @@ -0,0 +1,6 @@ +targets: + foo: + resources: + jobs: + bar: + edit_mode: whatever diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml new file mode 100644 index 000000000..c57a560aa --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + git_snapshot: + used_commit: abcdef diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml new file mode 100644 index 000000000..9973e3bd4 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml @@ -0,0 +1,9 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + job_source: + import_from_git_branch: master + job_config_path: def diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml index d9b0e832f..e13a52c03 100644 --- a/bundle/internal/schema/testdata/pass/job.yml +++ b/bundle/internal/schema/testdata/pass/job.yml @@ -32,7 +32,6 @@ resources: name: myjob continuous: pause_status: PAUSED - edit_mode: EDITABLE max_concurrent_runs: 10 description: "my job description" email_notifications: @@ -43,10 +42,12 @@ resources: dependencies: - python=3.7 client: "myclient" - format: MULTI_TASK tags: foo: bar bar: baz + git_source: + git_provider: gitHub + git_url: www.github.com/a/b tasks: - task_key: mytask notebook_task: diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index b71ea7d1c..0c4244089 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.54.0" diff --git a/bundle/internal/tf/schema/data_source_current_metastore.go b/bundle/internal/tf/schema/data_source_current_metastore.go index 11e647fd3..4f8c135a5 100644 --- a/bundle/internal/tf/schema/data_source_current_metastore.go +++ b/bundle/internal/tf/schema/data_source_current_metastore.go @@ -10,6 +10,7 @@ type DataSourceCurrentMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_metastore.go b/bundle/internal/tf/schema/data_source_metastore.go index ce2064794..4244febc6 100644 --- a/bundle/internal/tf/schema/data_source_metastore.go +++ b/bundle/internal/tf/schema/data_source_metastore.go @@ -10,6 +10,7 @@ type DataSourceMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_mlflow_models.go b/bundle/internal/tf/schema/data_source_mlflow_models.go new file mode 100644 index 000000000..360924e5c --- /dev/null +++ b/bundle/internal/tf/schema/data_source_mlflow_models.go @@ -0,0 +1,8 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceMlflowModels struct { + Id string `json:"id,omitempty"` + Names []string `json:"names,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_notification_destinations.go b/bundle/internal/tf/schema/data_source_notification_destinations.go new file mode 100644 index 000000000..c95ad6db9 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_notification_destinations.go @@ -0,0 +1,15 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceNotificationDestinationsNotificationDestinations struct { + DestinationType string `json:"destination_type,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Id string `json:"id,omitempty"` +} + +type DataSourceNotificationDestinations struct { + DisplayNameContains string `json:"display_name_contains,omitempty"` + Type string `json:"type,omitempty"` + NotificationDestinations []DataSourceNotificationDestinationsNotificationDestinations `json:"notification_destinations,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_registered_model.go b/bundle/internal/tf/schema/data_source_registered_model.go new file mode 100644 index 000000000..e19e0849a --- /dev/null +++ b/bundle/internal/tf/schema/data_source_registered_model.go @@ -0,0 +1,32 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceRegisteredModelModelInfoAliases struct { + AliasName string `json:"alias_name,omitempty"` + VersionNum int `json:"version_num,omitempty"` +} + +type DataSourceRegisteredModelModelInfo struct { + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Aliases []DataSourceRegisteredModelModelInfoAliases `json:"aliases,omitempty"` +} + +type DataSourceRegisteredModel struct { + FullName string `json:"full_name"` + IncludeAliases bool `json:"include_aliases,omitempty"` + IncludeBrowse bool `json:"include_browse,omitempty"` + ModelInfo []DataSourceRegisteredModelModelInfo `json:"model_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 4ac78613f..050e0bc1d 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -30,12 +30,15 @@ type DataSources struct { Metastores map[string]any `json:"databricks_metastores,omitempty"` MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MlflowModels map[string]any `json:"databricks_mlflow_models,omitempty"` MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` NodeType map[string]any `json:"databricks_node_type,omitempty"` Notebook map[string]any `json:"databricks_notebook,omitempty"` NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` + NotificationDestinations map[string]any `json:"databricks_notification_destinations,omitempty"` Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` Schema map[string]any `json:"databricks_schema,omitempty"` Schemas map[string]any `json:"databricks_schemas,omitempty"` ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` @@ -85,12 +88,15 @@ func NewDataSources() *DataSources { Metastores: make(map[string]any), MlflowExperiment: make(map[string]any), MlflowModel: make(map[string]any), + MlflowModels: make(map[string]any), MwsCredentials: make(map[string]any), MwsWorkspaces: make(map[string]any), NodeType: make(map[string]any), Notebook: make(map[string]any), NotebookPaths: make(map[string]any), + NotificationDestinations: make(map[string]any), Pipelines: make(map[string]any), + RegisteredModel: make(map[string]any), Schema: make(map[string]any), Schemas: make(map[string]any), ServicePrincipal: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_budget.go b/bundle/internal/tf/schema/resource_budget.go new file mode 100644 index 000000000..5566eb939 --- /dev/null +++ b/bundle/internal/tf/schema/resource_budget.go @@ -0,0 +1,49 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceBudgetAlertConfigurationsActionConfigurations struct { + ActionConfigurationId string `json:"action_configuration_id,omitempty"` + ActionType string `json:"action_type,omitempty"` + Target string `json:"target,omitempty"` +} + +type ResourceBudgetAlertConfigurations struct { + AlertConfigurationId string `json:"alert_configuration_id,omitempty"` + QuantityThreshold string `json:"quantity_threshold,omitempty"` + QuantityType string `json:"quantity_type,omitempty"` + TimePeriod string `json:"time_period,omitempty"` + TriggerType string `json:"trigger_type,omitempty"` + ActionConfigurations []ResourceBudgetAlertConfigurationsActionConfigurations `json:"action_configurations,omitempty"` +} + +type ResourceBudgetFilterTagsValue struct { + Operator string `json:"operator,omitempty"` + Values []string `json:"values,omitempty"` +} + +type ResourceBudgetFilterTags struct { + Key string `json:"key,omitempty"` + Value *ResourceBudgetFilterTagsValue `json:"value,omitempty"` +} + +type ResourceBudgetFilterWorkspaceId struct { + Operator string `json:"operator,omitempty"` + Values []int `json:"values,omitempty"` +} + +type ResourceBudgetFilter struct { + Tags []ResourceBudgetFilterTags `json:"tags,omitempty"` + WorkspaceId *ResourceBudgetFilterWorkspaceId `json:"workspace_id,omitempty"` +} + +type ResourceBudget struct { + AccountId string `json:"account_id,omitempty"` + BudgetConfigurationId string `json:"budget_configuration_id,omitempty"` + CreateTime int `json:"create_time,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Id string `json:"id,omitempty"` + UpdateTime int `json:"update_time,omitempty"` + AlertConfigurations []ResourceBudgetAlertConfigurations `json:"alert_configurations,omitempty"` + Filter *ResourceBudgetFilter `json:"filter,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 42b648b0f..c89eafab9 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -1448,6 +1448,7 @@ type ResourceJobWebhookNotifications struct { type ResourceJob struct { AlwaysRunning bool `json:"always_running,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` ControlRunState bool `json:"control_run_state,omitempty"` Description string `json:"description,omitempty"` EditMode string `json:"edit_mode,omitempty"` diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 29d55cd5f..71cf8925d 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -2,6 +2,57 @@ package schema +type ResourceModelServingAiGatewayGuardrailsInputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsInput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsInputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrailsOutputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsOutput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsOutputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrails struct { + Input *ResourceModelServingAiGatewayGuardrailsInput `json:"input,omitempty"` + Output *ResourceModelServingAiGatewayGuardrailsOutput `json:"output,omitempty"` +} + +type ResourceModelServingAiGatewayInferenceTableConfig struct { + CatalogName string `json:"catalog_name,omitempty"` + Enabled bool `json:"enabled,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + TableNamePrefix string `json:"table_name_prefix,omitempty"` +} + +type ResourceModelServingAiGatewayRateLimits struct { + Calls int `json:"calls"` + Key string `json:"key,omitempty"` + RenewalPeriod string `json:"renewal_period"` +} + +type ResourceModelServingAiGatewayUsageTrackingConfig struct { + Enabled bool `json:"enabled,omitempty"` +} + +type ResourceModelServingAiGateway struct { + Guardrails *ResourceModelServingAiGatewayGuardrails `json:"guardrails,omitempty"` + InferenceTableConfig *ResourceModelServingAiGatewayInferenceTableConfig `json:"inference_table_config,omitempty"` + RateLimits []ResourceModelServingAiGatewayRateLimits `json:"rate_limits,omitempty"` + UsageTrackingConfig *ResourceModelServingAiGatewayUsageTrackingConfig `json:"usage_tracking_config,omitempty"` +} + type ResourceModelServingConfigAutoCaptureConfig struct { CatalogName string `json:"catalog_name,omitempty"` Enabled bool `json:"enabled,omitempty"` @@ -139,6 +190,7 @@ type ResourceModelServing struct { Name string `json:"name"` RouteOptimized bool `json:"route_optimized,omitempty"` ServingEndpointId string `json:"serving_endpoint_id,omitempty"` + AiGateway *ResourceModelServingAiGateway `json:"ai_gateway,omitempty"` Config *ResourceModelServingConfig `json:"config,omitempty"` RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"` Tags []ResourceModelServingTags `json:"tags,omitempty"` diff --git a/bundle/internal/tf/schema/resource_online_table.go b/bundle/internal/tf/schema/resource_online_table.go index de671eade..58d6f4ba5 100644 --- a/bundle/internal/tf/schema/resource_online_table.go +++ b/bundle/internal/tf/schema/resource_online_table.go @@ -19,9 +19,10 @@ type ResourceOnlineTableSpec struct { } type ResourceOnlineTable struct { - Id string `json:"id,omitempty"` - Name string `json:"name"` - Status []any `json:"status,omitempty"` - TableServingUrl string `json:"table_serving_url,omitempty"` - Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + Status []any `json:"status,omitempty"` + TableServingUrl string `json:"table_serving_url,omitempty"` + UnityCatalogProvisioningState string `json:"unity_catalog_provisioning_state,omitempty"` + Spec *ResourceOnlineTableSpec `json:"spec,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index ee94a1a8f..0c3b90ed3 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -4,7 +4,7 @@ package schema type ResourcePermissionsAccessControl struct { GroupName string `json:"group_name,omitempty"` - PermissionLevel string `json:"permission_level"` + PermissionLevel string `json:"permission_level,omitempty"` ServicePrincipalName string `json:"service_principal_name,omitempty"` UserName string `json:"user_name,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 154686463..2cb459aba 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -142,10 +142,26 @@ type ResourcePipelineGatewayDefinition struct { GatewayStorageSchema string `json:"gateway_storage_schema,omitempty"` } +type ResourcePipelineIngestionDefinitionObjectsReportTableConfiguration struct { + PrimaryKeys []string `json:"primary_keys,omitempty"` + SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` + ScdType string `json:"scd_type,omitempty"` + SequenceBy []string `json:"sequence_by,omitempty"` +} + +type ResourcePipelineIngestionDefinitionObjectsReport struct { + DestinationCatalog string `json:"destination_catalog,omitempty"` + DestinationSchema string `json:"destination_schema,omitempty"` + DestinationTable string `json:"destination_table,omitempty"` + SourceUrl string `json:"source_url,omitempty"` + TableConfiguration *ResourcePipelineIngestionDefinitionObjectsReportTableConfiguration `json:"table_configuration,omitempty"` +} + type ResourcePipelineIngestionDefinitionObjectsSchemaTableConfiguration struct { PrimaryKeys []string `json:"primary_keys,omitempty"` SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` + SequenceBy []string `json:"sequence_by,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsSchema struct { @@ -160,6 +176,7 @@ type ResourcePipelineIngestionDefinitionObjectsTableTableConfiguration struct { PrimaryKeys []string `json:"primary_keys,omitempty"` SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` + SequenceBy []string `json:"sequence_by,omitempty"` } type ResourcePipelineIngestionDefinitionObjectsTable struct { @@ -173,6 +190,7 @@ type ResourcePipelineIngestionDefinitionObjectsTable struct { } type ResourcePipelineIngestionDefinitionObjects struct { + Report *ResourcePipelineIngestionDefinitionObjectsReport `json:"report,omitempty"` Schema *ResourcePipelineIngestionDefinitionObjectsSchema `json:"schema,omitempty"` Table *ResourcePipelineIngestionDefinitionObjectsTable `json:"table,omitempty"` } @@ -181,6 +199,7 @@ type ResourcePipelineIngestionDefinitionTableConfiguration struct { PrimaryKeys []string `json:"primary_keys,omitempty"` SalesforceIncludeFormulaFields bool `json:"salesforce_include_formula_fields,omitempty"` ScdType string `json:"scd_type,omitempty"` + SequenceBy []string `json:"sequence_by,omitempty"` } type ResourcePipelineIngestionDefinition struct { @@ -238,6 +257,7 @@ type ResourcePipelineTrigger struct { type ResourcePipeline struct { AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` Catalog string `json:"catalog,omitempty"` Cause string `json:"cause,omitempty"` Channel string `json:"channel,omitempty"` @@ -254,6 +274,7 @@ type ResourcePipeline struct { Name string `json:"name,omitempty"` Photon bool `json:"photon,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` + Schema string `json:"schema,omitempty"` Serverless bool `json:"serverless,omitempty"` State string `json:"state,omitempty"` Storage string `json:"storage,omitempty"` diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 4f305c52e..bcf2a8e84 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -4,9 +4,11 @@ package schema type ResourceSqlTableColumn struct { Comment string `json:"comment,omitempty"` + Identity string `json:"identity,omitempty"` Name string `json:"name"` Nullable bool `json:"nullable,omitempty"` Type string `json:"type,omitempty"` + TypeJson string `json:"type_json,omitempty"` } type ResourceSqlTable struct { diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 737b77a2a..53f558df6 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -10,6 +10,7 @@ type Resources struct { AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` + Budget map[string]any `json:"databricks_budget,omitempty"` Catalog map[string]any `json:"databricks_catalog,omitempty"` CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` Cluster map[string]any `json:"databricks_cluster,omitempty"` @@ -112,6 +113,7 @@ func NewResources() *Resources { AzureAdlsGen1Mount: make(map[string]any), AzureAdlsGen2Mount: make(map[string]any), AzureBlobMount: make(map[string]any), + Budget: make(map[string]any), Catalog: make(map[string]any), CatalogWorkspaceBinding: make(map[string]any), Cluster: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 5fc34d6b4..bf4283c9b 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.54.0" func NewRoot() *Root { return &Root{ diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 6d60d56bc..53b714104 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -57,6 +57,12 @@ func IsLibraryLocal(dep string) bool { } } + // If the dependency starts with --, it's a pip flag option which is a valid + // entry for environment dependencies but not a local path + if containsPipFlag(dep) { + return false + } + // If the dependency is a requirements file, it's not a valid local path if strings.HasPrefix(dep, "-r") { return false @@ -70,6 +76,11 @@ func IsLibraryLocal(dep string) bool { return IsLocalPath(dep) } +func containsPipFlag(input string) bool { + re := regexp.MustCompile(`--[a-zA-Z0-9-]+`) + return re.MatchString(input) +} + // ^[a-zA-Z0-9\-_]+: Matches the package name, allowing alphanumeric characters, dashes (-), and underscores (_). // \[.*\])?: Optionally matches any extras specified in square brackets, e.g., [security]. // ((==|!=|<=|>=|~=|>|<)\d+(\.\d+){0,2}(\.\*)?): Optionally matches version specifiers, supporting various operators (==, !=, etc.) followed by a version number (e.g., 2.25.1). diff --git a/bundle/libraries/workspace_path.go b/bundle/libraries/path.go similarity index 75% rename from bundle/libraries/workspace_path.go rename to bundle/libraries/path.go index 126ad3f13..418d9ca73 100644 --- a/bundle/libraries/workspace_path.go +++ b/bundle/libraries/path.go @@ -36,3 +36,13 @@ func IsWorkspaceLibrary(library *compute.Library) bool { return IsWorkspacePath(path) } + +// IsVolumesPath returns true if the specified path indicates that +// it should be interpreted as a Databricks Volumes path. +func IsVolumesPath(path string) bool { + return strings.HasPrefix(path, "/Volumes/") +} + +func IsWorkspaceSharedPath(path string) bool { + return strings.HasPrefix(path, "/Workspace/Shared/") +} diff --git a/bundle/libraries/workspace_path_test.go b/bundle/libraries/path_test.go similarity index 77% rename from bundle/libraries/workspace_path_test.go rename to bundle/libraries/path_test.go index feaaab7f7..90fe187a2 100644 --- a/bundle/libraries/workspace_path_test.go +++ b/bundle/libraries/path_test.go @@ -31,3 +31,13 @@ func TestIsWorkspaceLibrary(t *testing.T) { // Empty. assert.False(t, IsWorkspaceLibrary(&compute.Library{})) } + +func TestIsVolumesPath(t *testing.T) { + // Absolute paths with particular prefixes. + assert.True(t, IsVolumesPath("/Volumes/path/to/package")) + + // Relative paths. + assert.False(t, IsVolumesPath("myfile.txt")) + assert.False(t, IsVolumesPath("./myfile.txt")) + assert.False(t, IsVolumesPath("../myfile.txt")) +} diff --git a/bundle/paths/paths.go b/bundle/paths/paths.go new file mode 100644 index 000000000..50b75a6cd --- /dev/null +++ b/bundle/paths/paths.go @@ -0,0 +1,39 @@ +package paths + +import ( + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/libraries" +) + +func CollectUniqueWorkspacePathPrefixes(workspace config.Workspace) []string { + rootPath := workspace.RootPath + paths := []string{} + if !libraries.IsVolumesPath(rootPath) && !libraries.IsWorkspaceSharedPath(rootPath) { + paths = append(paths, rootPath) + } + + if !strings.HasSuffix(rootPath, "/") { + rootPath += "/" + } + + for _, p := range []string{ + workspace.ArtifactPath, + workspace.FilePath, + workspace.StatePath, + workspace.ResourcePath, + } { + if libraries.IsWorkspaceSharedPath(p) || libraries.IsVolumesPath(p) { + continue + } + + if strings.HasPrefix(p, rootPath) { + continue + } + + paths = append(paths, p) + } + + return paths +} diff --git a/bundle/permissions/mutator.go b/bundle/permissions/mutator.go index 7787bc048..bc1392d93 100644 --- a/bundle/permissions/mutator.go +++ b/bundle/permissions/mutator.go @@ -39,6 +39,10 @@ var levelsMap = map[string](map[string]string){ CAN_VIEW: "CAN_VIEW", CAN_RUN: "CAN_QUERY", }, + "dashboards": { + CAN_MANAGE: "CAN_MANAGE", + CAN_VIEW: "CAN_READ", + }, } type bundlePermissions struct{} diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go new file mode 100644 index 000000000..d2c24fa01 --- /dev/null +++ b/bundle/permissions/permission_diagnostics.go @@ -0,0 +1,110 @@ +package permissions + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/set" +) + +type permissionDiagnostics struct{} + +func PermissionDiagnostics() bundle.Mutator { + return &permissionDiagnostics{} +} + +func (m *permissionDiagnostics) Name() string { + return "CheckPermissions" +} + +func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if len(b.Config.Permissions) == 0 { + // Only warn if there is an explicit top-level permissions section + return nil + } + + canManageBundle, _ := analyzeBundlePermissions(b) + if canManageBundle { + return nil + } + + return diag.Diagnostics{{ + Severity: diag.Warning, + Summary: fmt.Sprintf("permissions section should include %s or one of their groups with CAN_MANAGE permissions", b.Config.Workspace.CurrentUser.UserName), + Locations: []dyn.Location{b.Config.GetLocation("permissions")}, + ID: diag.PermissionNotIncluded, + }} +} + +// analyzeBundlePermissions analyzes the top-level permissions of the bundle. +// This permission set is important since it determines the permissions of the +// target workspace folder. +// +// Returns: +// - isManager: true if the current user is can manage the bundle resources. +// - assistance: advice on who to contact as to manage this project +func analyzeBundlePermissions(b *bundle.Bundle) (bool, string) { + canManageBundle := false + otherManagers := set.NewSet[string]() + if b.Config.RunAs != nil && b.Config.RunAs.UserName != "" && b.Config.RunAs.UserName != b.Config.Workspace.CurrentUser.UserName { + // The run_as user is another human that could be contacted + // about this bundle. + otherManagers.Add(b.Config.RunAs.UserName) + } + + currentUser := b.Config.Workspace.CurrentUser.UserName + targetPermissions := b.Config.Permissions + for _, p := range targetPermissions { + if p.Level != CAN_MANAGE { + continue + } + + if p.UserName == currentUser || p.ServicePrincipalName == currentUser { + canManageBundle = true + continue + } + + if isGroupOfCurrentUser(b, p.GroupName) { + canManageBundle = true + continue + } + + // Permission doesn't apply to current user; add to otherManagers + otherManager := p.UserName + if otherManager == "" { + otherManager = p.GroupName + } + if otherManager == "" { + // Skip service principals + continue + } + otherManagers.Add(otherManager) + } + + assistance := "For assistance, contact the owners of this project." + if otherManagers.Size() > 0 { + list := otherManagers.Values() + sort.Strings(list) + assistance = fmt.Sprintf( + "For assistance, users or groups with appropriate permissions may include: %s.", + strings.Join(list, ", "), + ) + } + return canManageBundle, assistance +} + +func isGroupOfCurrentUser(b *bundle.Bundle, groupName string) bool { + currentUserGroups := b.Config.Workspace.CurrentUser.User.Groups + + for _, g := range currentUserGroups { + if g.Display == groupName { + return true + } + } + return false +} diff --git a/bundle/permissions/permission_diagnostics_test.go b/bundle/permissions/permission_diagnostics_test.go new file mode 100644 index 000000000..7b0afefa0 --- /dev/null +++ b/bundle/permissions/permission_diagnostics_test.go @@ -0,0 +1,52 @@ +package permissions_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestPermissionDiagnosticsApplySuccess(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) + require.NoError(t, diags.Error()) +} + +func TestPermissionDiagnosticsApplyFail(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_VIEW", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.PermissionDiagnostics().Apply(context.Background(), b) + require.Equal(t, diags[0].Severity, diag.Warning) + require.Contains(t, diags[0].Summary, "permissions section should include testuser@databricks.com or one of their groups with CAN_MANAGE permissions") +} + +func mockBundle(permissions []resources.Permission) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{ + UserName: "testuser@databricks.com", + DisplayName: "Test User", + Groups: []iam.ComplexValue{ + {Display: "testgroup"}, + }, + }, + }, + }, + Permissions: permissions, + }, + } +} diff --git a/bundle/permissions/permission_report.go b/bundle/permissions/permission_report.go new file mode 100644 index 000000000..36526eeeb --- /dev/null +++ b/bundle/permissions/permission_report.go @@ -0,0 +1,52 @@ +package permissions + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/log" +) + +// ReportPossiblePermissionDenied generates a diagnostic message when a permission denied error is encountered. +// +// Note that since the workspace API doesn't always distinguish between permission denied and path errors, +// we must treat this as a "possible permission error". See acquire.go for more about this. +func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) diag.Diagnostics { + log.Errorf(ctx, "Failed to update, encountered possible permission error: %v", path) + + me := b.Config.Workspace.CurrentUser.User + userName := me.UserName + if iamutil.IsServicePrincipal(me) { + userName = me.DisplayName + } + canManageBundle, assistance := analyzeBundlePermissions(b) + + if !canManageBundle { + return diag.Diagnostics{{ + Summary: fmt.Sprintf("unable to deploy to %s as %s.\n"+ + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n"+ + "%s\n"+ + "They may need to redeploy the bundle to apply the new permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + path, userName, assistance), + Severity: diag.Error, + ID: diag.PathPermissionDenied, + }} + } + + // According databricks.yml, the current user has the right permissions. + // But we're still seeing permission errors. So someone else will need + // to redeploy the bundle with the right set of permissions. + return diag.Diagnostics{{ + Summary: fmt.Sprintf("unable to deploy to %s as %s. Cannot apply local deployment permissions.\n"+ + "%s\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + path, userName, assistance), + Severity: diag.Error, + ID: diag.CannotChangePathPermissions, + }} +} diff --git a/bundle/permissions/permission_report_test.go b/bundle/permissions/permission_report_test.go new file mode 100644 index 000000000..61592f7e5 --- /dev/null +++ b/bundle/permissions/permission_report_test.go @@ -0,0 +1,76 @@ +package permissions_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/stretchr/testify/require" +) + +func TestPermissionsReportPermissionDeniedWithGroup(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "testgroup"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n" + + "For assistance, contact the owners of this project.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedWithOtherGroup(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "othergroup"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, users or groups with appropriate permissions may include: othergroup.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedWithoutPermission(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_VIEW", UserName: "testuser@databricks.com"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, contact the owners of this project.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportPermissionDeniedNilPermission(t *testing.T) { + b := mockBundle(nil) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + expected := "EPERM1: unable to deploy to testpath as testuser@databricks.com.\n" + + "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n" + + "For assistance, contact the owners of this project.\n" + + "They may need to redeploy the bundle to apply the new permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" + require.ErrorContains(t, diags.Error(), expected) +} + +func TestPermissionsReportFindOtherOwners(t *testing.T) { + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "testgroup"}, + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + }) + + diags := permissions.ReportPossiblePermissionDenied(context.Background(), b, "testpath") + require.ErrorContains(t, diags.Error(), "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n"+ + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.") +} diff --git a/bundle/permissions/terraform_errors.go b/bundle/permissions/terraform_errors.go new file mode 100644 index 000000000..cc4c9f614 --- /dev/null +++ b/bundle/permissions/terraform_errors.go @@ -0,0 +1,47 @@ +package permissions + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" +) + +func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) diag.Diagnostics { + _, assistance := analyzeBundlePermissions(b) + + // In a best-effort attempt to provide actionable error messages, we match + // against a few specific error messages that come from the Jobs and Pipelines API. + // For matching errors we provide a more specific error message that includes + // details on how to resolve the issue. + if !strings.Contains(err.Error(), "cannot update permissions") && + !strings.Contains(err.Error(), "permissions on pipeline") && + !strings.Contains(err.Error(), "cannot read permissions") && + !strings.Contains(err.Error(), "cannot set run_as to user") { + return nil + } + + log.Errorf(ctx, "Terraform error during deployment: %v", err.Error()) + + // Best-effort attempt to extract the resource name from the error message. + re := regexp.MustCompile(`databricks_(\w*)\.(\w*)`) + match := re.FindStringSubmatch(err.Error()) + resource := "resource" + if len(match) > 1 { + resource = match[2] + } + + return diag.Diagnostics{{ + Summary: fmt.Sprintf("permission denied creating or updating %s.\n"+ + "%s\n"+ + "They can redeploy the project to apply the latest set of permissions.\n"+ + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", + resource, assistance), + Severity: diag.Error, + ID: diag.ResourcePermissionDenied, + }} +} diff --git a/bundle/permissions/terraform_errors_test.go b/bundle/permissions/terraform_errors_test.go new file mode 100644 index 000000000..c9b5b1786 --- /dev/null +++ b/bundle/permissions/terraform_errors_test.go @@ -0,0 +1,97 @@ +package permissions_test + +import ( + "context" + "errors" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/permissions" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestTryExtendTerraformPermissionError1(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot update permissions: ...\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" + + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionError2(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, + {Level: "CAN_MANAGE", UserName: "bob@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com, bob@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionError3(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "testuser@databricks.com"}, + }) + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read permissions: 1706906c-c0a2-4c25-9f57-3a7aa3cb8b90 does not have Owner permissions on Job with ID: ElasticJobId(28263044278868). Please contact the owner or an administrator for access.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, contact the owners of this project.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} + +func TestTryExtendTerraformPermissionErrorNotOwner(t *testing.T) { + ctx := context.Background() + b := mockBundle([]resources.Permission{ + {Level: "CAN_MANAGE", GroupName: "data_team@databricks.com"}, + }) + b.Config.RunAs = &jobs.JobRunAs{ + UserName: "testuser@databricks.com", + } + err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ + "\n"+ + "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ + "\n"+ + " with databricks_pipeline.my_project_pipeline,\n"+ + " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ + " 39: }")).Error() + + expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + + "For assistance, users or groups with appropriate permissions may include: data_team@databricks.com.\n" + + "They can redeploy the project to apply the latest set of permissions.\n" + + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." + require.ErrorContains(t, err, expected) +} diff --git a/bundle/permissions/validate.go b/bundle/permissions/validate.go new file mode 100644 index 000000000..f1a18f430 --- /dev/null +++ b/bundle/permissions/validate.go @@ -0,0 +1,52 @@ +package permissions + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/diag" +) + +type validateSharedRootPermissions struct { +} + +func ValidateSharedRootPermissions() bundle.Mutator { + return &validateSharedRootPermissions{} +} + +func (*validateSharedRootPermissions) Name() string { + return "ValidateSharedRootPermissions" +} + +func (*validateSharedRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if libraries.IsWorkspaceSharedPath(b.Config.Workspace.RootPath) { + return isUsersGroupPermissionSet(b) + } + + return nil +} + +// isUsersGroupPermissionSet checks that top-level permissions set for bundle contain group_name: users with CAN_MANAGE permission. +func isUsersGroupPermissionSet(b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + allUsers := false + for _, p := range b.Config.Permissions { + if p.GroupName == "users" && p.Level == CAN_MANAGE { + allUsers = true + break + } + } + + if !allUsers { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("the bundle root path %s is writable by all workspace users", b.Config.Workspace.RootPath), + Detail: "The bundle is configured to use /Workspace/Shared, which will give read/write access to all users. If this is intentional, add CAN_MANAGE for 'group_name: users' permission to your bundle configuration. If the deployment should be restricted, move it to a restricted folder such as /Workspace/Users/.", + }) + } + + return diags +} diff --git a/bundle/permissions/validate_test.go b/bundle/permissions/validate_test.go new file mode 100644 index 000000000..ff132b4e1 --- /dev/null +++ b/bundle/permissions/validate_test.go @@ -0,0 +1,66 @@ +package permissions + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestValidateSharedRootPermissionsForShared(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Shared/foo/bar", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, GroupName: "users"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, + }, + }, + }, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ValidateSharedRootPermissions())) + require.Empty(t, diags) +} + +func TestValidateSharedRootPermissionsForSharedError(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Shared/foo/bar", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "foo@bar.com"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, + }, + }, + }, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ValidateSharedRootPermissions())) + require.Len(t, diags, 1) + require.Equal(t, "the bundle root path /Workspace/Shared/foo/bar is writable by all workspace users", diags[0].Summary) + require.Equal(t, diag.Warning, diags[0].Severity) +} diff --git a/bundle/permissions/workspace_path_permissions.go b/bundle/permissions/workspace_path_permissions.go new file mode 100644 index 000000000..a3b4424c1 --- /dev/null +++ b/bundle/permissions/workspace_path_permissions.go @@ -0,0 +1,89 @@ +package permissions + +import ( + "fmt" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type WorkspacePathPermissions struct { + Path string + Permissions []resources.Permission +} + +func ObjectAclToResourcePermissions(path string, acl []workspace.WorkspaceObjectAccessControlResponse) *WorkspacePathPermissions { + permissions := make([]resources.Permission, 0) + for _, a := range acl { + // Skip the admin group because it's added to all resources by default. + if a.GroupName == "admins" { + continue + } + + for _, pl := range a.AllPermissions { + permissions = append(permissions, resources.Permission{ + Level: convertWorkspaceObjectPermissionLevel(pl.PermissionLevel), + GroupName: a.GroupName, + UserName: a.UserName, + ServicePrincipalName: a.ServicePrincipalName, + }) + } + } + + return &WorkspacePathPermissions{Permissions: permissions, Path: path} +} + +func (p WorkspacePathPermissions) Compare(perms []resources.Permission) diag.Diagnostics { + var diags diag.Diagnostics + + // Check the permissions in the workspace and see if they are all set in the bundle. + ok, missing := containsAll(p.Permissions, perms) + if !ok { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "untracked permissions apply to target workspace path", + Detail: fmt.Sprintf("The following permissions apply to the workspace folder at %q but are not configured in the bundle:\n%s", p.Path, toString(missing)), + }) + } + + return diags +} + +// containsAll checks if permA contains all permissions in permB. +func containsAll(permA []resources.Permission, permB []resources.Permission) (bool, []resources.Permission) { + missing := make([]resources.Permission, 0) + for _, a := range permA { + found := false + for _, b := range permB { + if a == b { + found = true + break + } + } + if !found { + missing = append(missing, a) + } + } + return len(missing) == 0, missing +} + +// convertWorkspaceObjectPermissionLevel converts matching object permission levels to bundle ones. +// If there is no matching permission level, it returns permission level as is, for example, CAN_EDIT. +func convertWorkspaceObjectPermissionLevel(level workspace.WorkspaceObjectPermissionLevel) string { + switch level { + case workspace.WorkspaceObjectPermissionLevelCanRead: + return CAN_VIEW + default: + return string(level) + } +} + +func toString(p []resources.Permission) string { + var sb strings.Builder + for _, perm := range p { + sb.WriteString(fmt.Sprintf("- %s\n", perm.String())) + } + return sb.String() +} diff --git a/bundle/permissions/workspace_path_permissions_test.go b/bundle/permissions/workspace_path_permissions_test.go new file mode 100644 index 000000000..0bb00474c --- /dev/null +++ b/bundle/permissions/workspace_path_permissions_test.go @@ -0,0 +1,121 @@ +package permissions + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/stretchr/testify/require" +) + +func TestWorkspacePathPermissionsCompare(t *testing.T) { + testCases := []struct { + perms []resources.Permission + acl []workspace.WorkspaceObjectAccessControlResponse + expected diag.Diagnostics + }{ + { + perms: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "foo@bar.com"}, + }, + acl: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + expected: nil, + }, + { + perms: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "foo@bar.com"}, + }, + acl: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + { + GroupName: "admins", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + expected: nil, + }, + { + perms: []resources.Permission{ + {Level: CAN_VIEW, UserName: "foo@bar.com"}, + {Level: CAN_MANAGE, ServicePrincipalName: "sp.com"}, + }, + acl: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_READ"}, + }, + }, + }, + expected: nil, + }, + { + perms: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "foo@bar.com"}, + }, + acl: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + { + GroupName: "foo", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + expected: diag.Diagnostics{ + { + Severity: diag.Warning, + Summary: "untracked permissions apply to target workspace path", + Detail: "The following permissions apply to the workspace folder at \"path\" but are not configured in the bundle:\n- level: CAN_MANAGE, group_name: foo\n", + }, + }, + }, + { + perms: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "foo@bar.com"}, + }, + acl: []workspace.WorkspaceObjectAccessControlResponse{ + { + UserName: "foo2@bar.com", + AllPermissions: []workspace.WorkspaceObjectPermission{ + {PermissionLevel: "CAN_MANAGE"}, + }, + }, + }, + expected: diag.Diagnostics{ + { + Severity: diag.Warning, + Summary: "untracked permissions apply to target workspace path", + Detail: "The following permissions apply to the workspace folder at \"path\" but are not configured in the bundle:\n- level: CAN_MANAGE, user_name: foo2@bar.com\n", + }, + }, + }, + } + + for _, tc := range testCases { + wp := ObjectAclToResourcePermissions("path", tc.acl) + diags := wp.Compare(tc.perms) + require.Equal(t, tc.expected, diags) + } + +} diff --git a/bundle/permissions/workspace_root.go b/bundle/permissions/workspace_root.go index a59a039f6..4ab8198bb 100644 --- a/bundle/permissions/workspace_root.go +++ b/bundle/permissions/workspace_root.go @@ -5,8 +5,10 @@ import ( "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/paths" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/workspace" + "golang.org/x/sync/errgroup" ) type workspaceRootPermissions struct { @@ -16,6 +18,10 @@ func ApplyWorkspaceRootPermissions() bundle.Mutator { return &workspaceRootPermissions{} } +func (*workspaceRootPermissions) Name() string { + return "ApplyWorkspaceRootPermissions" +} + // Apply implements bundle.Mutator. func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := giveAccessForWorkspaceRoot(ctx, b) @@ -26,15 +32,11 @@ func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) di return nil } -func (*workspaceRootPermissions) Name() string { - return "ApplyWorkspaceRootPermissions" -} - func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error { permissions := make([]workspace.WorkspaceObjectAccessControlRequest, 0) for _, p := range b.Config.Permissions { - level, err := getWorkspaceObjectPermissionLevel(p.Level) + level, err := GetWorkspaceObjectPermissionLevel(p.Level) if err != nil { return err } @@ -52,20 +54,34 @@ func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error { } w := b.WorkspaceClient().Workspace - obj, err := w.GetStatusByPath(ctx, b.Config.Workspace.RootPath) + bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config.Workspace) + + g, ctx := errgroup.WithContext(ctx) + for _, p := range bundlePaths { + g.Go(func() error { + return setPermissions(ctx, w, p, permissions) + }) + } + + return g.Wait() +} + +func setPermissions(ctx context.Context, w workspace.WorkspaceInterface, path string, permissions []workspace.WorkspaceObjectAccessControlRequest) error { + obj, err := w.GetStatusByPath(ctx, path) if err != nil { return err } - _, err = w.UpdatePermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{ + _, err = w.SetPermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{ WorkspaceObjectId: fmt.Sprint(obj.ObjectId), WorkspaceObjectType: "directories", AccessControlList: permissions, }) + return err } -func getWorkspaceObjectPermissionLevel(bundlePermission string) (workspace.WorkspaceObjectPermissionLevel, error) { +func GetWorkspaceObjectPermissionLevel(bundlePermission string) (workspace.WorkspaceObjectPermissionLevel, error) { switch bundlePermission { case CAN_MANAGE: return workspace.WorkspaceObjectPermissionLevelCanManage, nil diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index 5e23a1da8..c48704a63 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -21,7 +21,11 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Workspace: config.Workspace{ - RootPath: "/Users/foo@bar.com", + RootPath: "/Users/foo@bar.com", + ArtifactPath: "/Users/foo@bar.com/artifacts", + FilePath: "/Users/foo@bar.com/files", + StatePath: "/Users/foo@bar.com/state", + ResourcePath: "/Users/foo@bar.com/resources", }, Permissions: []resources.Permission{ {Level: CAN_MANAGE, UserName: "TestUser"}, @@ -59,7 +63,7 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com").Return(&workspace.ObjectInfo{ ObjectId: 1234, }, nil) - workspaceApi.EXPECT().UpdatePermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, @@ -69,6 +73,119 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { WorkspaceObjectType: "directories", }).Return(nil, nil) + diags := bundle.Apply(context.Background(), b, bundle.Seq(ValidateSharedRootPermissions(), ApplyWorkspaceRootPermissions())) + require.Empty(t, diags) +} + +func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Some/Root/Path", + ArtifactPath: "/Users/foo@bar.com/artifacts", + FilePath: "/Users/foo@bar.com/files", + StatePath: "/Users/foo@bar.com/state", + ResourcePath: "/Users/foo@bar.com/resources", + }, + Permissions: []resources.Permission{ + {Level: CAN_MANAGE, UserName: "TestUser"}, + {Level: CAN_VIEW, GroupName: "TestGroup"}, + {Level: CAN_RUN, ServicePrincipalName: "TestServicePrincipal"}, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, + "pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}}, + }, + Models: map[string]*resources.MlflowModel{ + "model_1": {Model: &ml.Model{}}, + "model_2": {Model: &ml.Model{}}, + }, + Experiments: map[string]*resources.MlflowExperiment{ + "experiment_1": {Experiment: &ml.Experiment{}}, + "experiment_2": {Experiment: &ml.Experiment{}}, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint_1": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + "endpoint_2": {CreateServingEndpoint: &serving.CreateServingEndpoint{}}, + }, + }, + }, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + workspaceApi := m.GetMockWorkspaceAPI() + workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Some/Root/Path").Return(&workspace.ObjectInfo{ + ObjectId: 1, + }, nil) + workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/artifacts").Return(&workspace.ObjectInfo{ + ObjectId: 2, + }, nil) + workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/files").Return(&workspace.ObjectInfo{ + ObjectId: 3, + }, nil) + workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/state").Return(&workspace.ObjectInfo{ + ObjectId: 4, + }, nil) + workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/resources").Return(&workspace.ObjectInfo{ + ObjectId: 5, + }, nil) + + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ + {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, + {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, + {ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"}, + }, + WorkspaceObjectId: "1", + WorkspaceObjectType: "directories", + }).Return(nil, nil) + + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ + {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, + {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, + {ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"}, + }, + WorkspaceObjectId: "2", + WorkspaceObjectType: "directories", + }).Return(nil, nil) + + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ + {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, + {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, + {ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"}, + }, + WorkspaceObjectId: "3", + WorkspaceObjectType: "directories", + }).Return(nil, nil) + + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ + {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, + {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, + {ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"}, + }, + WorkspaceObjectId: "4", + WorkspaceObjectType: "directories", + }).Return(nil, nil) + + workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{ + AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{ + {UserName: "TestUser", PermissionLevel: "CAN_MANAGE"}, + {GroupName: "TestGroup", PermissionLevel: "CAN_READ"}, + {ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"}, + }, + WorkspaceObjectId: "5", + WorkspaceObjectType: "directories", + }).Return(nil, nil) + diags := bundle.Apply(context.Background(), b, ApplyWorkspaceRootPermissions()) require.NoError(t, diags.Error()) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index cb0ecf75d..e623c364f 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -152,6 +152,7 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { bundle.Defer( bundle.Seq( terraform.StatePull(), + terraform.CheckDashboardsModifiedRemotely(), deploy.StatePull(), mutator.ValidateGitDetails(), artifacts.CleanUp(), diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a41819c76..3d5ad5e8b 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -62,8 +62,11 @@ func Initialize() bundle.Mutator { "workspace", "variables", ), + // Provide permission config errors & warnings after initializing all variables + permissions.PermissionDiagnostics(), mutator.SetRunAs(), mutator.OverrideCompute(), + mutator.ConfigureDashboardDefaults(), mutator.ProcessTargetMode(), mutator.ApplyPresets(), mutator.DefaultQueueing(), @@ -74,8 +77,11 @@ func Initialize() bundle.Mutator { mutator.TranslatePaths(), trampoline.WrapperWarning(), + + permissions.ValidateSharedRootPermissions(), permissions.ApplyBundlePermissions(), permissions.FilterCurrentUser(), + metadata.AnnotateJobs(), metadata.AnnotatePipelines(), terraform.Initialize(), diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index e1fad98a3..2f7affbf3 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -1,13 +1,16 @@ package render import ( + "context" "fmt" "io" "path/filepath" + "sort" "strings" "text/template" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/fatih/color" @@ -28,35 +31,7 @@ var renderFuncMap = template.FuncMap{ }, } -const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} -{{- range $index, $element := .Paths }} - {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} -{{- end }} -{{- range $index, $element := .Locations }} - {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} -{{- end }} -{{- if .Detail }} - -{{ .Detail }} -{{- end }} - -` - -const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} -{{- range $index, $element := .Paths }} - {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} -{{- end }} -{{- range $index, $element := .Locations }} - {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} -{{- end }} -{{- if .Detail }} - -{{ .Detail }} -{{- end }} - -` - -const summaryTemplate = `{{- if .Name -}} +const summaryHeaderTemplate = `{{- if .Name -}} Name: {{ .Name | bold }} {{- if .Target }} Target: {{ .Target | bold }} @@ -73,12 +48,30 @@ Workspace: Path: {{ .Path | bold }} {{- end }} {{- end }} +{{ end -}}` -{{ end -}} - -{{ .Trailer }} +const resourcesTemplate = `Resources: +{{- range . }} + {{ .GroupName }}: + {{- range .Resources }} + {{ .Key | bold }}: + Name: {{ .Name }} + URL: {{ if .URL }}{{ .URL | cyan }}{{ else }}{{ "(not deployed)" | cyan }}{{ end }} + {{- end }} +{{- end }} ` +type ResourceGroup struct { + GroupName string + Resources []ResourceInfo +} + +type ResourceInfo struct { + Key string + Name string + URL string +} + func pluralize(n int, singular, plural string) string { if n == 1 { return fmt.Sprintf("%d %s", n, singular) @@ -94,16 +87,27 @@ func buildTrailer(diags diag.Diagnostics) string { if warnings := len(diags.Filter(diag.Warning)); warnings > 0 { parts = append(parts, color.YellowString(pluralize(warnings, "warning", "warnings"))) } - if len(parts) > 0 { - return fmt.Sprintf("Found %s", strings.Join(parts, " and ")) - } else { - return color.GreenString("Validation OK!") + if recommendations := len(diags.Filter(diag.Recommendation)); recommendations > 0 { + parts = append(parts, color.BlueString(pluralize(recommendations, "recommendation", "recommendations"))) + } + switch { + case len(parts) >= 3: + first := strings.Join(parts[:len(parts)-1], ", ") + last := parts[len(parts)-1] + return fmt.Sprintf("Found %s, and %s\n", first, last) + case len(parts) == 2: + return fmt.Sprintf("Found %s and %s\n", parts[0], parts[1]) + case len(parts) == 1: + return fmt.Sprintf("Found %s\n", parts[0]) + default: + // No diagnostics to print. + return color.GreenString("Validation OK!\n") } } -func renderSummaryTemplate(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { +func renderSummaryHeaderTemplate(out io.Writer, b *bundle.Bundle) error { if b == nil { - return renderSummaryTemplate(out, &bundle.Bundle{}, diags) + return renderSummaryHeaderTemplate(out, &bundle.Bundle{}) } var currentUser = &iam.User{} @@ -114,33 +118,20 @@ func renderSummaryTemplate(out io.Writer, b *bundle.Bundle, diags diag.Diagnosti } } - t := template.Must(template.New("summary").Funcs(renderFuncMap).Parse(summaryTemplate)) + t := template.Must(template.New("summary").Funcs(renderFuncMap).Parse(summaryHeaderTemplate)) err := t.Execute(out, map[string]any{ - "Name": b.Config.Bundle.Name, - "Target": b.Config.Bundle.Target, - "User": currentUser.UserName, - "Path": b.Config.Workspace.RootPath, - "Host": b.Config.Workspace.Host, - "Trailer": buildTrailer(diags), + "Name": b.Config.Bundle.Name, + "Target": b.Config.Bundle.Target, + "User": currentUser.UserName, + "Path": b.Config.Workspace.RootPath, + "Host": b.Config.Workspace.Host, }) return err } -func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { - errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) - warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) - - // Print errors and warnings. +func renderDiagnosticsOnly(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { for _, d := range diags { - var t *template.Template - switch d.Severity { - case diag.Error: - t = errorT - case diag.Warning: - t = warningT - } - for i := range d.Locations { if b == nil { break @@ -155,15 +146,9 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) } } } - - // Render the diagnostic with the appropriate template. - err := t.Execute(out, d) - if err != nil { - return fmt.Errorf("failed to render template: %w", err) - } } - return nil + return cmdio.RenderDiagnostics(out, diags) } // RenderOptions contains options for rendering diagnostics. @@ -173,19 +158,73 @@ type RenderOptions struct { RenderSummaryTable bool } -// RenderTextOutput renders the diagnostics in a human-readable format. -func RenderTextOutput(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics, opts RenderOptions) error { - err := renderDiagnostics(out, b, diags) +// RenderDiagnostics renders the diagnostics in a human-readable format. +func RenderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics, opts RenderOptions) error { + err := renderDiagnosticsOnly(out, b, diags) if err != nil { return fmt.Errorf("failed to render diagnostics: %w", err) } if opts.RenderSummaryTable { - err = renderSummaryTemplate(out, b, diags) - if err != nil { - return fmt.Errorf("failed to render summary: %w", err) + if b != nil { + err = renderSummaryHeaderTemplate(out, b) + if err != nil { + return fmt.Errorf("failed to render summary: %w", err) + } + io.WriteString(out, "\n") } + trailer := buildTrailer(diags) + io.WriteString(out, trailer) } return nil } + +func RenderSummary(ctx context.Context, out io.Writer, b *bundle.Bundle) error { + if err := renderSummaryHeaderTemplate(out, b); err != nil { + return err + } + + var resourceGroups []ResourceGroup + + for _, group := range b.Config.Resources.AllResources() { + resources := make([]ResourceInfo, 0, len(group.Resources)) + for key, resource := range group.Resources { + resources = append(resources, ResourceInfo{ + Key: key, + Name: resource.GetName(), + URL: resource.GetURL(), + }) + } + + if len(resources) > 0 { + resourceGroups = append(resourceGroups, ResourceGroup{ + GroupName: group.Description.PluralTitle, + Resources: resources, + }) + } + } + + if err := renderResourcesTemplate(out, resourceGroups); err != nil { + return fmt.Errorf("failed to render resources template: %w", err) + } + + return nil +} + +// Helper function to sort and render resource groups using the template +func renderResourcesTemplate(out io.Writer, resourceGroups []ResourceGroup) error { + // Sort everything to ensure consistent output + sort.Slice(resourceGroups, func(i, j int) bool { + return resourceGroups[i].GroupName < resourceGroups[j].GroupName + }) + for _, group := range resourceGroups { + sort.Slice(group.Resources, func(i, j int) bool { + return group.Resources[i].Key < group.Resources[j].Key + }) + } + + t := template.Must(template.New("resources").Funcs(renderFuncMap).Parse(resourcesTemplate)) + + return t.Execute(out, resourceGroups) +} diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 976f86e79..cd9e7723b 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -2,14 +2,21 @@ package render import ( "bytes" + "context" + "io" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - assert "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,6 +52,19 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "Found 1 error\n", }, + { + name: "nil bundle and 1 recommendation", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "recommendation", + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Recommendation: recommendation\n" + + "\n" + + "Found 1 recommendation\n", + }, { name: "bundle during 'load' and 1 error", bundle: loadingBundle, @@ -84,7 +104,7 @@ func TestRenderTextOutput(t *testing.T) { "Found 2 warnings\n", }, { - name: "bundle during 'load' and 2 errors, 1 warning with details", + name: "bundle during 'load' and 2 errors, 1 warning and 1 recommendation with details", bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -105,6 +125,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (3)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: error (1)\n" + @@ -122,10 +148,114 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "detail (3)\n" + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + "Name: test-bundle\n" + "Target: test-target\n" + "\n" + - "Found 2 errors and 1 warning\n", + "Found 2 errors, 1 warning, and 1 recommendation\n", + }, + { + name: "bundle during 'load' and 1 error and 1 warning", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error and 1 warning\n", + }, + { + name: "bundle during 'load' and 1 errors, 2 warning and 2 recommendations with details", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (5)", + Detail: "detail (5)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Warning: warning (3)\n" + + " in foo.py:3:1\n" + + "\n" + + "detail (3)\n" + + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + + "Recommendation: recommendation (5)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (5)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error, 2 warnings, and 2 recommendations\n", }, { name: "bundle during 'init'", @@ -158,7 +288,7 @@ func TestRenderTextOutput(t *testing.T) { "Validation OK!\n", }, { - name: "nil bundle without summary with 1 error and 1 warning", + name: "nil bundle without summary with 1 error, 1 warning and 1 recommendation", bundle: nil, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -173,6 +303,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (2)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: false}, expected: "Error: error (1)\n" + @@ -184,6 +320,11 @@ func TestRenderTextOutput(t *testing.T) { " in foo.py:3:1\n" + "\n" + "detail (2)\n" + + "\n" + + "Recommendation: recommendation (3)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (3)\n" + "\n", }, } @@ -192,7 +333,7 @@ func TestRenderTextOutput(t *testing.T) { t.Run(tc.name, func(t *testing.T) { writer := &bytes.Buffer{} - err := RenderTextOutput(writer, tc.bundle, tc.diags, tc.opts) + err := RenderDiagnostics(writer, tc.bundle, tc.diags, tc.opts) require.NoError(t, err) assert.Equal(t, tc.expected, writer.String()) @@ -304,13 +445,37 @@ func TestRenderDiagnostics(t *testing.T) { "\n" + "'name' is required\n\n", }, + { + name: "recommendation with multiple paths and locations", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "summary", + Detail: "detail", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + }, + Locations: []dyn.Location{ + {File: "foo.yaml", Line: 1, Column: 2}, + {File: "bar.yaml", Line: 3, Column: 4}, + }, + }, + }, + expected: "Recommendation: summary\n" + + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "detail\n\n", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { writer := &bytes.Buffer{} - err := renderDiagnostics(writer, bundle, tc.diags) + err := renderDiagnosticsOnly(writer, bundle, tc.diags) require.NoError(t, err) assert.Equal(t, tc.expected, writer.String()) @@ -321,8 +486,105 @@ func TestRenderDiagnostics(t *testing.T) { func TestRenderSummaryTemplate_nilBundle(t *testing.T) { writer := &bytes.Buffer{} - err := renderSummaryTemplate(writer, nil, nil) + err := renderSummaryHeaderTemplate(writer, nil) require.NoError(t, err) + io.WriteString(writer, buildTrailer(nil)) + assert.Equal(t, "Validation OK!\n", writer.String()) } + +func TestRenderSummary(t *testing.T) { + ctx := context.Background() + + // Create a mock bundle with various resources + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Name: "test-bundle", + Target: "test-target", + }, + Workspace: config.Workspace{ + Host: "https://mycompany.databricks.com/", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + ID: "1", + URL: "https://url1", + JobSettings: &jobs.JobSettings{Name: "job1-name"}, + }, + "job2": { + ID: "2", + URL: "https://url2", + JobSettings: &jobs.JobSettings{Name: "job2-name"}, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline2": { + ID: "4", + // no URL + PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"}, + }, + "pipeline1": { + ID: "3", + URL: "https://url3", + PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"}, + }, + }, + Schemas: map[string]*resources.Schema{ + "schema1": { + ID: "catalog.schema", + CreateSchema: &catalog.CreateSchema{ + Name: "schema", + }, + // no URL + }, + }, + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint1": { + ID: "7", + CreateServingEndpoint: &serving.CreateServingEndpoint{ + Name: "my_serving_endpoint", + }, + URL: "https://url4", + }, + }, + }, + }, + } + + writer := &bytes.Buffer{} + err := RenderSummary(ctx, writer, b) + require.NoError(t, err) + + expectedSummary := `Name: test-bundle +Target: test-target +Workspace: + Host: https://mycompany.databricks.com/ +Resources: + Jobs: + job1: + Name: job1-name + URL: https://url1 + job2: + Name: job2-name + URL: https://url2 + Model Serving Endpoints: + endpoint1: + Name: my_serving_endpoint + URL: https://url4 + Pipelines: + pipeline1: + Name: pipeline1-name + URL: https://url3 + pipeline2: + Name: pipeline2-name + URL: (not deployed) + Schemas: + schema1: + Name: schema + URL: (not deployed) +` + assert.Equal(t, expectedSummary, writer.String()) +} diff --git a/bundle/resources/completion.go b/bundle/resources/completion.go new file mode 100644 index 000000000..c1bcd022f --- /dev/null +++ b/bundle/resources/completion.go @@ -0,0 +1,17 @@ +package resources + +import "github.com/databricks/cli/bundle" + +// Completions returns the same as [References] except +// that every key maps directly to a single reference. +func Completions(b *bundle.Bundle, filters ...Filter) map[string]Reference { + out := make(map[string]Reference) + keyOnlyRefs, _ := References(b, filters...) + for k, refs := range keyOnlyRefs { + if len(refs) != 1 { + continue + } + out[k] = refs[0] + } + return out +} diff --git a/bundle/resources/completion_test.go b/bundle/resources/completion_test.go new file mode 100644 index 000000000..2f7942aae --- /dev/null +++ b/bundle/resources/completion_test.go @@ -0,0 +1,58 @@ +package resources + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestCompletions_SkipDuplicates(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": {}, + "bar": {}, + }, + Pipelines: map[string]*resources.Pipeline{ + "foo": {}, + }, + }, + }, + } + + // Test that this skips duplicates and only includes unambiguous completions. + out := Completions(b) + if assert.Len(t, out, 1) { + assert.Contains(t, out, "bar") + } +} + +func TestCompletions_Filter(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": {}, + }, + Pipelines: map[string]*resources.Pipeline{ + "bar": {}, + }, + }, + }, + } + + includeJobs := func(ref Reference) bool { + _, ok := ref.Resource.(*resources.Job) + return ok + } + + // Test that this does not include the pipeline. + out := Completions(b, includeJobs) + if assert.Len(t, out, 1) { + assert.Contains(t, out, "foo") + } +} diff --git a/bundle/resources/lookup.go b/bundle/resources/lookup.go new file mode 100644 index 000000000..48d862230 --- /dev/null +++ b/bundle/resources/lookup.go @@ -0,0 +1,98 @@ +package resources + +import ( + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" +) + +// Reference is a reference to a resource. +// It includes the resource type description, and a reference to the resource itself. +type Reference struct { + // Key is the unique key of the resource, e.g. "my_job". + Key string + + // KeyWithType is the unique key of the resource, including the resource type, e.g. "jobs.my_job". + KeyWithType string + + // Description is the resource type description. + Description config.ResourceDescription + + // Resource is the resource itself. + Resource config.ConfigResource +} + +// Map is the core type for resource lookup and completion. +type Map map[string][]Reference + +// Filter defines the function signature for filtering resources. +type Filter func(Reference) bool + +// includeReference checks if the specified reference passes all filters. +// If the list of filters is empty, the reference is always included. +func includeReference(filters []Filter, ref Reference) bool { + for _, filter := range filters { + if !filter(ref) { + return false + } + } + return true +} + +// References returns maps of resource keys to a slice of [Reference]. +// +// The first map is indexed by the resource key only. +// The second map is indexed by the resource type name and its key. +// +// While the return types allows for multiple resources to share the same key, +// this is confirmed not to happen in the [validate.UniqueResourceKeys] mutator. +func References(b *bundle.Bundle, filters ...Filter) (Map, Map) { + keyOnly := make(Map) + keyWithType := make(Map) + + // Collect map of resource references indexed by their keys. + for _, group := range b.Config.Resources.AllResources() { + for k, v := range group.Resources { + ref := Reference{ + Key: k, + KeyWithType: fmt.Sprintf("%s.%s", group.Description.PluralName, k), + Description: group.Description, + Resource: v, + } + + // Skip resources that do not pass all filters. + if !includeReference(filters, ref) { + continue + } + + keyOnly[ref.Key] = append(keyOnly[ref.Key], ref) + keyWithType[ref.KeyWithType] = append(keyWithType[ref.KeyWithType], ref) + } + } + + return keyOnly, keyWithType +} + +// Lookup returns the resource with the specified key. +// If the key maps to more than one resource, an error is returned. +// If the key does not map to any resource, an error is returned. +func Lookup(b *bundle.Bundle, key string, filters ...Filter) (Reference, error) { + keyOnlyRefs, keyWithTypeRefs := References(b, filters...) + refs, ok := keyOnlyRefs[key] + if !ok { + refs, ok = keyWithTypeRefs[key] + if !ok { + return Reference{}, fmt.Errorf("resource with key %q not found", key) + } + } + + switch { + case len(refs) == 1: + return refs[0], nil + case len(refs) > 1: + return Reference{}, fmt.Errorf("multiple resources with key %q found", key) + default: + panic("unreachable") + } +} diff --git a/bundle/resources/lookup_test.go b/bundle/resources/lookup_test.go new file mode 100644 index 000000000..b2eaafd1a --- /dev/null +++ b/bundle/resources/lookup_test.go @@ -0,0 +1,117 @@ +package resources + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLookup_EmptyBundle(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{}, + }, + } + + _, err := Lookup(b, "foo") + require.Error(t, err) + assert.ErrorContains(t, err, "resource with key \"foo\" not found") +} + +func TestLookup_NotFound(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": {}, + "bar": {}, + }, + }, + }, + } + + _, err := Lookup(b, "qux") + require.Error(t, err) + assert.ErrorContains(t, err, `resource with key "qux" not found`) +} + +func TestLookup_MultipleFound(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": {}, + }, + Pipelines: map[string]*resources.Pipeline{ + "foo": {}, + }, + }, + }, + } + + _, err := Lookup(b, "foo") + require.Error(t, err) + assert.ErrorContains(t, err, `multiple resources with key "foo" found`) +} + +func TestLookup_Nominal(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Name: "Foo job", + }, + }, + }, + }, + }, + } + + // Lookup by key only. + out, err := Lookup(b, "foo") + if assert.NoError(t, err) { + assert.Equal(t, "Foo job", out.Resource.GetName()) + } + + // Lookup by type and key. + out, err = Lookup(b, "jobs.foo") + if assert.NoError(t, err) { + assert.Equal(t, "Foo job", out.Resource.GetName()) + } +} + +func TestLookup_NominalWithFilters(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": {}, + }, + Pipelines: map[string]*resources.Pipeline{ + "bar": {}, + }, + }, + }, + } + + includeJobs := func(ref Reference) bool { + _, ok := ref.Resource.(*resources.Job) + return ok + } + + // This should succeed because the filter includes jobs. + _, err := Lookup(b, "foo", includeJobs) + require.NoError(t, err) + + // This should fail because the filter excludes pipelines. + _, err = Lookup(b, "bar", includeJobs) + require.Error(t, err) + assert.ErrorContains(t, err, `resource with key "bar" not found`) +} diff --git a/bundle/run/job.go b/bundle/run/job.go index 8003c7d29..340af961c 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -317,6 +317,29 @@ func (r *jobRunner) Cancel(ctx context.Context) error { return errGroup.Wait() } +func (r *jobRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { + // We don't need to cancel existing runs if the job is continuous and unpaused. + // the /jobs/run-now API will automatically cancel any existing runs before starting a new one. + // + // /jobs/run-now will not cancel existing runs if the job is continuous and paused. + // New job runs will be queued instead and will wait for existing runs to finish. + // In this case, we need to cancel the existing runs before starting a new one. + continuous := r.job.JobSettings.Continuous + if continuous != nil && continuous.PauseStatus == jobs.PauseStatusUnpaused { + return r.Run(ctx, opts) + } + + s := cmdio.Spinner(ctx) + s <- "Cancelling all active job runs" + err := r.Cancel(ctx) + close(s) + if err != nil { + return nil, err + } + + return r.Run(ctx, opts) +} + func (r *jobRunner) ParseArgs(args []string, opts *Options) error { return r.posArgsHandler().ParseArgs(args, opts) } diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go index be189306b..369c546aa 100644 --- a/bundle/run/job_test.go +++ b/bundle/run/job_test.go @@ -1,6 +1,7 @@ package run import ( + "bytes" "context" "testing" "time" @@ -8,6 +9,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/mock" @@ -126,3 +129,132 @@ func TestJobRunnerCancelWithNoActiveRuns(t *testing.T) { err := runner.Cancel(context.Background()) require.NoError(t, err) } + +func TestJobRunnerRestart(t *testing.T) { + for _, jobSettings := range []*jobs.JobSettings{ + {}, + { + Continuous: &jobs.Continuous{ + PauseStatus: jobs.PauseStatusPaused, + }, + }, + } { + job := &resources.Job{ + ID: "123", + JobSettings: jobSettings, + } + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": job, + }, + }, + }, + } + + runner := jobRunner{key: "test", bundle: b, job: job} + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + ctx := context.Background() + ctx = cmdio.InContext(ctx, cmdio.NewIO(flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "")) + ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend)) + + jobApi := m.GetMockJobsAPI() + jobApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{ + ActiveOnly: true, + JobId: 123, + }).Return([]jobs.BaseRun{ + {RunId: 1}, + {RunId: 2}, + }, nil) + + // Mock the runner cancelling existing job runs. + mockWait := &jobs.WaitGetRunJobTerminatedOrSkipped[struct{}]{ + Poll: func(time time.Duration, f func(j *jobs.Run)) (*jobs.Run, error) { + return nil, nil + }, + } + jobApi.EXPECT().CancelRun(mock.Anything, jobs.CancelRun{ + RunId: 1, + }).Return(mockWait, nil) + jobApi.EXPECT().CancelRun(mock.Anything, jobs.CancelRun{ + RunId: 2, + }).Return(mockWait, nil) + + // Mock the runner triggering a job run + mockWaitForRun := &jobs.WaitGetRunJobTerminatedOrSkipped[jobs.RunNowResponse]{ + Poll: func(d time.Duration, f func(*jobs.Run)) (*jobs.Run, error) { + return &jobs.Run{ + State: &jobs.RunState{ + ResultState: jobs.RunResultStateSuccess, + }, + }, nil + }, + } + jobApi.EXPECT().RunNow(mock.Anything, jobs.RunNow{ + JobId: 123, + }).Return(mockWaitForRun, nil) + + // Mock the runner getting the job output + jobApi.EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{}).Return(&jobs.Run{}, nil) + + _, err := runner.Restart(ctx, &Options{}) + require.NoError(t, err) + } +} + +func TestJobRunnerRestartForContinuousUnpausedJobs(t *testing.T) { + job := &resources.Job{ + ID: "123", + JobSettings: &jobs.JobSettings{ + Continuous: &jobs.Continuous{ + PauseStatus: jobs.PauseStatusUnpaused, + }, + }, + } + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": job, + }, + }, + }, + } + + runner := jobRunner{key: "test", bundle: b, job: job} + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + ctx := context.Background() + ctx = cmdio.InContext(ctx, cmdio.NewIO(flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "...")) + ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend)) + + jobApi := m.GetMockJobsAPI() + + // The runner should not try and cancel existing job runs for unpaused continuous jobs. + jobApi.AssertNotCalled(t, "ListRunsAll") + jobApi.AssertNotCalled(t, "CancelRun") + + // Mock the runner triggering a job run + mockWaitForRun := &jobs.WaitGetRunJobTerminatedOrSkipped[jobs.RunNowResponse]{ + Poll: func(d time.Duration, f func(*jobs.Run)) (*jobs.Run, error) { + return &jobs.Run{ + State: &jobs.RunState{ + ResultState: jobs.RunResultStateSuccess, + }, + }, nil + }, + } + jobApi.EXPECT().RunNow(mock.Anything, jobs.RunNow{ + JobId: 123, + }).Return(mockWaitForRun, nil) + + // Mock the runner getting the job output + jobApi.EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{}).Return(&jobs.Run{}, nil) + + _, err := runner.Restart(ctx, &Options{}) + require.NoError(t, err) +} diff --git a/bundle/run/keys.go b/bundle/run/keys.go deleted file mode 100644 index 76ec50ac8..000000000 --- a/bundle/run/keys.go +++ /dev/null @@ -1,69 +0,0 @@ -package run - -import ( - "fmt" - - "github.com/databricks/cli/bundle" - "golang.org/x/exp/maps" -) - -// RunnerLookup maps identifiers to a list of workloads that match that identifier. -// The list can have more than 1 entry if resources of different types use the -// same key. When this happens, the user should disambiguate between them. -type RunnerLookup map[string][]Runner - -// ResourceKeys computes a map with -func ResourceKeys(b *bundle.Bundle) (keyOnly RunnerLookup, keyWithType RunnerLookup) { - keyOnly = make(RunnerLookup) - keyWithType = make(RunnerLookup) - - r := b.Config.Resources - for k, v := range r.Jobs { - kt := fmt.Sprintf("jobs.%s", k) - w := jobRunner{key: key(kt), bundle: b, job: v} - keyOnly[k] = append(keyOnly[k], &w) - keyWithType[kt] = append(keyWithType[kt], &w) - } - for k, v := range r.Pipelines { - kt := fmt.Sprintf("pipelines.%s", k) - w := pipelineRunner{key: key(kt), bundle: b, pipeline: v} - keyOnly[k] = append(keyOnly[k], &w) - keyWithType[kt] = append(keyWithType[kt], &w) - } - return -} - -// ResourceCompletionMap returns a map of resource keys to their respective names. -func ResourceCompletionMap(b *bundle.Bundle) map[string]string { - out := make(map[string]string) - keyOnly, keyWithType := ResourceKeys(b) - - // Keep track of resources we have seen by their fully qualified key. - seen := make(map[string]bool) - - // First add resources that can be identified by key alone. - for k, v := range keyOnly { - // Invariant: len(v) >= 1. See [ResourceKeys]. - if len(v) == 1 { - seen[v[0].Key()] = true - out[k] = v[0].Name() - } - } - - // Then add resources that can only be identified by their type and key. - for k, v := range keyWithType { - // Invariant: len(v) == 1. See [ResourceKeys]. - _, ok := seen[v[0].Key()] - if ok { - continue - } - out[k] = v[0].Name() - } - - return out -} - -// ResourceCompletions returns a list of keys that unambiguously reference resources in the bundle. -func ResourceCompletions(b *bundle.Bundle) []string { - return maps.Keys(ResourceCompletionMap(b)) -} diff --git a/bundle/run/keys_test.go b/bundle/run/keys_test.go deleted file mode 100644 index 5ab73b13d..000000000 --- a/bundle/run/keys_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package run - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" -) - -func TestResourceCompletionsUnique(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "foo": {}, - "bar": {}, - }, - }, - }, - } - - assert.ElementsMatch(t, []string{"foo", "bar"}, ResourceCompletions(b)) -} diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index d684f8388..ffe012843 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -183,6 +183,18 @@ func (r *pipelineRunner) Cancel(ctx context.Context) error { return err } +func (r *pipelineRunner) Restart(ctx context.Context, opts *Options) (output.RunOutput, error) { + s := cmdio.Spinner(ctx) + s <- "Cancelling the active pipeline update" + err := r.Cancel(ctx) + close(s) + if err != nil { + return nil, err + } + + return r.Run(ctx, opts) +} + func (r *pipelineRunner) ParseArgs(args []string, opts *Options) error { if len(args) == 0 { return nil diff --git a/bundle/run/pipeline_test.go b/bundle/run/pipeline_test.go index 29b57ffdb..e4608061c 100644 --- a/bundle/run/pipeline_test.go +++ b/bundle/run/pipeline_test.go @@ -1,6 +1,7 @@ package run import ( + "bytes" "context" "testing" "time" @@ -8,8 +9,12 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + sdk_config "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -47,3 +52,68 @@ func TestPipelineRunnerCancel(t *testing.T) { err := runner.Cancel(context.Background()) require.NoError(t, err) } + +func TestPipelineRunnerRestart(t *testing.T) { + pipeline := &resources.Pipeline{ + ID: "123", + } + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "test_pipeline": pipeline, + }, + }, + }, + } + + runner := pipelineRunner{key: "test", bundle: b, pipeline: pipeline} + + m := mocks.NewMockWorkspaceClient(t) + m.WorkspaceClient.Config = &sdk_config.Config{ + Host: "https://test.com", + } + b.SetWorkpaceClient(m.WorkspaceClient) + ctx := context.Background() + ctx = cmdio.InContext(ctx, cmdio.NewIO(flags.OutputText, &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}, "", "...")) + ctx = cmdio.NewContext(ctx, cmdio.NewLogger(flags.ModeAppend)) + + mockWait := &pipelines.WaitGetPipelineIdle[struct{}]{ + Poll: func(time.Duration, func(*pipelines.GetPipelineResponse)) (*pipelines.GetPipelineResponse, error) { + return nil, nil + }, + } + + pipelineApi := m.GetMockPipelinesAPI() + pipelineApi.EXPECT().Stop(mock.Anything, pipelines.StopRequest{ + PipelineId: "123", + }).Return(mockWait, nil) + + pipelineApi.EXPECT().GetByPipelineId(mock.Anything, "123").Return(&pipelines.GetPipelineResponse{}, nil) + + // Mock runner starting a new update + pipelineApi.EXPECT().StartUpdate(mock.Anything, pipelines.StartUpdate{ + PipelineId: "123", + }).Return(&pipelines.StartUpdateResponse{ + UpdateId: "456", + }, nil) + + // Mock runner polling for events + pipelineApi.EXPECT().ListPipelineEventsAll(mock.Anything, pipelines.ListPipelineEventsRequest{ + Filter: `update_id = '456'`, + MaxResults: 100, + PipelineId: "123", + }).Return([]pipelines.PipelineEvent{}, nil) + + // Mock runner polling for update status + pipelineApi.EXPECT().GetUpdateByPipelineIdAndUpdateId(mock.Anything, "123", "456"). + Return(&pipelines.GetUpdateResponse{ + Update: &pipelines.UpdateInfo{ + State: pipelines.UpdateInfoStateCompleted, + }, + }, nil) + + _, err := runner.Restart(ctx, &Options{}) + require.NoError(t, err) +} diff --git a/bundle/run/runner.go b/bundle/run/runner.go index 0f202ce7d..4c907d068 100644 --- a/bundle/run/runner.go +++ b/bundle/run/runner.go @@ -3,9 +3,10 @@ package run import ( "context" "fmt" - "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + refs "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run/output" ) @@ -27,6 +28,10 @@ type Runner interface { // Run the underlying worklow. Run(ctx context.Context, opts *Options) (output.RunOutput, error) + // Restart the underlying workflow by cancelling any existing runs before + // starting a new one. + Restart(ctx context.Context, opts *Options) (output.RunOutput, error) + // Cancel the underlying workflow. Cancel(ctx context.Context) error @@ -34,34 +39,24 @@ type Runner interface { argsHandler } -// Find locates a runner matching the specified argument. -// -// Its behavior is as follows: -// 1. Try to find a resource with identical to the argument. -// 2. Try to find a resource with . identical to the argument. -// -// If an argument resolves to multiple resources, it returns an error. -func Find(b *bundle.Bundle, arg string) (Runner, error) { - keyOnly, keyWithType := ResourceKeys(b) - if len(keyWithType) == 0 { - return nil, fmt.Errorf("bundle defines no resources") +// IsRunnable returns a filter that only allows runnable resources. +func IsRunnable(ref refs.Reference) bool { + switch ref.Resource.(type) { + case *resources.Job, *resources.Pipeline: + return true + default: + return false + } +} + +// ToRunner converts a resource reference to a runnable resource. +func ToRunner(b *bundle.Bundle, ref refs.Reference) (Runner, error) { + switch resource := ref.Resource.(type) { + case *resources.Job: + return &jobRunner{key: key(ref.KeyWithType), bundle: b, job: resource}, nil + case *resources.Pipeline: + return &pipelineRunner{key: key(ref.KeyWithType), bundle: b, pipeline: resource}, nil + default: + return nil, fmt.Errorf("unsupported resource type: %T", resource) } - - runners, ok := keyOnly[arg] - if !ok { - runners, ok = keyWithType[arg] - if !ok { - return nil, fmt.Errorf("no such resource: %s", arg) - } - } - - if len(runners) != 1 { - var keys []string - for _, runner := range runners { - keys = append(keys, runner.Key()) - } - return nil, fmt.Errorf("ambiguous: %s (can resolve to all of %s)", arg, strings.Join(keys, ", ")) - } - - return runners[0], nil } diff --git a/bundle/run/runner_test.go b/bundle/run/runner_test.go index 85baa192f..2fc5fa6fa 100644 --- a/bundle/run/runner_test.go +++ b/bundle/run/runner_test.go @@ -3,82 +3,14 @@ package run import ( "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + refs "github.com/databricks/cli/bundle/resources" "github.com/stretchr/testify/assert" ) -func TestFindNoResources(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{}, - }, - } - - _, err := Find(b, "foo") - assert.ErrorContains(t, err, "bundle defines no resources") -} - -func TestFindSingleArg(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "foo": {}, - }, - }, - }, - } - - _, err := Find(b, "foo") - assert.NoError(t, err) -} - -func TestFindSingleArgNotFound(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "foo": {}, - }, - }, - }, - } - - _, err := Find(b, "bar") - assert.ErrorContains(t, err, "no such resource: bar") -} - -func TestFindSingleArgAmbiguous(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "key": {}, - }, - Pipelines: map[string]*resources.Pipeline{ - "key": {}, - }, - }, - }, - } - - _, err := Find(b, "key") - assert.ErrorContains(t, err, "ambiguous: ") -} - -func TestFindSingleArgWithType(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "key": {}, - }, - }, - }, - } - - _, err := Find(b, "jobs.key") - assert.NoError(t, err) +func TestRunner_IsRunnable(t *testing.T) { + assert.True(t, IsRunnable(refs.Reference{Resource: &resources.Job{}})) + assert.True(t, IsRunnable(refs.Reference{Resource: &resources.Pipeline{}})) + assert.False(t, IsRunnable(refs.Reference{Resource: &resources.MlflowModel{}})) + assert.False(t, IsRunnable(refs.Reference{Resource: &resources.MlflowExperiment{}})) } diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go index ee0b5a615..dcb381b83 100644 --- a/bundle/schema/embed_test.go +++ b/bundle/schema/embed_test.go @@ -39,7 +39,7 @@ func TestJsonSchema(t *testing.T) { // Assert job fields have their descriptions loaded. resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job") - fields := []string{"name", "continuous", "deployment", "tasks", "trigger"} + fields := []string{"name", "continuous", "tasks", "trigger"} for _, field := range fields { assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description) } @@ -53,7 +53,7 @@ func TestJsonSchema(t *testing.T) { // Assert descriptions are loaded for pipelines pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline") - fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"} + fields = []string{"name", "catalog", "clusters", "channel", "continuous", "development"} for _, field := range fields { assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index afdf9fb9e..62e5fe6d8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -180,6 +180,48 @@ } ] }, + "resources.Dashboard": { + "anyOf": [ + { + "type": "object", + "properties": { + "display_name": { + "description": "The display name of the dashboard.", + "$ref": "#/$defs/string" + }, + "embed_credentials": { + "$ref": "#/$defs/bool" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "parent_path": { + "description": "The workspace path of the folder containing the dashboard. Includes leading slash and no\ntrailing slash.\nThis field is excluded in List Dashboards responses.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_dashboard": { + "description": "The contents of the dashboard in serialized string form.\nThis field is excluded in List Dashboards responses.\nUse the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get)\nto retrieve an example response, which includes the `serialized_dashboard` field.\nThis field provides the structure of the JSON string that represents the dashboard's\nlayout and components.", + "$ref": "#/$defs/interface" + }, + "warehouse_id": { + "description": "The warehouse ID used to run the dashboard.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "display_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Grant": { "anyOf": [ { @@ -209,22 +251,18 @@ { "type": "object", "properties": { + "budget_policy_id": { + "description": "The id of the user specified budget policy to use for this job.\nIf not specified, a default budget policy may be applied when creating or modifying the job.\nSee `effective_budget_policy_id` for the budget policy used by this workload.", + "$ref": "#/$defs/string" + }, "continuous": { "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment" - }, "description": { "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", "$ref": "#/$defs/string" }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode" - }, "email_notifications": { "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" @@ -233,10 +271,6 @@ "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format" - }, "git_source": { "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" @@ -402,6 +436,10 @@ { "type": "object", "properties": { + "ai_gateway": { + "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig" + }, "config": { "description": "The core config of the serving endpoint.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" @@ -472,6 +510,10 @@ { "type": "object", "properties": { + "budget_policy_id": { + "description": "Budget policy of this pipeline.", + "$ref": "#/$defs/string" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string" @@ -539,6 +581,10 @@ "description": "Whether Photon is enabled for this pipeline.", "$ref": "#/$defs/bool" }, + "schema": { + "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", + "$ref": "#/$defs/string" + }, "serverless": { "description": "Whether serverless compute is enabled for this pipeline.", "$ref": "#/$defs/bool" @@ -1050,6 +1096,9 @@ "clusters": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster" }, + "dashboards": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard" + }, "experiments": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" }, @@ -1206,6 +1255,9 @@ "profile": { "$ref": "#/$defs/string" }, + "resource_path": { + "$ref": "#/$defs/string" + }, "root_path": { "$ref": "#/$defs/string" }, @@ -2532,9 +2584,6 @@ "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" }, - "git_snapshot": { - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot" - }, "git_tag": { "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", "$ref": "#/$defs/string" @@ -2542,10 +2591,6 @@ "git_url": { "description": "URL of the repository to be cloned by this job.", "$ref": "#/$defs/string" - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource" } }, "additionalProperties": false, @@ -2632,7 +2677,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -3073,6 +3118,7 @@ "$ref": "#/$defs/map/string" }, "pipeline_params": { + "description": "Controls whether the pipeline should perform a full refresh", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" }, "python_named_params": { @@ -3547,7 +3593,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -3904,6 +3950,10 @@ { "type": "object", "properties": { + "report": { + "description": "Select tables from a specific source report.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ReportSpec" + }, "schema": { "description": "Select tables from a specific source schema.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec" @@ -4236,6 +4286,40 @@ } ] }, + "pipelines.ReportSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store table.", + "$ref": "#/$defs/string" + }, + "destination_table": { + "description": "Required. Destination table name. The pipeline fails if a table with that name already exists.", + "$ref": "#/$defs/string" + }, + "source_url": { + "description": "Required. Report URL in the source system.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.SchemaSpec": { "anyOf": [ { @@ -4284,7 +4368,7 @@ "$ref": "#/$defs/string" }, "destination_table": { - "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used.", + "description": "Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used.", "$ref": "#/$defs/string" }, "source_catalog": { @@ -4332,6 +4416,10 @@ "SCD_TYPE_1", "SCD_TYPE_2" ] + }, + "sequence_by": { + "description": "The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.", + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -4365,6 +4453,207 @@ } ] }, + "serving.AiGatewayConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "guardrails": { + "description": "Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails" + }, + "inference_table_config": { + "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig" + }, + "rate_limits": { + "description": "Configuration for rate limits which can be set to limit endpoint traffic.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + }, + "usage_tracking_config": { + "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailParameters": { + "anyOf": [ + { + "type": "object", + "properties": { + "invalid_keywords": { + "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", + "$ref": "#/$defs/slice/string" + }, + "pii": { + "description": "Configuration for guardrail PII filter.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior" + }, + "safety": { + "description": "Indicates whether the safety filter is enabled.", + "$ref": "#/$defs/bool" + }, + "valid_topics": { + "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehavior": { + "anyOf": [ + { + "type": "object", + "properties": { + "behavior": { + "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior", + "enum": [ + "NONE", + "BLOCK" + ] + } + }, + "additionalProperties": false, + "required": [ + "behavior" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehaviorBehavior": { + "type": "string" + }, + "serving.AiGatewayGuardrails": { + "anyOf": [ + { + "type": "object", + "properties": { + "input": { + "description": "Configuration for input guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + }, + "output": { + "description": "Configuration for output guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayInferenceTableConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int" + }, + "key": { + "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey", + "enum": [ + "user", + "endpoint" + ] + }, + "renewal_period": { + "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod", + "enum": [ + "minute" + ] + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimitKey": { + "type": "string" + }, + "serving.AiGatewayRateLimitRenewalPeriod": { + "type": "string" + }, + "serving.AiGatewayUsageTrackingConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to enable usage tracking.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.AmazonBedrockConfig": { "anyOf": [ { @@ -5048,6 +5337,20 @@ } ] }, + "resources.Dashboard": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Dashboard" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "anyOf": [ { @@ -5569,6 +5872,20 @@ } ] }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.EndpointTag": { "anyOf": [ { diff --git a/bundle/tests/issue_1828/databricks.yml b/bundle/tests/issue_1828/databricks.yml new file mode 100644 index 000000000..d5f60ce73 --- /dev/null +++ b/bundle/tests/issue_1828/databricks.yml @@ -0,0 +1,33 @@ +bundle: + name: issue_1828 + +variables: + # One entry for each of the underlying YAML (or [dyn.Kind]) types. + # The test confirms we can convert to and from the typed configuration without losing information. + + map: + default: + foo: bar + + sequence: + default: + - foo + - bar + + string: + default: foo + + bool: + default: true + + int: + default: 42 + + float: + default: 3.14 + + time: + default: 2021-01-01 + + nil: + default: diff --git a/bundle/tests/issue_1828_test.go b/bundle/tests/issue_1828_test.go new file mode 100644 index 000000000..5f2becce5 --- /dev/null +++ b/bundle/tests/issue_1828_test.go @@ -0,0 +1,48 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIssue1828(t *testing.T) { + b := load(t, "./issue_1828") + + if assert.Contains(t, b.Config.Variables, "map") { + assert.Equal(t, map[string]any{ + "foo": "bar", + }, b.Config.Variables["map"].Default) + } + + if assert.Contains(t, b.Config.Variables, "sequence") { + assert.Equal(t, []any{ + "foo", + "bar", + }, b.Config.Variables["sequence"].Default) + } + + if assert.Contains(t, b.Config.Variables, "string") { + assert.Equal(t, "foo", b.Config.Variables["string"].Default) + } + + if assert.Contains(t, b.Config.Variables, "bool") { + assert.Equal(t, true, b.Config.Variables["bool"].Default) + } + + if assert.Contains(t, b.Config.Variables, "int") { + assert.Equal(t, 42, b.Config.Variables["int"].Default) + } + + if assert.Contains(t, b.Config.Variables, "float") { + assert.Equal(t, 3.14, b.Config.Variables["float"].Default) + } + + if assert.Contains(t, b.Config.Variables, "time") { + assert.Equal(t, "2021-01-01", b.Config.Variables["time"].Default) + } + + if assert.Contains(t, b.Config.Variables, "nil") { + assert.Equal(t, nil, b.Config.Variables["nil"].Default) + } +} diff --git a/bundle/tests/run_as_test.go b/bundle/tests/run_as_test.go index 6c07cc537..920577146 100644 --- a/bundle/tests/run_as_test.go +++ b/bundle/tests/run_as_test.go @@ -3,7 +3,6 @@ package config_tests import ( "context" "fmt" - "path/filepath" "testing" "github.com/databricks/cli/bundle" @@ -113,8 +112,9 @@ func TestRunAsErrorForPipelines(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/pipelines/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("pipelines are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath)) + assert.ErrorContains(t, err, "pipelines do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+ + "See https://docs") } func TestRunAsNoErrorForPipelines(t *testing.T) { @@ -152,8 +152,9 @@ func TestRunAsErrorForModelServing(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/model_serving/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("model_serving_endpoints are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Please refer to the documentation at https://docs.databricks.com/dev-tools/bundles/run-as.html for more details. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal", configPath)) + assert.ErrorContains(t, err, "model_serving_endpoints do not support a setting a run_as user that is different from the owner.\n"+ + "Current identity: jane@doe.com. Run as identity: my_service_principal.\n"+ + "See https://docs") } func TestRunAsNoErrorForModelServingEndpoints(t *testing.T) { @@ -191,8 +192,7 @@ func TestRunAsErrorWhenBothUserAndSpSpecified(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/both_sp_and_user/databricks.yml") - assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. A service_principal_name \"my_service_principal\" is specified at %s:6:27. A user_name \"my_user_name\" is defined at %s:7:14", configPath, configPath)) + assert.ErrorContains(t, err, "run_as section cannot specify both user_name and service_principal_name") } func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) { @@ -202,19 +202,19 @@ func TestRunAsErrorNeitherUserOrSpSpecified(t *testing.T) { }{ { name: "empty_run_as", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:8", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_run_as/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_sp", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_sp/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_user", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, { name: "empty_user_and_sp", - err: fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:5:3", filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/empty_user_and_sp/databricks.yml")), + err: "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified", }, } @@ -257,8 +257,7 @@ func TestRunAsErrorNeitherUserOrSpSpecifiedAtTargetOverride(t *testing.T) { diags := bundle.Apply(ctx, b, mutator.SetRunAs()) err := diags.Error() - configPath := filepath.FromSlash("run_as/not_allowed/neither_sp_nor_user/override/override.yml") - assert.EqualError(t, err, fmt.Sprintf("run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:12", configPath)) + assert.EqualError(t, err, "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified") } func TestLegacyRunAs(t *testing.T) { diff --git a/cmd/account/access-control/access-control.go b/cmd/account/access-control/access-control.go index f6761a1b4..07a7fce9b 100755 --- a/cmd/account/access-control/access-control.go +++ b/cmd/account/access-control/access-control.go @@ -205,9 +205,15 @@ func newUpdateRuleSet() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateRuleSetJson.Unmarshal(&updateRuleSetReq) - if err != nil { - return err + diags := updateRuleSetJson.Unmarshal(&updateRuleSetReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/budgets/budgets.go b/cmd/account/budgets/budgets.go index 6b47bb32c..87ed41d57 100755 --- a/cmd/account/budgets/budgets.go +++ b/cmd/account/budgets/budgets.go @@ -78,9 +78,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -316,9 +322,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/credentials/credentials.go b/cmd/account/credentials/credentials.go index ed071cda3..4ab70ef00 100755 --- a/cmd/account/credentials/credentials.go +++ b/cmd/account/credentials/credentials.go @@ -90,9 +90,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/csp-enablement-account/csp-enablement-account.go b/cmd/account/csp-enablement-account/csp-enablement-account.go index d6fce9537..9fb94a616 100755 --- a/cmd/account/csp-enablement-account/csp-enablement-account.go +++ b/cmd/account/csp-enablement-account/csp-enablement-account.go @@ -129,9 +129,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/custom-app-integration/custom-app-integration.go b/cmd/account/custom-app-integration/custom-app-integration.go index 5cdf422d7..9d16a44d4 100755 --- a/cmd/account/custom-app-integration/custom-app-integration.go +++ b/cmd/account/custom-app-integration/custom-app-integration.go @@ -88,9 +88,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -320,9 +326,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.IntegrationId = args[0] diff --git a/cmd/account/disable-legacy-features/disable-legacy-features.go b/cmd/account/disable-legacy-features/disable-legacy-features.go index 6d25b943d..5e732b87b 100755 --- a/cmd/account/disable-legacy-features/disable-legacy-features.go +++ b/cmd/account/disable-legacy-features/disable-legacy-features.go @@ -185,9 +185,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/encryption-keys/encryption-keys.go b/cmd/account/encryption-keys/encryption-keys.go index 44545ccfa..3f7082cd7 100755 --- a/cmd/account/encryption-keys/encryption-keys.go +++ b/cmd/account/encryption-keys/encryption-keys.go @@ -107,9 +107,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go index 71149e5ad..1f820e3a9 100755 --- a/cmd/account/esm-enablement-account/esm-enablement-account.go +++ b/cmd/account/esm-enablement-account/esm-enablement-account.go @@ -127,9 +127,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/groups/groups.go b/cmd/account/groups/groups.go index a7e1ac430..7c2e83319 100755 --- a/cmd/account/groups/groups.go +++ b/cmd/account/groups/groups.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -446,9 +458,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/ip-access-lists/ip-access-lists.go b/cmd/account/ip-access-lists/ip-access-lists.go index 5c6d27dd7..61fa944ab 100755 --- a/cmd/account/ip-access-lists/ip-access-lists.go +++ b/cmd/account/ip-access-lists/ip-access-lists.go @@ -132,9 +132,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -411,9 +417,15 @@ func newReplace() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.IpAccessListId = args[0] @@ -505,9 +517,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/log-delivery/log-delivery.go b/cmd/account/log-delivery/log-delivery.go index 4584f4d2b..cb083f3ba 100755 --- a/cmd/account/log-delivery/log-delivery.go +++ b/cmd/account/log-delivery/log-delivery.go @@ -162,9 +162,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -369,9 +375,15 @@ func newPatchStatus() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchStatusJson.Unmarshal(&patchStatusReq) - if err != nil { - return err + diags := patchStatusJson.Unmarshal(&patchStatusReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } patchStatusReq.LogDeliveryConfigurationId = args[0] diff --git a/cmd/account/metastore-assignments/metastore-assignments.go b/cmd/account/metastore-assignments/metastore-assignments.go index d7f32ccb9..9d2490754 100755 --- a/cmd/account/metastore-assignments/metastore-assignments.go +++ b/cmd/account/metastore-assignments/metastore-assignments.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &createReq.WorkspaceId) @@ -343,9 +349,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) diff --git a/cmd/account/metastores/metastores.go b/cmd/account/metastores/metastores.go index 7c8e3f2c1..eb5755782 100755 --- a/cmd/account/metastores/metastores.go +++ b/cmd/account/metastores/metastores.go @@ -80,9 +80,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -304,9 +310,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.MetastoreId = args[0] diff --git a/cmd/account/network-connectivity/network-connectivity.go b/cmd/account/network-connectivity/network-connectivity.go index cd8da2905..168de9c1d 100755 --- a/cmd/account/network-connectivity/network-connectivity.go +++ b/cmd/account/network-connectivity/network-connectivity.go @@ -96,9 +96,15 @@ func newCreateNetworkConnectivityConfiguration() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createNetworkConnectivityConfigurationJson.Unmarshal(&createNetworkConnectivityConfigurationReq) - if err != nil { - return err + diags := createNetworkConnectivityConfigurationJson.Unmarshal(&createNetworkConnectivityConfigurationReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -187,9 +193,15 @@ func newCreatePrivateEndpointRule() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createPrivateEndpointRuleJson.Unmarshal(&createPrivateEndpointRuleReq) - if err != nil { - return err + diags := createPrivateEndpointRuleJson.Unmarshal(&createPrivateEndpointRuleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createPrivateEndpointRuleReq.NetworkConnectivityConfigId = args[0] diff --git a/cmd/account/networks/networks.go b/cmd/account/networks/networks.go index 05ef0c815..086d24be8 100755 --- a/cmd/account/networks/networks.go +++ b/cmd/account/networks/networks.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/account/personal-compute/personal-compute.go b/cmd/account/personal-compute/personal-compute.go index 2a14b0b33..dac7e2e30 100755 --- a/cmd/account/personal-compute/personal-compute.go +++ b/cmd/account/personal-compute/personal-compute.go @@ -189,9 +189,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/private-access/private-access.go b/cmd/account/private-access/private-access.go index d527fa64e..312f6d023 100755 --- a/cmd/account/private-access/private-access.go +++ b/cmd/account/private-access/private-access.go @@ -109,9 +109,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -411,9 +417,15 @@ func newReplace() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.PrivateAccessSettingsId = args[0] diff --git a/cmd/account/published-app-integration/published-app-integration.go b/cmd/account/published-app-integration/published-app-integration.go index 5143d53cc..c97bcfc61 100755 --- a/cmd/account/published-app-integration/published-app-integration.go +++ b/cmd/account/published-app-integration/published-app-integration.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -315,9 +321,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.IntegrationId = args[0] diff --git a/cmd/account/service-principals/service-principals.go b/cmd/account/service-principals/service-principals.go index c86810f1d..f6ec1fa54 100755 --- a/cmd/account/service-principals/service-principals.go +++ b/cmd/account/service-principals/service-principals.go @@ -95,9 +95,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -448,9 +460,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/storage-credentials/storage-credentials.go b/cmd/account/storage-credentials/storage-credentials.go index 4280ae8c3..b3b8ad367 100755 --- a/cmd/account/storage-credentials/storage-credentials.go +++ b/cmd/account/storage-credentials/storage-credentials.go @@ -88,9 +88,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.MetastoreId = args[0] @@ -340,9 +346,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.MetastoreId = args[0] diff --git a/cmd/account/storage/storage.go b/cmd/account/storage/storage.go index 50460ed0a..24bb6d80b 100755 --- a/cmd/account/storage/storage.go +++ b/cmd/account/storage/storage.go @@ -87,9 +87,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/account/usage-dashboards/usage-dashboards.go b/cmd/account/usage-dashboards/usage-dashboards.go index 8a1c32476..a482b466d 100755 --- a/cmd/account/usage-dashboards/usage-dashboards.go +++ b/cmd/account/usage-dashboards/usage-dashboards.go @@ -80,9 +80,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/account/users/users.go b/cmd/account/users/users.go index 289d2972f..10990084f 100755 --- a/cmd/account/users/users.go +++ b/cmd/account/users/users.go @@ -103,9 +103,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -374,9 +380,15 @@ func newPatch() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -465,9 +477,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/account/vpc-endpoints/vpc-endpoints.go b/cmd/account/vpc-endpoints/vpc-endpoints.go index e6c6c126a..c1aab0d24 100755 --- a/cmd/account/vpc-endpoints/vpc-endpoints.go +++ b/cmd/account/vpc-endpoints/vpc-endpoints.go @@ -104,9 +104,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/account/workspace-assignment/workspace-assignment.go b/cmd/account/workspace-assignment/workspace-assignment.go index 58468d09f..e09095d37 100755 --- a/cmd/account/workspace-assignment/workspace-assignment.go +++ b/cmd/account/workspace-assignment/workspace-assignment.go @@ -273,9 +273,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.WorkspaceId) diff --git a/cmd/account/workspaces/workspaces.go b/cmd/account/workspaces/workspaces.go index 1ec6230b2..82d3d7db9 100755 --- a/cmd/account/workspaces/workspaces.go +++ b/cmd/account/workspaces/workspaces.go @@ -133,9 +133,15 @@ func newCreate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -551,9 +557,15 @@ func newUpdate() *cobra.Command { a := root.AccountClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/api/api.go b/cmd/api/api.go index 03460f717..d33939a52 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -42,9 +42,9 @@ func makeCommand(method string) *cobra.Command { var path = args[0] var request any - err := payload.Unmarshal(&request) - if err != nil { - return err + diags := payload.Unmarshal(&request) + if diags.HasError() { + return diags.Error() } cfg := &config.Config{} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 0880c9c44..fb88cd7d0 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -27,5 +27,6 @@ func New() *cobra.Command { cmd.AddCommand(newGenerateCommand()) cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) + cmd.AddCommand(newOpenCommand()) return cmd } diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index f1c85cb3d..a25e02f6c 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -78,7 +78,7 @@ func newDeployCommand() *cobra.Command { } renderOpts := render.RenderOptions{RenderSummaryTable: false} - err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) + err := render.RenderDiagnostics(cmd.OutOrStdout(), b, diags, renderOpts) if err != nil { return fmt.Errorf("failed to render output: %w", err) } diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 1e3d56e43..7dea19ff9 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -16,6 +16,7 @@ func newGenerateCommand() *cobra.Command { cmd.AddCommand(generate.NewGenerateJobCommand()) cmd.AddCommand(generate.NewGeneratePipelineCommand()) + cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go new file mode 100644 index 000000000..4a538a293 --- /dev/null +++ b/cmd/bundle/generate/dashboard.go @@ -0,0 +1,467 @@ +package generate + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/generate" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/render" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + "gopkg.in/yaml.v3" +) + +type dashboard struct { + // Lookup flags for one-time generate. + existingPath string + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and dashboard representation. + resourceDir string + dashboardDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the dashboard. + watch bool + + // Relative path from the resource directory to the dashboard directory. + relativeDashboardDir string +} + +func (d *dashboard) resolveID(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) { + switch { + case d.existingPath != "": + return d.resolveFromPath(ctx, b) + case d.existingID != "": + return d.resolveFromID(ctx, b) + } + + return "", diag.Errorf("expected one of --dashboard-path, --dashboard-id") +} + +func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) { + w := b.WorkspaceClient() + obj, err := w.Workspace.GetStatusByPath(ctx, d.existingPath) + if err != nil { + if apierr.IsMissing(err) { + return "", diag.Errorf("dashboard %q not found", path.Base(d.existingPath)) + } + + // Emit a more descriptive error message for legacy dashboards. + if errors.Is(err, apierr.ErrBadRequest) && strings.HasPrefix(err.Error(), "dbsqlDashboard ") { + return "", diag.Diagnostics{ + { + Severity: diag.Error, + Summary: fmt.Sprintf("dashboard %q is a legacy dashboard", path.Base(d.existingPath)), + Detail: "" + + "Databricks Asset Bundles work exclusively with AI/BI dashboards.\n" + + "\n" + + "Instructions on how to convert a legacy dashboard to an AI/BI dashboard\n" + + "can be found at: https://docs.databricks.com/en/dashboards/clone-legacy-to-aibi.html.", + }, + } + } + + return "", diag.FromErr(err) + } + + if obj.ObjectType != workspace.ObjectTypeDashboard { + found := strings.ToLower(obj.ObjectType.String()) + return "", diag.Diagnostics{ + { + Severity: diag.Error, + Summary: fmt.Sprintf("expected a dashboard, found a %s", found), + }, + } + } + + if obj.ResourceId == "" { + return "", diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "expected a non-empty dashboard resource ID", + }, + } + } + + return obj.ResourceId, nil +} + +func (d *dashboard) resolveFromID(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) { + w := b.WorkspaceClient() + obj, err := w.Lakeview.GetByDashboardId(ctx, d.existingID) + if err != nil { + if apierr.IsMissing(err) { + return "", diag.Errorf("dashboard with ID %s not found", d.existingID) + } + return "", diag.FromErr(err) + } + + return obj.DashboardId, nil +} + +func remarshalJSON(data []byte) ([]byte, error) { + var tmp any + var err error + err = json.Unmarshal(data, &tmp) + if err != nil { + return nil, err + } + + // Remarshal the data to ensure its formatting is stable. + // The result will have alphabetically sorted keys and be indented. + // HTML escaping is disabled to retain characters such as &, <, and >. + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) + err = enc.Encode(tmp) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (d *dashboard) saveSerializedDashboard(_ context.Context, b *bundle.Bundle, dashboard *dashboards.Dashboard, filename string) error { + // Unmarshal and remarshal the serialized dashboard to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(dashboard.SerializedDashboard)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", rel) + } + if !d.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", rel) + } + } + + fmt.Printf("Writing dashboard to %q\n", rel) + return os.WriteFile(filename, data, 0644) +} + +func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, dashboard *dashboards.Dashboard, key string) error { + // Save serialized dashboard definition to the dashboard directory. + dashboardBasename := fmt.Sprintf("%s.lvdash.json", key) + dashboardPath := filepath.Join(d.dashboardDir, dashboardBasename) + err := d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertDashboardToValue(dashboard, path.Join(d.relativeDashboardDir, dashboardBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "dashboards": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(d.resourceDir, 0755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(d.resourceDir, fmt.Sprintf("%s.dashboard.yml", key)) + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "display_name": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + fmt.Printf("Writing configuration to %q\n", rel) + err = saver.SaveAsYAML(result, resourcePath, d.force) + if err != nil { + return err + } + + return nil +} + +func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboard *dashboards.Dashboard) diag.Diagnostics { + // Compute [time.Time] for the most recent update. + tref, err := time.Parse(time.RFC3339, dashboard.UpdateTime) + if err != nil { + return diag.FromErr(err) + } + + for { + obj, err := w.Workspace.GetStatusByPath(ctx, dashboard.Path) + if err != nil { + return diag.FromErr(err) + } + + // Compute [time.Time] from timestamp in millis since epoch. + tcur := time.Unix(0, obj.ModifiedAt*int64(time.Millisecond)) + if tcur.After(tref) { + break + } + + time.Sleep(1 * time.Second) + } + + return nil +} + +func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + resource, ok := b.Config.Resources.Dashboards[d.resource] + if !ok { + return diag.Errorf("dashboard resource %q is not defined", d.resource) + } + + if resource.FilePath == "" { + return diag.Errorf("dashboard resource %q has no file path defined", d.resource) + } + + // Resolve the dashboard ID from the resource. + dashboardID := resource.ID + + // Overwrite the dashboard at the path referenced from the resource. + dashboardPath := resource.FilePath + + w := b.WorkspaceClient() + + // Start polling the underlying dashboard for changes. + var etag string + for { + dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID) + if err != nil { + return diag.FromErr(err) + } + + if etag != dashboard.Etag { + err = d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath) + if err != nil { + return diag.FromErr(err) + } + } + + // Abort if we are not watching for changes. + if !d.watch { + return nil + } + + // Update the etag for the next iteration. + etag = dashboard.Etag + + // Now poll the workspace API for changes. + // This is much more efficient than polling the dashboard API because it + // includes the entire serialized dashboard whereas we're only interested + // in the last modified time of the dashboard here. + waitForChanges(ctx, w, dashboard) + } +} + +func (d *dashboard) generateForExisting(ctx context.Context, b *bundle.Bundle, dashboardID string) diag.Diagnostics { + w := b.WorkspaceClient() + dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID) + if err != nil { + return diag.FromErr(err) + } + + key := textutil.NormalizeString(dashboard.DisplayName) + err = d.saveConfiguration(ctx, b, dashboard, key) + if err != nil { + return diag.FromErr(err) + } + + return nil +} + +func (d *dashboard) initialize(b *bundle.Bundle) diag.Diagnostics { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(d.resourceDir) { + d.resourceDir = filepath.Join(b.BundleRootPath, d.resourceDir) + } + if !filepath.IsAbs(d.dashboardDir) { + d.dashboardDir = filepath.Join(b.BundleRootPath, d.dashboardDir) + } + + // Make sure we know how the dashboard path is relative to the resource path. + rel, err := filepath.Rel(d.resourceDir, d.dashboardDir) + if err != nil { + return diag.FromErr(err) + } + + d.relativeDashboardDir = filepath.ToSlash(rel) + return nil +} + +func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := bundle.Apply(ctx, b, bundle.Seq( + phases.Initialize(), + terraform.Interpolate(), + terraform.Write(), + terraform.StatePull(), + terraform.Load(), + )) + if diags.HasError() { + return diags + } + + return d.updateDashboardForResource(ctx, b) +} + +func (d *dashboard) runForExisting(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Resolve the ID of the dashboard to generate configuration for. + dashboardID, diags := d.resolveID(ctx, b) + if diags.HasError() { + return diags + } + + return d.generateForExisting(ctx, b, dashboardID) +} + +func (d *dashboard) RunE(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + b, diags := root.MustConfigureBundle(cmd) + if diags.HasError() { + return diags.Error() + } + + diags = d.initialize(b) + if diags.HasError() { + return diags.Error() + } + + if d.resource != "" { + diags = d.runForResource(ctx, b) + } else { + diags = d.runForExisting(ctx, b) + } + + renderOpts := render.RenderOptions{RenderSummaryTable: false} + err := render.RenderDiagnostics(cmd.OutOrStdout(), b, diags, renderOpts) + if err != nil { + return fmt.Errorf("failed to render output: %w", err) + } + + if diags.HasError() { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterDashboards returns a filter that only includes dashboards. +func filterDashboards(ref resources.Reference) bool { + return ref.Description.SingularName == "dashboard" +} + +// dashboardResourceCompletion executes to autocomplete the argument to the resource flag. +func dashboardResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b, diags := root.MustConfigureBundle(cmd) + if err := diags.Error(); err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return maps.Keys(resources.Completions(b, filterDashboards)), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateDashboardCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dashboard", + Short: "Generate configuration for a dashboard", + } + + d := &dashboard{} + + // Lookup flags. + cmd.Flags().StringVar(&d.existingPath, "existing-path", "", `workspace path of the dashboard to generate configuration for`) + cmd.Flags().StringVar(&d.existingID, "existing-id", "", `ID of the dashboard to generate configuration for`) + cmd.Flags().StringVar(&d.resource, "resource", "", `resource key of dashboard to watch for changes`) + + // Alias lookup flags that include the resource type name. + // Included for symmetry with the other generate commands, but we prefer the shorter flags. + cmd.Flags().StringVar(&d.existingPath, "existing-dashboard-path", "", `workspace path of the dashboard to generate configuration for`) + cmd.Flags().StringVar(&d.existingID, "existing-dashboard-id", "", `ID of the dashboard to generate configuration for`) + cmd.Flags().MarkHidden("existing-dashboard-path") + cmd.Flags().MarkHidden("existing-dashboard-id") + + // Output flags. + cmd.Flags().StringVarP(&d.resourceDir, "resource-dir", "d", "./resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&d.dashboardDir, "dashboard-dir", "s", "./src", `directory to write the dashboard representation to`) + cmd.Flags().BoolVarP(&d.force, "force", "f", false, `force overwrite existing files in the output directory`) + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-path", + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&d.watch, "watch", false, `watch for changes to the dashboard and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", dashboardResourceCompletion) + + cmd.RunE = d.RunE + return cmd +} diff --git a/cmd/bundle/generate/dashboard_test.go b/cmd/bundle/generate/dashboard_test.go new file mode 100644 index 000000000..6741e6a39 --- /dev/null +++ b/cmd/bundle/generate/dashboard_test.go @@ -0,0 +1,182 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) { + // Response to a GetStatus request on a path pointing to a legacy dashboard. + // + // < HTTP/2.0 400 Bad Request + // < { + // < "error_code": "BAD_REQUEST", + // < "message": "dbsqlDashboard is not user-facing." + // < } + + d := dashboard{ + existingPath: "/path/to/legacy dashboard", + } + + m := mocks.NewMockWorkspaceClient(t) + w := m.GetMockWorkspaceAPI() + w.On("GetStatusByPath", mock.Anything, "/path/to/legacy dashboard").Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "BAD_REQUEST", + Message: "dbsqlDashboard is not user-facing.", + }) + + ctx := context.Background() + b := &bundle.Bundle{} + b.SetWorkpaceClient(m.WorkspaceClient) + + _, diags := d.resolveID(ctx, b) + require.Len(t, diags, 1) + assert.Equal(t, diags[0].Summary, "dashboard \"legacy dashboard\" is a legacy dashboard") +} + +func TestDashboard_ExistingID_Nominal(t *testing.T) { + root := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: root, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(&dashboards.Dashboard{ + DashboardId: "f00dcafe", + DisplayName: "This is a test dashboard", + SerializedDashboard: `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, + WarehouseId: "w4r3h0us3", + }, nil) + + ctx := bundle.Context(context.Background(), b) + cmd := NewGenerateDashboardCommand() + cmd.SetContext(ctx) + cmd.Flag("existing-id").Value.Set("f00dcafe") + + err := cmd.RunE(cmd, []string{}) + require.NoError(t, err) + + // Assert the contents of the generated configuration + data, err := os.ReadFile(filepath.Join(root, "resources", "this_is_a_test_dashboard.dashboard.yml")) + require.NoError(t, err) + assert.Equal(t, `resources: + dashboards: + this_is_a_test_dashboard: + display_name: "This is a test dashboard" + warehouse_id: w4r3h0us3 + file_path: ../src/this_is_a_test_dashboard.lvdash.json +`, string(data)) + + data, err = os.ReadFile(filepath.Join(root, "src", "this_is_a_test_dashboard.lvdash.json")) + require.NoError(t, err) + assert.JSONEq(t, `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, string(data)) +} + +func TestDashboard_ExistingID_NotFound(t *testing.T) { + root := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: root, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + + ctx := bundle.Context(context.Background(), b) + cmd := NewGenerateDashboardCommand() + cmd.SetContext(ctx) + cmd.Flag("existing-id").Value.Set("f00dcafe") + + err := cmd.RunE(cmd, []string{}) + require.Error(t, err) +} + +func TestDashboard_ExistingPath_Nominal(t *testing.T) { + root := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: root, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + workspaceAPI := m.GetMockWorkspaceAPI() + workspaceAPI.EXPECT().GetStatusByPath(mock.Anything, "/path/to/dashboard").Return(&workspace.ObjectInfo{ + ObjectType: workspace.ObjectTypeDashboard, + ResourceId: "f00dcafe", + }, nil) + + dashboardsAPI := m.GetMockLakeviewAPI() + dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(&dashboards.Dashboard{ + DashboardId: "f00dcafe", + DisplayName: "This is a test dashboard", + SerializedDashboard: `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, + WarehouseId: "w4r3h0us3", + }, nil) + + ctx := bundle.Context(context.Background(), b) + cmd := NewGenerateDashboardCommand() + cmd.SetContext(ctx) + cmd.Flag("existing-path").Value.Set("/path/to/dashboard") + + err := cmd.RunE(cmd, []string{}) + require.NoError(t, err) + + // Assert the contents of the generated configuration + data, err := os.ReadFile(filepath.Join(root, "resources", "this_is_a_test_dashboard.dashboard.yml")) + require.NoError(t, err) + assert.Equal(t, `resources: + dashboards: + this_is_a_test_dashboard: + display_name: "This is a test dashboard" + warehouse_id: w4r3h0us3 + file_path: ../src/this_is_a_test_dashboard.lvdash.json +`, string(data)) + + data, err = os.ReadFile(filepath.Join(root, "src", "this_is_a_test_dashboard.lvdash.json")) + require.NoError(t, err) + assert.JSONEq(t, `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, string(data)) +} + +func TestDashboard_ExistingPath_NotFound(t *testing.T) { + root := t.TempDir() + b := &bundle.Bundle{ + BundleRootPath: root, + } + + m := mocks.NewMockWorkspaceClient(t) + b.SetWorkpaceClient(m.WorkspaceClient) + + workspaceAPI := m.GetMockWorkspaceAPI() + workspaceAPI.EXPECT().GetStatusByPath(mock.Anything, "/path/to/dashboard").Return(nil, &apierr.APIError{ + StatusCode: 404, + }) + + ctx := bundle.Context(context.Background(), b) + cmd := NewGenerateDashboardCommand() + cmd.SetContext(ctx) + cmd.Flag("existing-path").Value.Set("/path/to/dashboard") + + err := cmd.RunE(cmd, []string{}) + require.Error(t, err) +} diff --git a/cmd/bundle/open.go b/cmd/bundle/open.go new file mode 100644 index 000000000..a2ad32fd8 --- /dev/null +++ b/cmd/bundle/open.go @@ -0,0 +1,144 @@ +package bundle + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + + "github.com/pkg/browser" +) + +func promptOpenArgument(ctx context.Context, b *bundle.Bundle) (string, error) { + // Compute map of "Human readable name of resource" -> "resource key". + inv := make(map[string]string) + for k, ref := range resources.Completions(b) { + title := fmt.Sprintf("%s: %s", ref.Description.SingularTitle, ref.Resource.GetName()) + inv[title] = k + } + + key, err := cmdio.Select(ctx, inv, "Resource to open") + if err != nil { + return "", err + } + + return key, nil +} + +func resolveOpenArgument(ctx context.Context, b *bundle.Bundle, args []string) (string, error) { + // If no arguments are specified, prompt the user to select the resource to open. + if len(args) == 0 && cmdio.IsPromptSupported(ctx) { + return promptOpenArgument(ctx, b) + } + + if len(args) < 1 { + return "", fmt.Errorf("expected a KEY of the resource to open") + } + + return args[0], nil +} + +func newOpenCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "open", + Short: "Open a resource in the browser", + Args: root.MaximumNArgs(1), + } + + var forcePull bool + cmd.Flags().BoolVar(&forcePull, "force-pull", false, "Skip local cache and load the state from the remote workspace") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + b, diags := utils.ConfigureBundleWithVariables(cmd) + if err := diags.Error(); err != nil { + return diags.Error() + } + + diags = bundle.Apply(ctx, b, phases.Initialize()) + if err := diags.Error(); err != nil { + return err + } + + arg, err := resolveOpenArgument(ctx, b, args) + if err != nil { + return err + } + + cacheDir, err := terraform.Dir(ctx, b) + if err != nil { + return err + } + _, stateFileErr := os.Stat(filepath.Join(cacheDir, terraform.TerraformStateFileName)) + _, configFileErr := os.Stat(filepath.Join(cacheDir, terraform.TerraformConfigFileName)) + noCache := errors.Is(stateFileErr, os.ErrNotExist) || errors.Is(configFileErr, os.ErrNotExist) + + if forcePull || noCache { + diags = bundle.Apply(ctx, b, bundle.Seq( + terraform.StatePull(), + terraform.Interpolate(), + terraform.Write(), + )) + if err := diags.Error(); err != nil { + return err + } + } + + diags = bundle.Apply(ctx, b, bundle.Seq( + terraform.Load(), + mutator.InitializeURLs(), + )) + if err := diags.Error(); err != nil { + return err + } + + // Locate resource to open. + ref, err := resources.Lookup(b, arg) + if err != nil { + return err + } + + // Confirm that the resource has a URL. + url := ref.Resource.GetURL() + if url == "" { + return fmt.Errorf("resource does not have a URL associated with it (has it been deployed?)") + } + + return browser.OpenURL(url) + } + + cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b, diags := root.MustConfigureBundle(cmd) + if err := diags.Error(); err != nil { + cobra.CompErrorln(err.Error()) + return nil, cobra.ShellCompDirectiveError + } + + // No completion in the context of a bundle. + // Source and destination paths are taken from bundle configuration. + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + if len(args) == 0 { + completions := resources.Completions(b) + return maps.Keys(completions), cobra.ShellCompDirectiveNoFileComp + } else { + return nil, cobra.ShellCompDirectiveNoFileComp + } + } + + return cmd +} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 9ef5eb8ff..96851d0c0 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -1,20 +1,69 @@ package bundle import ( + "context" "encoding/json" "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" + "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/spf13/cobra" + "golang.org/x/exp/maps" ) +func promptRunArgument(ctx context.Context, b *bundle.Bundle) (string, error) { + // Compute map of "Human readable name of resource" -> "resource key". + inv := make(map[string]string) + for k, ref := range resources.Completions(b, run.IsRunnable) { + title := fmt.Sprintf("%s: %s", ref.Description.SingularTitle, ref.Resource.GetName()) + inv[title] = k + } + + key, err := cmdio.Select(ctx, inv, "Resource to run") + if err != nil { + return "", err + } + + return key, nil +} + +func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (string, error) { + // If no arguments are specified, prompt the user to select something to run. + if len(args) == 0 && cmdio.IsPromptSupported(ctx) { + return promptRunArgument(ctx, b) + } + + if len(args) < 1 { + return "", fmt.Errorf("expected a KEY of the resource to run") + } + + return args[0], nil +} + +func keyToRunner(b *bundle.Bundle, arg string) (run.Runner, error) { + // Locate the resource to run. + ref, err := resources.Lookup(b, arg, run.IsRunnable) + if err != nil { + return nil, err + } + + // Convert the resource to a runnable resource. + runner, err := run.ToRunner(b, ref) + if err != nil { + return nil, err + } + + return runner, nil +} + func newRunCommand() *cobra.Command { cmd := &cobra.Command{ Use: "run [flags] KEY", @@ -60,22 +109,9 @@ task or a Python wheel task, the second example applies. return err } - // If no arguments are specified, prompt the user to select something to run. - if len(args) == 0 && cmdio.IsPromptSupported(ctx) { - // Invert completions from KEY -> NAME, to NAME -> KEY. - inv := make(map[string]string) - for k, v := range run.ResourceCompletionMap(b) { - inv[v] = k - } - id, err := cmdio.Select(ctx, inv, "Resource to run") - if err != nil { - return err - } - args = append(args, id) - } - - if len(args) < 1 { - return fmt.Errorf("expected a KEY of the resource to run") + arg, err := resolveRunArgument(ctx, b, args) + if err != nil { + return err } diags = bundle.Apply(ctx, b, bundle.Seq( @@ -88,7 +124,7 @@ task or a Python wheel task, the second example applies. return err } - runner, err := run.Find(b, args[0]) + runner, err := keyToRunner(b, arg) if err != nil { return err } @@ -100,19 +136,16 @@ task or a Python wheel task, the second example applies. } runOptions.NoWait = noWait + var output output.RunOutput if restart { - s := cmdio.Spinner(ctx) - s <- "Cancelling all runs" - err := runner.Cancel(ctx) - close(s) - if err != nil { - return err - } + output, err = runner.Restart(ctx, &runOptions) + } else { + output, err = runner.Run(ctx, &runOptions) } - output, err := runner.Run(ctx, &runOptions) if err != nil { return err } + if output != nil { switch root.OutputType(cmd) { case flags.OutputText: @@ -148,10 +181,11 @@ task or a Python wheel task, the second example applies. } if len(args) == 0 { - return run.ResourceCompletions(b), cobra.ShellCompDirectiveNoFileComp + completions := resources.Completions(b, run.IsRunnable) + return maps.Keys(completions), cobra.ShellCompDirectiveNoFileComp } else { // If we know the resource to run, we can complete additional positional arguments. - runner, err := run.Find(b, args[0]) + runner, err := keyToRunner(b, args[0]) if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index 5a64b46c0..8c34dd612 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -8,8 +8,10 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/render" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/flags" @@ -19,11 +21,8 @@ import ( func newSummaryCommand() *cobra.Command { cmd := &cobra.Command{ Use: "summary", - Short: "Describe the bundle resources and their deployment states", + Short: "Summarize resources deployed by this bundle", Args: root.NoArgs, - - // This command is currently intended for the Databricks VSCode extension only - Hidden: true, } var forcePull bool @@ -60,14 +59,15 @@ func newSummaryCommand() *cobra.Command { } } - diags = bundle.Apply(ctx, b, terraform.Load()) + diags = bundle.Apply(ctx, b, + bundle.Seq(terraform.Load(), mutator.InitializeURLs())) if err := diags.Error(); err != nil { return err } switch root.OutputType(cmd) { case flags.OutputText: - return fmt.Errorf("%w, only json output is supported", errors.ErrUnsupported) + return render.RenderSummary(ctx, cmd.OutOrStdout(), b) case flags.OutputJSON: buf, err := json.MarshalIndent(b.Config, "", " ") if err != nil { diff --git a/cmd/bundle/sync.go b/cmd/bundle/sync.go index df3e087c2..274bba0e0 100644 --- a/cmd/bundle/sync.go +++ b/cmd/bundle/sync.go @@ -1,7 +1,9 @@ package bundle import ( + "context" "fmt" + "io" "time" "github.com/databricks/cli/bundle" @@ -9,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/sync" "github.com/spf13/cobra" @@ -18,6 +21,7 @@ type syncFlags struct { interval time.Duration full bool watch bool + output flags.Output } func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) { @@ -26,6 +30,21 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) return nil, fmt.Errorf("cannot get sync options: %w", err) } + if f.output != "" { + var outputFunc func(context.Context, <-chan sync.Event, io.Writer) + switch f.output { + case flags.OutputText: + outputFunc = sync.TextOutput + case flags.OutputJSON: + outputFunc = sync.JsonOutput + } + if outputFunc != nil { + opts.OutputHandler = func(ctx context.Context, c <-chan sync.Event) { + outputFunc(ctx, c, cmd.OutOrStdout()) + } + } + } + opts.Full = f.full opts.PollInterval = f.interval return opts, nil @@ -42,6 +61,7 @@ func newSyncCommand() *cobra.Command { cmd.Flags().DurationVar(&f.interval, "interval", 1*time.Second, "file system polling interval (for --watch)") cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)") cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes") + cmd.Flags().Var(&f.output, "output", "type of the output format") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -65,6 +85,7 @@ func newSyncCommand() *cobra.Command { if err != nil { return err } + defer s.Close() log.Infof(ctx, "Remote file sync location: %v", opts.RemotePath) diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 496d5d2b5..5331e7e7b 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -54,7 +54,7 @@ func newValidateCommand() *cobra.Command { switch root.OutputType(cmd) { case flags.OutputText: renderOpts := render.RenderOptions{RenderSummaryTable: true} - err := render.RenderTextOutput(cmd.OutOrStdout(), b, diags, renderOpts) + err := render.RenderDiagnostics(cmd.OutOrStdout(), b, diags, renderOpts) if err != nil { return fmt.Errorf("failed to render output: %w", err) } diff --git a/cmd/workspace/alerts-legacy/alerts-legacy.go b/cmd/workspace/alerts-legacy/alerts-legacy.go index 1046b1124..4f7d5f966 100755 --- a/cmd/workspace/alerts-legacy/alerts-legacy.go +++ b/cmd/workspace/alerts-legacy/alerts-legacy.go @@ -93,9 +93,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -357,9 +363,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/alerts/alerts.go b/cmd/workspace/alerts/alerts.go index cfaa3f55f..fcf18652b 100755 --- a/cmd/workspace/alerts/alerts.go +++ b/cmd/workspace/alerts/alerts.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -354,9 +360,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index baec6d03c..9331ddc2e 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -28,9 +28,6 @@ func New() *cobra.Command { Annotations: map[string]string{ "package": "apps", }, - - // This service is being previewed; hide from help output. - Hidden: true, } // Add methods @@ -81,6 +78,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "create NAME" cmd.Short = `Create an app.` @@ -112,9 +110,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -266,9 +270,15 @@ func newDeploy() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deployJson.Unmarshal(&deployReq) - if err != nil { - return err + diags := deployJson.Unmarshal(&deployReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } deployReq.AppName = args[0] @@ -701,9 +711,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.AppName = args[0] @@ -910,6 +926,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "update NAME" cmd.Short = `Update an app.` @@ -934,9 +951,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -1003,9 +1026,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.AppName = args[0] diff --git a/cmd/workspace/artifact-allowlists/artifact-allowlists.go b/cmd/workspace/artifact-allowlists/artifact-allowlists.go index fc25e3cb8..e66833697 100755 --- a/cmd/workspace/artifact-allowlists/artifact-allowlists.go +++ b/cmd/workspace/artifact-allowlists/artifact-allowlists.go @@ -145,9 +145,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go index 2385195bb..dca88f3d1 100755 --- a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go +++ b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go @@ -127,9 +127,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/catalogs/catalogs.go b/cmd/workspace/catalogs/catalogs.go index a17bb0072..9294c192b 100755 --- a/cmd/workspace/catalogs/catalogs.go +++ b/cmd/workspace/catalogs/catalogs.go @@ -105,9 +105,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -363,9 +369,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/clean-rooms/clean-rooms.go b/cmd/workspace/clean-rooms/clean-rooms.go index 9466c4b94..72560b846 100755 --- a/cmd/workspace/clean-rooms/clean-rooms.go +++ b/cmd/workspace/clean-rooms/clean-rooms.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -344,9 +350,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/cluster-policies/cluster-policies.go b/cmd/workspace/cluster-policies/cluster-policies.go index 830d44ca3..b34dd53db 100755 --- a/cmd/workspace/cluster-policies/cluster-policies.go +++ b/cmd/workspace/cluster-policies/cluster-policies.go @@ -113,9 +113,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -185,9 +191,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -284,9 +296,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -630,9 +648,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -711,9 +735,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index b36102d98..0ed454de2 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -134,9 +134,15 @@ func newChangeOwner() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = changeOwnerJson.Unmarshal(&changeOwnerReq) - if err != nil { - return err + diags := changeOwnerJson.Unmarshal(&changeOwnerReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -268,9 +274,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -362,9 +374,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -519,9 +537,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -617,9 +641,15 @@ func newEvents() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = eventsJson.Unmarshal(&eventsReq) - if err != nil { - return err + diags := eventsJson.Unmarshal(&eventsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1069,9 +1099,15 @@ func newPermanentDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = permanentDeleteJson.Unmarshal(&permanentDeleteReq) - if err != nil { - return err + diags := permanentDeleteJson.Unmarshal(&permanentDeleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1161,9 +1197,15 @@ func newPin() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = pinJson.Unmarshal(&pinReq) - if err != nil { - return err + diags := pinJson.Unmarshal(&pinReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1260,9 +1302,15 @@ func newResize() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = resizeJson.Unmarshal(&resizeReq) - if err != nil { - return err + diags := resizeJson.Unmarshal(&resizeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1370,9 +1418,15 @@ func newRestart() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restartJson.Unmarshal(&restartReq) - if err != nil { - return err + diags := restartJson.Unmarshal(&restartReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1464,9 +1518,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1608,9 +1668,15 @@ func newStart() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startJson.Unmarshal(&startReq) - if err != nil { - return err + diags := startJson.Unmarshal(&startReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1712,9 +1778,15 @@ func newUnpin() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = unpinJson.Unmarshal(&unpinReq) - if err != nil { - return err + diags := unpinJson.Unmarshal(&unpinReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1824,9 +1896,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1905,9 +1983,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/compliance-security-profile/compliance-security-profile.go b/cmd/workspace/compliance-security-profile/compliance-security-profile.go index a7b45901f..58f3eddad 100755 --- a/cmd/workspace/compliance-security-profile/compliance-security-profile.go +++ b/cmd/workspace/compliance-security-profile/compliance-security-profile.go @@ -130,9 +130,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go index f76420fbe..161f5ab41 100755 --- a/cmd/workspace/connections/connections.go +++ b/cmd/workspace/connections/connections.go @@ -92,9 +92,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -355,9 +361,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go index 92f61789f..1848cb8fc 100755 --- a/cmd/workspace/consumer-installations/consumer-installations.go +++ b/cmd/workspace/consumer-installations/consumer-installations.go @@ -86,9 +86,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.ListingId = args[0] @@ -319,9 +325,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go index 8b0af3cc6..6d751c631 100755 --- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go +++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/credentials-manager/credentials-manager.go b/cmd/workspace/credentials-manager/credentials-manager.go index 5a40232b3..e29bc0bd4 100755 --- a/cmd/workspace/credentials-manager/credentials-manager.go +++ b/cmd/workspace/credentials-manager/credentials-manager.go @@ -75,9 +75,15 @@ func newExchangeToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = exchangeTokenJson.Unmarshal(&exchangeTokenReq) - if err != nil { - return err + diags := exchangeTokenJson.Unmarshal(&exchangeTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/dashboard-widgets/dashboard-widgets.go b/cmd/workspace/dashboard-widgets/dashboard-widgets.go index 02b13739a..e42818266 100755 --- a/cmd/workspace/dashboard-widgets/dashboard-widgets.go +++ b/cmd/workspace/dashboard-widgets/dashboard-widgets.go @@ -75,9 +75,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -196,9 +202,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/dashboards/dashboards.go b/cmd/workspace/dashboards/dashboards.go index fcab0aa2a..5bdd7b137 100755 --- a/cmd/workspace/dashboards/dashboards.go +++ b/cmd/workspace/dashboards/dashboards.go @@ -78,9 +78,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -405,9 +411,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/default-namespace/default-namespace.go b/cmd/workspace/default-namespace/default-namespace.go index b15907bec..e5039cbaf 100755 --- a/cmd/workspace/default-namespace/default-namespace.go +++ b/cmd/workspace/default-namespace/default-namespace.go @@ -199,9 +199,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/disable-legacy-access/disable-legacy-access.go b/cmd/workspace/disable-legacy-access/disable-legacy-access.go index fea2b3c40..c50de446b 100755 --- a/cmd/workspace/disable-legacy-access/disable-legacy-access.go +++ b/cmd/workspace/disable-legacy-access/disable-legacy-access.go @@ -187,9 +187,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go b/cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go new file mode 100755 index 000000000..d09755370 --- /dev/null +++ b/cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go @@ -0,0 +1,220 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package disable_legacy_dbfs + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "disable-legacy-dbfs", + Short: `When this setting is on, access to DBFS root and DBFS mounts is disallowed (as well as creation of new mounts).`, + Long: `When this setting is on, access to DBFS root and DBFS mounts is disallowed (as + well as creation of new mounts). When the setting is off, all DBFS + functionality is enabled`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDisableLegacyDbfsRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDisableLegacyDbfsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the disable legacy DBFS setting.` + cmd.Long = `Delete the disable legacy DBFS setting. + + Deletes the disable legacy DBFS setting for a workspace, reverting back to the + default.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyDbfs().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDisableLegacyDbfsRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDisableLegacyDbfsRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the disable legacy DBFS setting.` + cmd.Long = `Get the disable legacy DBFS setting. + + Gets the disable legacy DBFS setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyDbfs().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDisableLegacyDbfsRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDisableLegacyDbfsRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the disable legacy DBFS setting.` + cmd.Long = `Update the disable legacy DBFS setting. + + Updates the disable legacy DBFS setting for the workspace.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.Settings.DisableLegacyDbfs().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DisableLegacyDbfs diff --git a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go index a8acc5cd1..3d99ecef8 100755 --- a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go +++ b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go @@ -132,9 +132,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index b1af2f868..4c6b57d18 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -130,9 +130,15 @@ func newCreateExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createExperimentJson.Unmarshal(&createExperimentReq) - if err != nil { - return err + diags := createExperimentJson.Unmarshal(&createExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -203,9 +209,15 @@ func newCreateRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createRunJson.Unmarshal(&createRunReq) - if err != nil { - return err + diags := createRunJson.Unmarshal(&createRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -277,9 +289,15 @@ func newDeleteExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteExperimentJson.Unmarshal(&deleteExperimentReq) - if err != nil { - return err + diags := deleteExperimentJson.Unmarshal(&deleteExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -352,9 +370,15 @@ func newDeleteRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunJson.Unmarshal(&deleteRunReq) - if err != nil { - return err + diags := deleteRunJson.Unmarshal(&deleteRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -435,9 +459,15 @@ func newDeleteRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunsJson.Unmarshal(&deleteRunsReq) - if err != nil { - return err + diags := deleteRunsJson.Unmarshal(&deleteRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -518,9 +548,15 @@ func newDeleteTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteTagJson.Unmarshal(&deleteTagReq) - if err != nil { - return err + diags := deleteTagJson.Unmarshal(&deleteTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1108,9 +1144,15 @@ func newLogBatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logBatchJson.Unmarshal(&logBatchReq) - if err != nil { - return err + diags := logBatchJson.Unmarshal(&logBatchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1174,9 +1216,15 @@ func newLogInputs() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logInputsJson.Unmarshal(&logInputsReq) - if err != nil { - return err + diags := logInputsJson.Unmarshal(&logInputsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1254,9 +1302,15 @@ func newLogMetric() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logMetricJson.Unmarshal(&logMetricReq) - if err != nil { - return err + diags := logMetricJson.Unmarshal(&logMetricReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1335,9 +1389,15 @@ func newLogModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logModelJson.Unmarshal(&logModelReq) - if err != nil { - return err + diags := logModelJson.Unmarshal(&logModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1414,9 +1474,15 @@ func newLogParam() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = logParamJson.Unmarshal(&logParamReq) - if err != nil { - return err + diags := logParamJson.Unmarshal(&logParamReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1497,9 +1563,15 @@ func newRestoreExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreExperimentJson.Unmarshal(&restoreExperimentReq) - if err != nil { - return err + diags := restoreExperimentJson.Unmarshal(&restoreExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1572,9 +1644,15 @@ func newRestoreRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreRunJson.Unmarshal(&restoreRunReq) - if err != nil { - return err + diags := restoreRunJson.Unmarshal(&restoreRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1655,9 +1733,15 @@ func newRestoreRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = restoreRunsJson.Unmarshal(&restoreRunsReq) - if err != nil { - return err + diags := restoreRunsJson.Unmarshal(&restoreRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1732,9 +1816,15 @@ func newSearchExperiments() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = searchExperimentsJson.Unmarshal(&searchExperimentsReq) - if err != nil { - return err + diags := searchExperimentsJson.Unmarshal(&searchExperimentsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1800,9 +1890,15 @@ func newSearchRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = searchRunsJson.Unmarshal(&searchRunsReq) - if err != nil { - return err + diags := searchRunsJson.Unmarshal(&searchRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1874,9 +1970,15 @@ func newSetExperimentTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setExperimentTagJson.Unmarshal(&setExperimentTagReq) - if err != nil { - return err + diags := setExperimentTagJson.Unmarshal(&setExperimentTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1951,9 +2053,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.ExperimentId = args[0] @@ -2032,9 +2140,15 @@ func newSetTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setTagJson.Unmarshal(&setTagReq) - if err != nil { - return err + diags := setTagJson.Unmarshal(&setTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2112,9 +2226,15 @@ func newUpdateExperiment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateExperimentJson.Unmarshal(&updateExperimentReq) - if err != nil { - return err + diags := updateExperimentJson.Unmarshal(&updateExperimentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2183,9 +2303,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.ExperimentId = args[0] @@ -2251,9 +2377,15 @@ func newUpdateRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateRunJson.Unmarshal(&updateRunReq) - if err != nil { - return err + diags := updateRunJson.Unmarshal(&updateRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/external-locations/external-locations.go b/cmd/workspace/external-locations/external-locations.go index 42493fc46..97d34df09 100755 --- a/cmd/workspace/external-locations/external-locations.go +++ b/cmd/workspace/external-locations/external-locations.go @@ -112,9 +112,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -381,9 +387,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] diff --git a/cmd/workspace/functions/functions.go b/cmd/workspace/functions/functions.go index c8de48797..86b292674 100755 --- a/cmd/workspace/functions/functions.go +++ b/cmd/workspace/functions/functions.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -381,9 +387,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go index e4a059091..287bcde63 100755 --- a/cmd/workspace/genie/genie.go +++ b/cmd/workspace/genie/genie.go @@ -105,9 +105,15 @@ func newCreateMessage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createMessageJson.Unmarshal(&createMessageReq) - if err != nil { - return err + diags := createMessageJson.Unmarshal(&createMessageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createMessageReq.SpaceId = args[0] @@ -392,9 +398,15 @@ func newStartConversation() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startConversationJson.Unmarshal(&startConversationReq) - if err != nil { - return err + diags := startConversationJson.Unmarshal(&startConversationReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } startConversationReq.SpaceId = args[0] diff --git a/cmd/workspace/git-credentials/git-credentials.go b/cmd/workspace/git-credentials/git-credentials.go index 2e8cc2cd4..978ca6bac 100755 --- a/cmd/workspace/git-credentials/git-credentials.go +++ b/cmd/workspace/git-credentials/git-credentials.go @@ -53,13 +53,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateCredentials, + *workspace.CreateCredentialsRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateCredentials + var createReq workspace.CreateCredentialsRequest var createJson flags.JsonFlag // TODO: short flags @@ -79,8 +79,9 @@ func newCreate() *cobra.Command { Arguments: GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -102,9 +103,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -136,13 +143,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *workspace.DeleteGitCredentialRequest, + *workspace.DeleteCredentialsRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq workspace.DeleteGitCredentialRequest + var deleteReq workspace.DeleteCredentialsRequest // TODO: short flags @@ -209,13 +216,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *workspace.GetGitCredentialRequest, + *workspace.GetCredentialsRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq workspace.GetGitCredentialRequest + var getReq workspace.GetCredentialsRequest // TODO: short flags @@ -322,65 +329,72 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateCredentials, + *workspace.UpdateCredentialsRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateCredentials + var updateReq workspace.UpdateCredentialsRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateReq.GitProvider, "git-provider", updateReq.GitProvider, `Git provider.`) cmd.Flags().StringVar(&updateReq.GitUsername, "git-username", updateReq.GitUsername, `The username or email provided with your Git provider account, depending on which provider you are using.`) cmd.Flags().StringVar(&updateReq.PersonalAccessToken, "personal-access-token", updateReq.PersonalAccessToken, `The personal access token used to authenticate to the corresponding Git provider.`) - cmd.Use = "update CREDENTIAL_ID" + cmd.Use = "update CREDENTIAL_ID GIT_PROVIDER" cmd.Short = `Update a credential.` cmd.Long = `Update a credential. Updates the specified Git credential. Arguments: - CREDENTIAL_ID: The ID for the corresponding credential to access.` + CREDENTIAL_ID: The ID for the corresponding credential to access. + GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only CREDENTIAL_ID as positional arguments. Provide 'git_provider' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() } - } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No CREDENTIAL_ID argument specified. Loading names for Git Credentials drop-down." - names, err := w.GitCredentials.CredentialInfoGitProviderToCredentialIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Git Credentials drop-down. Please manually specify required arguments. Original error: %w", err) + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding credential to access") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding credential to access") } _, err = fmt.Sscan(args[0], &updateReq.CredentialId) if err != nil { return fmt.Errorf("invalid CREDENTIAL_ID: %s", args[0]) } + if !cmd.Flags().Changed("json") { + updateReq.GitProvider = args[1] + } err = w.GitCredentials.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/global-init-scripts/global-init-scripts.go b/cmd/workspace/global-init-scripts/global-init-scripts.go index 92dcb2592..52adde3fb 100755 --- a/cmd/workspace/global-init-scripts/global-init-scripts.go +++ b/cmd/workspace/global-init-scripts/global-init-scripts.go @@ -101,9 +101,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -367,9 +373,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.ScriptId = args[0] diff --git a/cmd/workspace/grants/grants.go b/cmd/workspace/grants/grants.go index 876f0343e..b1dd4e8cb 100755 --- a/cmd/workspace/grants/grants.go +++ b/cmd/workspace/grants/grants.go @@ -223,9 +223,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateReq.SecurableType) diff --git a/cmd/workspace/groups/groups.go b/cmd/workspace/groups/groups.go index 14650d984..a2d323106 100755 --- a/cmd/workspace/groups/groups.go +++ b/cmd/workspace/groups/groups.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -446,9 +458,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/instance-pools/instance-pools.go b/cmd/workspace/instance-pools/instance-pools.go index db96f1466..8a84df946 100755 --- a/cmd/workspace/instance-pools/instance-pools.go +++ b/cmd/workspace/instance-pools/instance-pools.go @@ -128,9 +128,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -206,9 +212,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -309,9 +321,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -631,9 +649,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -712,9 +736,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/instance-profiles/instance-profiles.go b/cmd/workspace/instance-profiles/instance-profiles.go index 7134c16c6..5c4bc8d9b 100755 --- a/cmd/workspace/instance-profiles/instance-profiles.go +++ b/cmd/workspace/instance-profiles/instance-profiles.go @@ -99,9 +99,15 @@ func newAdd() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = addJson.Unmarshal(&addReq) - if err != nil { - return err + diags := addJson.Unmarshal(&addReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -192,9 +198,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -311,9 +323,15 @@ func newRemove() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = removeJson.Unmarshal(&removeReq) - if err != nil { - return err + diags := removeJson.Unmarshal(&removeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/ip-access-lists/ip-access-lists.go b/cmd/workspace/ip-access-lists/ip-access-lists.go index ec5958b5b..070f279ec 100755 --- a/cmd/workspace/ip-access-lists/ip-access-lists.go +++ b/cmd/workspace/ip-access-lists/ip-access-lists.go @@ -133,9 +133,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -414,9 +420,15 @@ func newReplace() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = replaceJson.Unmarshal(&replaceReq) - if err != nil { - return err + diags := replaceJson.Unmarshal(&replaceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } replaceReq.IpAccessListId = args[0] @@ -510,9 +522,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 2d422fa8c..9e8db43d0 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -116,9 +116,15 @@ func newCancelAllRuns() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = cancelAllRunsJson.Unmarshal(&cancelAllRunsReq) - if err != nil { - return err + diags := cancelAllRunsJson.Unmarshal(&cancelAllRunsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -193,9 +199,15 @@ func newCancelRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = cancelRunJson.Unmarshal(&cancelRunReq) - if err != nil { - return err + diags := cancelRunJson.Unmarshal(&cancelRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -291,9 +303,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -364,9 +382,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -457,9 +481,15 @@ func newDeleteRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteRunJson.Unmarshal(&deleteRunReq) - if err != nil { - return err + diags := deleteRunJson.Unmarshal(&deleteRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1143,9 +1173,15 @@ func newRepairRun() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = repairRunJson.Unmarshal(&repairRunReq) - if err != nil { - return err + diags := repairRunJson.Unmarshal(&repairRunReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1242,9 +1278,15 @@ func newReset() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = resetJson.Unmarshal(&resetReq) - if err != nil { - return err + diags := resetJson.Unmarshal(&resetReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -1332,9 +1374,15 @@ func newRunNow() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = runNowJson.Unmarshal(&runNowReq) - if err != nil { - return err + diags := runNowJson.Unmarshal(&runNowReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1436,9 +1484,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1503,6 +1557,7 @@ func newSubmit() *cobra.Command { cmd.Flags().Var(&submitJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: access_control_list + cmd.Flags().StringVar(&submitReq.BudgetPolicyId, "budget-policy-id", submitReq.BudgetPolicyId, `The user specified id of the budget policy to use for this one-time run.`) // TODO: complex arg: email_notifications // TODO: array: environments // TODO: complex arg: git_source @@ -1538,9 +1593,15 @@ func newSubmit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = submitJson.Unmarshal(&submitReq) - if err != nil { - return err + diags := submitJson.Unmarshal(&submitReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1632,9 +1693,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -1717,9 +1784,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/lakeview/lakeview.go b/cmd/workspace/lakeview/lakeview.go index ef2d6845b..33a45c65f 100755 --- a/cmd/workspace/lakeview/lakeview.go +++ b/cmd/workspace/lakeview/lakeview.go @@ -108,9 +108,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -180,9 +186,15 @@ func newCreateSchedule() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createScheduleJson.Unmarshal(&createScheduleReq) - if err != nil { - return err + diags := createScheduleJson.Unmarshal(&createScheduleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -250,9 +262,15 @@ func newCreateSubscription() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createSubscriptionJson.Unmarshal(&createSubscriptionReq) - if err != nil { - return err + diags := createSubscriptionJson.Unmarshal(&createSubscriptionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -870,9 +888,15 @@ func newMigrate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = migrateJson.Unmarshal(&migrateReq) - if err != nil { - return err + diags := migrateJson.Unmarshal(&migrateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -941,9 +965,15 @@ func newPublish() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = publishJson.Unmarshal(&publishReq) - if err != nil { - return err + diags := publishJson.Unmarshal(&publishReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } publishReq.DashboardId = args[0] @@ -1128,9 +1158,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.DashboardId = args[0] @@ -1200,9 +1236,15 @@ func newUpdateSchedule() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateScheduleJson.Unmarshal(&updateScheduleReq) - if err != nil { - return err + diags := updateScheduleJson.Unmarshal(&updateScheduleReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/libraries/libraries.go b/cmd/workspace/libraries/libraries.go index 2c10d8161..e6b332aef 100755 --- a/cmd/workspace/libraries/libraries.go +++ b/cmd/workspace/libraries/libraries.go @@ -190,9 +190,15 @@ func newInstall() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = installJson.Unmarshal(&installReq) - if err != nil { - return err + diags := installJson.Unmarshal(&installReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -251,9 +257,15 @@ func newUninstall() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = uninstallJson.Unmarshal(&uninstallReq) - if err != nil { - return err + diags := uninstallJson.Unmarshal(&uninstallReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/metastores/metastores.go b/cmd/workspace/metastores/metastores.go index 22bcd3dcf..563beb2f4 100755 --- a/cmd/workspace/metastores/metastores.go +++ b/cmd/workspace/metastores/metastores.go @@ -112,9 +112,15 @@ func newAssign() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = assignJson.Unmarshal(&assignReq) - if err != nil { - return err + diags := assignJson.Unmarshal(&assignReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &assignReq.WorkspaceId) @@ -201,9 +207,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -606,9 +618,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -690,9 +708,15 @@ func newUpdateAssignment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateAssignmentJson.Unmarshal(&updateAssignmentReq) - if err != nil { - return err + diags := updateAssignmentJson.Unmarshal(&updateAssignmentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/model-registry/model-registry.go b/cmd/workspace/model-registry/model-registry.go index 41f06ac4d..b45d83e3d 100755 --- a/cmd/workspace/model-registry/model-registry.go +++ b/cmd/workspace/model-registry/model-registry.go @@ -141,9 +141,15 @@ func newApproveTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = approveTransitionRequestJson.Unmarshal(&approveTransitionRequestReq) - if err != nil { - return err + diags := approveTransitionRequestJson.Unmarshal(&approveTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -235,9 +241,15 @@ func newCreateComment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createCommentJson.Unmarshal(&createCommentReq) - if err != nil { - return err + diags := createCommentJson.Unmarshal(&createCommentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -322,9 +334,15 @@ func newCreateModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createModelJson.Unmarshal(&createModelReq) - if err != nil { - return err + diags := createModelJson.Unmarshal(&createModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -403,9 +421,15 @@ func newCreateModelVersion() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createModelVersionJson.Unmarshal(&createModelVersionReq) - if err != nil { - return err + diags := createModelVersionJson.Unmarshal(&createModelVersionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -493,9 +517,15 @@ func newCreateTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createTransitionRequestJson.Unmarshal(&createTransitionRequestReq) - if err != nil { - return err + diags := createTransitionRequestJson.Unmarshal(&createTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -570,9 +600,15 @@ func newCreateWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createWebhookJson.Unmarshal(&createWebhookReq) - if err != nil { - return err + diags := createWebhookJson.Unmarshal(&createWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -1079,9 +1115,15 @@ func newGetLatestVersions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = getLatestVersionsJson.Unmarshal(&getLatestVersionsReq) - if err != nil { - return err + diags := getLatestVersionsJson.Unmarshal(&getLatestVersionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1629,9 +1671,15 @@ func newRejectTransitionRequest() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = rejectTransitionRequestJson.Unmarshal(&rejectTransitionRequestReq) - if err != nil { - return err + diags := rejectTransitionRequestJson.Unmarshal(&rejectTransitionRequestReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1715,9 +1763,15 @@ func newRenameModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = renameModelJson.Unmarshal(&renameModelReq) - if err != nil { - return err + diags := renameModelJson.Unmarshal(&renameModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1907,9 +1961,15 @@ func newSetModelTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setModelTagJson.Unmarshal(&setModelTagReq) - if err != nil { - return err + diags := setModelTagJson.Unmarshal(&setModelTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -1996,9 +2056,15 @@ func newSetModelVersionTag() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setModelVersionTagJson.Unmarshal(&setModelVersionTagReq) - if err != nil { - return err + diags := setModelVersionTagJson.Unmarshal(&setModelVersionTagReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2076,9 +2142,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.RegisteredModelId = args[0] @@ -2166,9 +2238,15 @@ func newTestRegistryWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = testRegistryWebhookJson.Unmarshal(&testRegistryWebhookReq) - if err != nil { - return err + diags := testRegistryWebhookJson.Unmarshal(&testRegistryWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2259,9 +2337,15 @@ func newTransitionStage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = transitionStageJson.Unmarshal(&transitionStageReq) - if err != nil { - return err + diags := transitionStageJson.Unmarshal(&transitionStageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2350,9 +2434,15 @@ func newUpdateComment() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateCommentJson.Unmarshal(&updateCommentReq) - if err != nil { - return err + diags := updateCommentJson.Unmarshal(&updateCommentReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2430,9 +2520,15 @@ func newUpdateModel() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateModelJson.Unmarshal(&updateModelReq) - if err != nil { - return err + diags := updateModelJson.Unmarshal(&updateModelReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2508,9 +2604,15 @@ func newUpdateModelVersion() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateModelVersionJson.Unmarshal(&updateModelVersionReq) - if err != nil { - return err + diags := updateModelVersionJson.Unmarshal(&updateModelVersionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -2582,9 +2684,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.RegisteredModelId = args[0] @@ -2663,9 +2771,15 @@ func newUpdateWebhook() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateWebhookJson.Unmarshal(&updateWebhookReq) - if err != nil { - return err + diags := updateWebhookJson.Unmarshal(&updateWebhookReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go index d2f054045..439e5f657 100755 --- a/cmd/workspace/model-versions/model-versions.go +++ b/cmd/workspace/model-versions/model-versions.go @@ -377,9 +377,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.FullName = args[0] diff --git a/cmd/workspace/notification-destinations/notification-destinations.go b/cmd/workspace/notification-destinations/notification-destinations.go index 5ad47cc95..470765879 100755 --- a/cmd/workspace/notification-destinations/notification-destinations.go +++ b/cmd/workspace/notification-destinations/notification-destinations.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -313,9 +319,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/online-tables/online-tables.go b/cmd/workspace/online-tables/online-tables.go index da2f8c041..1c25d1e26 100755 --- a/cmd/workspace/online-tables/online-tables.go +++ b/cmd/workspace/online-tables/online-tables.go @@ -79,9 +79,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/permission-migration/permission-migration.go b/cmd/workspace/permission-migration/permission-migration.go index 2e50b1231..15ff1b751 100755 --- a/cmd/workspace/permission-migration/permission-migration.go +++ b/cmd/workspace/permission-migration/permission-migration.go @@ -92,9 +92,15 @@ func newMigratePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = migratePermissionsJson.Unmarshal(&migratePermissionsReq) - if err != nil { - return err + diags := migratePermissionsJson.Unmarshal(&migratePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/permissions/permissions.go b/cmd/workspace/permissions/permissions.go index c6033e4a4..d007a425f 100755 --- a/cmd/workspace/permissions/permissions.go +++ b/cmd/workspace/permissions/permissions.go @@ -265,9 +265,15 @@ func newSet() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setJson.Unmarshal(&setReq) - if err != nil { - return err + diags := setJson.Unmarshal(&setReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setReq.RequestObjectType = args[0] @@ -340,9 +346,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.RequestObjectType = args[0] diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 5b4d9645e..5bd94e0b6 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -98,9 +98,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -699,9 +705,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -788,9 +800,15 @@ func newStartUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = startUpdateJson.Unmarshal(&startUpdateReq) - if err != nil { - return err + diags := startUpdateJson.Unmarshal(&startUpdateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -954,6 +972,7 @@ func newUpdate() *cobra.Command { // TODO: array: notifications cmd.Flags().BoolVar(&updateReq.Photon, "photon", updateReq.Photon, `Whether Photon is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.PipelineId, "pipeline-id", updateReq.PipelineId, `Unique identifier for this pipeline.`) + cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`) cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`) cmd.Flags().StringVar(&updateReq.Target, "target", updateReq.Target, `Target schema (database) to add tables in this pipeline to.`) @@ -976,9 +995,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -1057,9 +1082,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go index 1274c8790..d128d80b8 100755 --- a/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go +++ b/cmd/workspace/policy-compliance-for-clusters/policy-compliance-for-clusters.go @@ -110,9 +110,15 @@ func newEnforceCompliance() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) - if err != nil { - return err + diags := enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go index d74caa572..384dab2c9 100755 --- a/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go +++ b/cmd/workspace/policy-compliance-for-jobs/policy-compliance-for-jobs.go @@ -104,9 +104,15 @@ func newEnforceCompliance() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = enforceComplianceJson.Unmarshal(&enforceComplianceReq) - if err != nil { - return err + diags := enforceComplianceJson.Unmarshal(&enforceComplianceReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go index a3f746214..fea836d20 100755 --- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go +++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go @@ -73,9 +73,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -259,9 +265,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go index b92403755..a96f0673e 100755 --- a/cmd/workspace/provider-exchanges/provider-exchanges.go +++ b/cmd/workspace/provider-exchanges/provider-exchanges.go @@ -91,9 +91,15 @@ func newAddListingToExchange() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = addListingToExchangeJson.Unmarshal(&addListingToExchangeReq) - if err != nil { - return err + diags := addListingToExchangeJson.Unmarshal(&addListingToExchangeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -154,9 +160,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -546,9 +558,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go index 62dcb6de9..392ed2890 100755 --- a/cmd/workspace/provider-files/provider-files.go +++ b/cmd/workspace/provider-files/provider-files.go @@ -77,9 +77,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -273,9 +279,15 @@ func newList() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = listJson.Unmarshal(&listReq) - if err != nil { - return err + diags := listJson.Unmarshal(&listReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go index 18c99c53d..4c7c6c563 100755 --- a/cmd/workspace/provider-listings/provider-listings.go +++ b/cmd/workspace/provider-listings/provider-listings.go @@ -75,9 +75,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -326,9 +332,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go index d18e2e578..48c444f1f 100755 --- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go +++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go @@ -142,9 +142,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.ListingId = args[0] diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go index bb3ca9666..a8d151a20 100755 --- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go +++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go @@ -208,9 +208,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go index 94d12d6f0..3c9c024e8 100755 --- a/cmd/workspace/provider-providers/provider-providers.go +++ b/cmd/workspace/provider-providers/provider-providers.go @@ -74,9 +74,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -325,9 +331,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/providers/providers.go b/cmd/workspace/providers/providers.go index af2737a0f..504beac5e 100755 --- a/cmd/workspace/providers/providers.go +++ b/cmd/workspace/providers/providers.go @@ -97,9 +97,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -446,9 +452,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/quality-monitors/quality-monitors.go b/cmd/workspace/quality-monitors/quality-monitors.go index 1ff9b017e..58075aa54 100755 --- a/cmd/workspace/quality-monitors/quality-monitors.go +++ b/cmd/workspace/quality-monitors/quality-monitors.go @@ -196,9 +196,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } createReq.TableName = args[0] @@ -559,9 +565,15 @@ func newRegenerateDashboard() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = regenerateDashboardJson.Unmarshal(®enerateDashboardReq) - if err != nil { - return err + diags := regenerateDashboardJson.Unmarshal(®enerateDashboardReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } regenerateDashboardReq.TableName = args[0] @@ -724,9 +736,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.TableName = args[0] diff --git a/cmd/workspace/queries-legacy/queries-legacy.go b/cmd/workspace/queries-legacy/queries-legacy.go index fa78bb2b0..e35e1828b 100755 --- a/cmd/workspace/queries-legacy/queries-legacy.go +++ b/cmd/workspace/queries-legacy/queries-legacy.go @@ -96,9 +96,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -454,9 +460,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index fea01451a..208f887da 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -85,9 +85,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -425,9 +431,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go index 4f45ab23e..f48acff1e 100755 --- a/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go +++ b/cmd/workspace/query-visualizations-legacy/query-visualizations-legacy.go @@ -87,9 +87,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -222,9 +228,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/query-visualizations/query-visualizations.go b/cmd/workspace/query-visualizations/query-visualizations.go index 042594529..621661952 100755 --- a/cmd/workspace/query-visualizations/query-visualizations.go +++ b/cmd/workspace/query-visualizations/query-visualizations.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -217,9 +223,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Id = args[0] diff --git a/cmd/workspace/recipients/recipients.go b/cmd/workspace/recipients/recipients.go index f4472cf37..56abd2014 100755 --- a/cmd/workspace/recipients/recipients.go +++ b/cmd/workspace/recipients/recipients.go @@ -118,9 +118,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -404,9 +410,15 @@ func newRotateToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = rotateTokenJson.Unmarshal(&rotateTokenReq) - if err != nil { - return err + diags := rotateTokenJson.Unmarshal(&rotateTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } rotateTokenReq.Name = args[0] @@ -554,9 +566,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/registered-models/registered-models.go b/cmd/workspace/registered-models/registered-models.go index 5aa6cdf15..63f307a32 100755 --- a/cmd/workspace/registered-models/registered-models.go +++ b/cmd/workspace/registered-models/registered-models.go @@ -135,9 +135,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -509,9 +515,15 @@ func newSetAlias() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setAliasJson.Unmarshal(&setAliasReq) - if err != nil { - return err + diags := setAliasJson.Unmarshal(&setAliasReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setAliasReq.FullName = args[0] @@ -589,9 +601,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/repos/overrides.go b/cmd/workspace/repos/overrides.go index 96d645efb..aad38ecc7 100644 --- a/cmd/workspace/repos/overrides.go +++ b/cmd/workspace/repos/overrides.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -19,7 +20,7 @@ func listOverride(listCmd *cobra.Command, listReq *workspace.ListReposRequest) { {{end}}`) } -func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepo) { +func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepoRequest) { createCmd.Use = "create URL [PROVIDER]" createCmd.Args = func(cmd *cobra.Command, args []string) error { // If the provider argument is not specified, we try to detect it from the URL. @@ -35,9 +36,15 @@ func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepo) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(createReq) - if err != nil { - return err + diags := createJson.Unmarshal(createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { createReq.Url = args[0] @@ -95,17 +102,24 @@ func getOverride(getCmd *cobra.Command, getReq *workspace.GetRepoRequest) { } } -func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepo) { +func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepoRequest) { updateCmd.Use = "update REPO_ID_OR_PATH" updateJson := updateCmd.Flag("json").Value.(*flags.JsonFlag) updateCmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() + var diags diag.Diagnostics w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags = updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { updateReq.RepoId, err = repoArgumentToRepoID(ctx, w, args) diff --git a/cmd/workspace/repos/repos.go b/cmd/workspace/repos/repos.go index fb3d51b06..b77347b06 100755 --- a/cmd/workspace/repos/repos.go +++ b/cmd/workspace/repos/repos.go @@ -61,13 +61,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateRepo, + *workspace.CreateRepoRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateRepo + var createReq workspace.CreateRepoRequest var createJson flags.JsonFlag // TODO: short flags @@ -87,8 +87,9 @@ func newCreate() *cobra.Command { Arguments: URL: URL of the Git repository to be linked. PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -110,9 +111,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -164,7 +171,7 @@ func newDelete() *cobra.Command { Deletes the specified repo. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -181,14 +188,14 @@ func newDelete() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &deleteReq.RepoId) if err != nil { @@ -237,7 +244,7 @@ func newGet() *cobra.Command { Returns the repo with the given repo ID. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -254,14 +261,14 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &getReq.RepoId) if err != nil { @@ -451,8 +458,8 @@ func newList() *cobra.Command { cmd.Short = `Get repos.` cmd.Long = `Get repos. - Returns repos that the calling user has Manage permissions on. Results are - paginated with each page containing twenty repos.` + Returns repos that the calling user has Manage permissions on. Use + next_page_token to iterate through additional pages.` cmd.Annotations = make(map[string]string) @@ -520,9 +527,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -569,13 +582,13 @@ func newSetPermissions() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateRepo, + *workspace.UpdateRepoRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateRepo + var updateReq workspace.UpdateRepoRequest var updateJson flags.JsonFlag // TODO: short flags @@ -593,7 +606,7 @@ func newUpdate() *cobra.Command { latest commit on the same branch. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -603,9 +616,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -616,14 +635,14 @@ func newUpdate() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &updateReq.RepoId) if err != nil { @@ -687,9 +706,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go b/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go index 5e9f59d2c..5d0fba923 100755 --- a/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go +++ b/cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go @@ -197,9 +197,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/schemas/schemas.go b/cmd/workspace/schemas/schemas.go index 3a398251f..3ce573bad 100755 --- a/cmd/workspace/schemas/schemas.go +++ b/cmd/workspace/schemas/schemas.go @@ -100,9 +100,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -386,9 +392,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/secrets/put_secret.go b/cmd/workspace/secrets/put_secret.go index e323c7a10..f24814f05 100644 --- a/cmd/workspace/secrets/put_secret.go +++ b/cmd/workspace/secrets/put_secret.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" @@ -60,6 +61,7 @@ func newPutSecret() *cobra.Command { cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() + var diags diag.Diagnostics w := root.WorkspaceClient(ctx) bytesValueChanged := cmd.Flags().Changed("bytes-value") @@ -69,9 +71,15 @@ func newPutSecret() *cobra.Command { } if cmd.Flags().Changed("json") { - err = putSecretJson.Unmarshal(&putSecretReq) - if err != nil { - return err + diags = putSecretJson.Unmarshal(&putSecretReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { putSecretReq.Scope = args[0] diff --git a/cmd/workspace/secrets/secrets.go b/cmd/workspace/secrets/secrets.go index f836a2670..e9547b621 100755 --- a/cmd/workspace/secrets/secrets.go +++ b/cmd/workspace/secrets/secrets.go @@ -110,9 +110,15 @@ func newCreateScope() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createScopeJson.Unmarshal(&createScopeReq) - if err != nil { - return err + diags := createScopeJson.Unmarshal(&createScopeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -191,9 +197,15 @@ func newDeleteAcl() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteAclJson.Unmarshal(&deleteAclReq) - if err != nil { - return err + diags := deleteAclJson.Unmarshal(&deleteAclReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -273,9 +285,15 @@ func newDeleteScope() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteScopeJson.Unmarshal(&deleteScopeReq) - if err != nil { - return err + diags := deleteScopeJson.Unmarshal(&deleteScopeReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -354,9 +372,15 @@ func newDeleteSecret() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteSecretJson.Unmarshal(&deleteSecretReq) - if err != nil { - return err + diags := deleteSecretJson.Unmarshal(&deleteSecretReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -759,9 +783,15 @@ func newPutAcl() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putAclJson.Unmarshal(&putAclReq) - if err != nil { - return err + diags := putAclJson.Unmarshal(&putAclReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/service-principals/service-principals.go b/cmd/workspace/service-principals/service-principals.go index 957cb1265..317779f30 100755 --- a/cmd/workspace/service-principals/service-principals.go +++ b/cmd/workspace/service-principals/service-principals.go @@ -95,9 +95,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -358,9 +364,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -448,9 +460,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index 0837652db..363e9ea16 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -169,9 +169,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -706,9 +712,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } patchReq.Name = args[0] @@ -777,9 +789,15 @@ func newPut() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putJson.Unmarshal(&putReq) - if err != nil { - return err + diags := putJson.Unmarshal(&putReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } putReq.Name = args[0] @@ -850,9 +868,15 @@ func newPutAiGateway() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = putAiGatewayJson.Unmarshal(&putAiGatewayReq) - if err != nil { - return err + diags := putAiGatewayJson.Unmarshal(&putAiGatewayReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } putAiGatewayReq.Name = args[0] @@ -928,9 +952,15 @@ func newQuery() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryJson.Unmarshal(&queryReq) - if err != nil { - return err + diags := queryJson.Unmarshal(&queryReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } queryReq.Name = args[0] @@ -997,9 +1027,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.ServingEndpointId = args[0] @@ -1076,9 +1112,15 @@ func newUpdateConfig() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateConfigJson.Unmarshal(&updateConfigReq) - if err != nil { - return err + diags := updateConfigJson.Unmarshal(&updateConfigReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateConfigReq.Name = args[0] @@ -1158,9 +1200,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.ServingEndpointId = args[0] diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go index aaeecf41b..31e6ceee4 100755 --- a/cmd/workspace/settings/settings.go +++ b/cmd/workspace/settings/settings.go @@ -9,6 +9,7 @@ import ( compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile" default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace" disable_legacy_access "github.com/databricks/cli/cmd/workspace/disable-legacy-access" + disable_legacy_dbfs "github.com/databricks/cli/cmd/workspace/disable-legacy-dbfs" enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring" restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins" ) @@ -33,6 +34,7 @@ func New() *cobra.Command { cmd.AddCommand(compliance_security_profile.New()) cmd.AddCommand(default_namespace.New()) cmd.AddCommand(disable_legacy_access.New()) + cmd.AddCommand(disable_legacy_dbfs.New()) cmd.AddCommand(enhanced_security_monitoring.New()) cmd.AddCommand(restrict_workspace_admins.New()) diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index 67f870177..62c3407f4 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -100,9 +100,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -427,9 +433,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -501,9 +513,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.Name = args[0] diff --git a/cmd/workspace/storage-credentials/storage-credentials.go b/cmd/workspace/storage-credentials/storage-credentials.go index f4ec5eb4f..2caf09041 100755 --- a/cmd/workspace/storage-credentials/storage-credentials.go +++ b/cmd/workspace/storage-credentials/storage-credentials.go @@ -111,9 +111,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -377,9 +383,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -478,9 +490,15 @@ func newValidate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = validateJson.Unmarshal(&validateReq) - if err != nil { - return err + diags := validateJson.Unmarshal(&validateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/table-constraints/table-constraints.go b/cmd/workspace/table-constraints/table-constraints.go index 166da146c..4ac7cb9fb 100755 --- a/cmd/workspace/table-constraints/table-constraints.go +++ b/cmd/workspace/table-constraints/table-constraints.go @@ -92,9 +92,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/tables/tables.go b/cmd/workspace/tables/tables.go index ec297f294..35775f17f 100755 --- a/cmd/workspace/tables/tables.go +++ b/cmd/workspace/tables/tables.go @@ -479,9 +479,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go index 8718f7ba1..210a59f8e 100755 --- a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go +++ b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go @@ -94,9 +94,15 @@ func newGenerateTemporaryTableCredentials() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) - if err != nil { - return err + diags := generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/token-management/token-management.go b/cmd/workspace/token-management/token-management.go index dea94edb0..6deb8d125 100755 --- a/cmd/workspace/token-management/token-management.go +++ b/cmd/workspace/token-management/token-management.go @@ -96,9 +96,15 @@ func newCreateOboToken() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createOboTokenJson.Unmarshal(&createOboTokenReq) - if err != nil { - return err + diags := createOboTokenJson.Unmarshal(&createOboTokenReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -458,9 +464,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -523,9 +535,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/tokens/tokens.go b/cmd/workspace/tokens/tokens.go index afe4b9a03..5c9b4994d 100755 --- a/cmd/workspace/tokens/tokens.go +++ b/cmd/workspace/tokens/tokens.go @@ -84,9 +84,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -158,9 +164,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { diff --git a/cmd/workspace/users/users.go b/cmd/workspace/users/users.go index 53ba2e85c..b085ab413 100755 --- a/cmd/workspace/users/users.go +++ b/cmd/workspace/users/users.go @@ -107,9 +107,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -463,9 +469,15 @@ func newPatch() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = patchJson.Unmarshal(&patchReq) - if err != nil { - return err + diags := patchJson.Unmarshal(&patchReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -546,9 +558,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -619,9 +637,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -702,9 +726,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } diff --git a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go index dd9d57835..0cfb76172 100755 --- a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go +++ b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go @@ -95,9 +95,15 @@ func newCreateEndpoint() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createEndpointJson.Unmarshal(&createEndpointReq) - if err != nil { - return err + diags := createEndpointJson.Unmarshal(&createEndpointReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go index 158474770..832f4a6dc 100755 --- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go +++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go @@ -114,9 +114,15 @@ func newCreateIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createIndexJson.Unmarshal(&createIndexReq) - if err != nil { - return err + diags := createIndexJson.Unmarshal(&createIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -195,9 +201,15 @@ func newDeleteDataVectorIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteDataVectorIndexJson.Unmarshal(&deleteDataVectorIndexReq) - if err != nil { - return err + diags := deleteDataVectorIndexJson.Unmarshal(&deleteDataVectorIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -443,9 +455,15 @@ func newQueryIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryIndexJson.Unmarshal(&queryIndexReq) - if err != nil { - return err + diags := queryIndexJson.Unmarshal(&queryIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") @@ -515,9 +533,15 @@ func newQueryNextPage() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = queryNextPageJson.Unmarshal(&queryNextPageReq) - if err != nil { - return err + diags := queryNextPageJson.Unmarshal(&queryNextPageReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } queryNextPageReq.IndexName = args[0] @@ -585,9 +609,15 @@ func newScanIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = scanIndexJson.Unmarshal(&scanIndexReq) - if err != nil { - return err + diags := scanIndexJson.Unmarshal(&scanIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } scanIndexReq.IndexName = args[0] @@ -718,9 +748,15 @@ func newUpsertDataVectorIndex() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = upsertDataVectorIndexJson.Unmarshal(&upsertDataVectorIndexReq) - if err != nil { - return err + diags := upsertDataVectorIndexJson.Unmarshal(&upsertDataVectorIndexReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } upsertDataVectorIndexReq.IndexName = args[0] diff --git a/cmd/workspace/volumes/volumes.go b/cmd/workspace/volumes/volumes.go index 3fc1f447b..2f4555736 100755 --- a/cmd/workspace/volumes/volumes.go +++ b/cmd/workspace/volumes/volumes.go @@ -119,9 +119,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -427,9 +433,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/warehouses/warehouses.go b/cmd/workspace/warehouses/warehouses.go index cdf106365..43d6c8ab9 100755 --- a/cmd/workspace/warehouses/warehouses.go +++ b/cmd/workspace/warehouses/warehouses.go @@ -109,9 +109,15 @@ func newCreate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&createReq) - if err != nil { - return err + diags := createJson.Unmarshal(&createReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -277,9 +283,15 @@ func newEdit() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = editJson.Unmarshal(&editReq) - if err != nil { - return err + diags := editJson.Unmarshal(&editReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -688,9 +700,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { @@ -779,9 +797,15 @@ func newSetWorkspaceWarehouseConfig() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setWorkspaceWarehouseConfigJson.Unmarshal(&setWorkspaceWarehouseConfigReq) - if err != nil { - return err + diags := setWorkspaceWarehouseConfigJson.Unmarshal(&setWorkspaceWarehouseConfigReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } @@ -1030,9 +1054,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if len(args) == 0 { diff --git a/cmd/workspace/workspace-bindings/workspace-bindings.go b/cmd/workspace/workspace-bindings/workspace-bindings.go index 4993f1aff..20f54e1dc 100755 --- a/cmd/workspace/workspace-bindings/workspace-bindings.go +++ b/cmd/workspace/workspace-bindings/workspace-bindings.go @@ -226,9 +226,15 @@ func newUpdate() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateJson.Unmarshal(&updateReq) - if err != nil { - return err + diags := updateJson.Unmarshal(&updateReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updateReq.Name = args[0] @@ -297,9 +303,15 @@ func newUpdateBindings() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updateBindingsJson.Unmarshal(&updateBindingsReq) - if err != nil { - return err + diags := updateBindingsJson.Unmarshal(&updateBindingsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } _, err = fmt.Sscan(args[0], &updateBindingsReq.SecurableType) diff --git a/cmd/workspace/workspace-conf/workspace-conf.go b/cmd/workspace/workspace-conf/workspace-conf.go index 92b2f0f3a..a17bc1630 100755 --- a/cmd/workspace/workspace-conf/workspace-conf.go +++ b/cmd/workspace/workspace-conf/workspace-conf.go @@ -127,9 +127,15 @@ func newSetStatus() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setStatusJson.Unmarshal(&setStatusReq) - if err != nil { - return err + diags := setStatusJson.Unmarshal(&setStatusReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") diff --git a/cmd/workspace/workspace/workspace.go b/cmd/workspace/workspace/workspace.go index 183cac898..21da478c4 100755 --- a/cmd/workspace/workspace/workspace.go +++ b/cmd/workspace/workspace/workspace.go @@ -106,9 +106,15 @@ func newDelete() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = deleteJson.Unmarshal(&deleteReq) - if err != nil { - return err + diags := deleteJson.Unmarshal(&deleteReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -482,9 +488,15 @@ func newImport() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = importJson.Unmarshal(&importReq) - if err != nil { - return err + diags := importJson.Unmarshal(&importReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } if !cmd.Flags().Changed("json") { @@ -622,9 +634,15 @@ func newMkdirs() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = mkdirsJson.Unmarshal(&mkdirsReq) - if err != nil { - return err + diags := mkdirsJson.Unmarshal(&mkdirsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } else { if len(args) == 0 { @@ -710,9 +728,15 @@ func newSetPermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = setPermissionsJson.Unmarshal(&setPermissionsReq) - if err != nil { - return err + diags := setPermissionsJson.Unmarshal(&setPermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } setPermissionsReq.WorkspaceObjectType = args[0] @@ -781,9 +805,15 @@ func newUpdatePermissions() *cobra.Command { w := root.WorkspaceClient(ctx) if cmd.Flags().Changed("json") { - err = updatePermissionsJson.Unmarshal(&updatePermissionsReq) - if err != nil { - return err + diags := updatePermissionsJson.Unmarshal(&updatePermissionsReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnosticsToErrorOut(ctx, diags) + if err != nil { + return err + } } } updatePermissionsReq.WorkspaceObjectType = args[0] diff --git a/go.mod b/go.mod index 9141274c2..d8679fd60 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0 - github.com/fatih/color v1.17.0 // MIT + github.com/databricks/databricks-sdk-go v0.49.0 // Apache 2.0 + github.com/fatih/color v1.18.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 @@ -27,8 +27,8 @@ require ( golang.org/x/mod v0.21.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.24.0 - golang.org/x/text v0.18.0 + golang.org/x/term v0.25.0 + golang.org/x/text v0.19.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -64,7 +64,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 177707a50..c47ae7693 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ= -github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.49.0 h1:VBTeZZMLIuBSM4kxOCfUcW9z4FUQZY2QeNRD5qm9FUQ= +github.com/databricks/databricks-sdk-go v0.49.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,8 +44,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -212,14 +212,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/acc/workspace.go b/internal/acc/workspace.go index 39374f229..69ab0e715 100644 --- a/internal/acc/workspace.go +++ b/internal/acc/workspace.go @@ -2,11 +2,14 @@ package acc import ( "context" + "fmt" "os" "testing" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/require" ) @@ -94,3 +97,30 @@ func (t *WorkspaceT) RunPython(code string) (string, error) { require.True(t, ok, "unexpected type %T", results.Data) return output, nil } + +func (t *WorkspaceT) TemporaryWorkspaceDir(name ...string) string { + ctx := context.Background() + me, err := t.W.CurrentUser.Me(ctx) + require.NoError(t, err) + + basePath := fmt.Sprintf("/Users/%s/%s", me.UserName, RandomName(name...)) + + t.Logf("Creating %s", basePath) + err = t.W.Workspace.MkdirsByPath(ctx, basePath) + require.NoError(t, err) + + // Remove test directory on test completion. + t.Cleanup(func() { + t.Logf("Removing %s", basePath) + err := t.W.Workspace.Delete(ctx, workspace.Delete{ + Path: basePath, + Recursive: true, + }) + if err == nil || apierr.IsMissing(err) { + return + } + t.Logf("Unable to remove temporary workspace directory %s: %#v", basePath, err) + }) + + return basePath +} diff --git a/internal/bugbash/README.md b/internal/bugbash/README.md new file mode 100644 index 000000000..941ab6227 --- /dev/null +++ b/internal/bugbash/README.md @@ -0,0 +1,13 @@ +# Bugbash + +The script in this directory can be used to conveniently exec into a shell +where a CLI build for a specific branch is made available. + +## Usage + +This script prompts if you do NOT have at least Bash 5 installed, +but works without command completion with earlier versions. + +```shell +bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch +``` diff --git a/internal/bugbash/exec.sh b/internal/bugbash/exec.sh new file mode 100755 index 000000000..ac25b16ed --- /dev/null +++ b/internal/bugbash/exec.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Set the GitHub repository for the Databricks CLI. +export GH_REPO="databricks/cli" + +# Synthesize the directory name for the snapshot build. +function cli_snapshot_directory() { + dir="cli" + + # Append OS + case "$(uname -s)" in + Linux) + dir="${dir}_linux" + ;; + Darwin) + dir="${dir}_darwin" + ;; + *) + echo "Unknown operating system: $os" + ;; + esac + + # Append architecture + case "$(uname -m)" in + x86_64) + dir="${dir}_amd64_v1" + ;; + i386|i686) + dir="${dir}_386" + ;; + arm64|aarch64) + dir="${dir}_arm64" + ;; + armv7l|armv8l) + dir="${dir}_arm_6" + ;; + *) + echo "Unknown architecture: $arch" + ;; + esac + + echo $dir +} + +BRANCH=$1 +shift + +# Default to main branch if branch is not specified. +if [ -z "$BRANCH" ]; then + BRANCH=main +fi + +if [ -z "$BRANCH" ]; then + echo "Please specify which branch to bugbash..." + exit 1 +fi + +# Check if the "gh" command is available. +if ! command -v gh &> /dev/null; then + echo "The GitHub CLI (gh) is required to download the snapshot build." + echo "Install and configure it with:" + echo "" + echo " brew install gh" + echo " gh auth login" + echo "" + exit 1 +fi + +echo "Looking for a snapshot build of the Databricks CLI on branch $BRANCH..." + +# Find last successful build on $BRANCH. +last_successful_run_id=$( + gh run list -b "$BRANCH" -w release-snapshot --json 'databaseId,conclusion' | + jq 'limit(1; .[] | select(.conclusion == "success")) | .databaseId' +) +if [ -z "$last_successful_run_id" ]; then + echo "Unable to find last successful build of the release-snapshot workflow for branch $BRANCH." + exit 1 +fi + +# Determine artifact name with the right binaries for this runner. +case "$(uname -s)" in +Linux) + artifact="cli_linux_snapshot" + ;; +Darwin) + artifact="cli_darwin_snapshot" + ;; +esac + +# Create a temporary directory to download the artifact. +dir=$(mktemp -d) + +# Download the artifact. +echo "Downloading the snapshot build..." +gh run download "$last_successful_run_id" -n "$artifact" -D "$dir/.bin" +dir="$dir/.bin/$(cli_snapshot_directory)" +if [ ! -d "$dir" ]; then + echo "Directory does not exist: $dir" + exit 1 +fi + +# Make CLI available on $PATH. +chmod +x "$dir/databricks" +export PATH="$dir:$PATH" + +# Set the prompt to indicate the bugbash environment and exec. +export PS1="(bugbash $BRANCH) \[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ " + +# Display completion instructions. +echo "" +echo "==================================================================" + +if [[ ${BASH_VERSINFO[0]} -lt 5 ]]; then + echo -en "\033[31m" + echo "You have Bash version < 5 installed... completion won't work." + echo -en "\033[0m" + echo "" + echo "Install it with:" + echo "" + echo " brew install bash bash-completion" + echo "" + echo "==================================================================" +fi + +echo "" +echo "To load completions in your current shell session:" +echo "" +echo " source /opt/homebrew/etc/profile.d/bash_completion.sh" +echo " source <(databricks completion bash)" +echo "" +echo "==================================================================" +echo "" + +# Exec into a new shell. +# Note: don't use zsh because on macOS it _always_ overwrites PS1. +exec /usr/bin/env bash diff --git a/internal/build/sdk_consistency_test.go b/internal/build/sdk_consistency_test.go new file mode 100644 index 000000000..37f60250c --- /dev/null +++ b/internal/build/sdk_consistency_test.go @@ -0,0 +1,73 @@ +package build + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/mod/modfile" +) + +// This test ensures that the OpenAPI SHA the CLI is being generated from matches +// the OpenAPI SHA of the Go SDK version used in the CLI. We should always upgrade +// the Go SDK version before generating the CLI because downstream generated assets +// like the bundle schema depend on the Go SDK itself. +func TestConsistentDatabricksSdkVersion(t *testing.T) { + // Read the go.mod file + b, err := os.ReadFile("../../go.mod") + require.NoError(t, err) + + // Parse the go.mod file to get the databricks-sdk version + modFile, err := modfile.Parse("../../go.mod", b, nil) + require.NoError(t, err) + + modulePath := "github.com/databricks/databricks-sdk-go" + var version string + for _, r := range modFile.Require { + if r.Mod.Path == modulePath { + version = r.Mod.Version + } + } + require.NotEmpty(t, version) + + // Full path of the package. For example: github.com/databricks/databricks-sdk-go@v0.47.1-0.20241002195128-6cecc224cbf7 + fullPath := fmt.Sprintf("%s@%s", modulePath, version) + + type goListResponse struct { + Origin struct { + Hash string + } + } + + // Using the go CLI query for the git hash corresponding to the databricks-sdk-go version + cmd := exec.Command("go", "list", "-m", "-json", "-mod=readonly", fullPath) + out, err := cmd.Output() + require.NoError(t, err) + parsedOutput := new(goListResponse) + err = json.Unmarshal(out, parsedOutput) + require.NoError(t, err) + hash := parsedOutput.Origin.Hash + require.NotEmpty(t, hash) + + // Read the OpenAPI SHA from the Go SDK. + url := fmt.Sprintf("https://raw.githubusercontent.com/databricks/databricks-sdk-go/%s/.codegen/_openapi_sha", hash) + resp, err := http.Get(url) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + sdkSha, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + cliSha, err := os.ReadFile("../../.codegen/_openapi_sha") + require.NoError(t, err) + + assert.Equal(t, strings.TrimSpace(string(cliSha)), strings.TrimSpace(string(sdkSha)), "please update the SDK version before generating the CLI") +} diff --git a/internal/bundle/bundles/dashboards/databricks_template_schema.json b/internal/bundle/bundles/dashboards/databricks_template_schema.json new file mode 100644 index 000000000..1aa5728fc --- /dev/null +++ b/internal/bundle/bundles/dashboards/databricks_template_schema.json @@ -0,0 +1,12 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "warehouse_id": { + "type": "string", + "description": "The SQL warehouse ID to use for the dashboard" + } + } +} diff --git a/internal/bundle/bundles/dashboards/template/dashboard.lvdash.json b/internal/bundle/bundles/dashboards/template/dashboard.lvdash.json new file mode 100644 index 000000000..397a9a125 --- /dev/null +++ b/internal/bundle/bundles/dashboards/template/dashboard.lvdash.json @@ -0,0 +1,34 @@ +{ + "pages": [ + { + "displayName": "New Page", + "layout": [ + { + "position": { + "height": 2, + "width": 6, + "x": 0, + "y": 0 + }, + "widget": { + "name": "82eb9107", + "textbox_spec": "# I'm a title" + } + }, + { + "position": { + "height": 2, + "width": 6, + "x": 0, + "y": 2 + }, + "widget": { + "name": "ffa6de4f", + "textbox_spec": "Text" + } + } + ], + "name": "fdd21a3c" + } + ] +} diff --git a/internal/bundle/bundles/dashboards/template/databricks.yml.tmpl b/internal/bundle/bundles/dashboards/template/databricks.yml.tmpl new file mode 100644 index 000000000..e77712381 --- /dev/null +++ b/internal/bundle/bundles/dashboards/template/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dashboards + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +resources: + dashboards: + file_reference: + display_name: test-dashboard-{{.unique_id}} + file_path: ./dashboard.lvdash.json + warehouse_id: {{.warehouse_id}} diff --git a/internal/bundle/dashboards_test.go b/internal/bundle/dashboards_test.go new file mode 100644 index 000000000..b12cc040c --- /dev/null +++ b/internal/bundle/dashboards_test.go @@ -0,0 +1,63 @@ +package bundle + +import ( + "fmt" + "testing" + + "github.com/databricks/cli/internal/acc" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAccDashboards(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + warehouseID := acc.GetEnvOrSkipTest(t, "TEST_DEFAULT_WAREHOUSE_ID") + uniqueID := uuid.New().String() + root, err := initTestTemplate(t, ctx, "dashboards", map[string]any{ + "unique_id": uniqueID, + "warehouse_id": warehouseID, + }) + require.NoError(t, err) + + t.Cleanup(func() { + err = destroyBundle(t, ctx, root) + require.NoError(t, err) + }) + + err = deployBundle(t, ctx, root) + require.NoError(t, err) + + // Load bundle configuration by running the validate command. + b := unmarshalConfig(t, mustValidateBundle(t, ctx, root)) + + // Assert that the dashboard exists at the expected path and is, indeed, a dashboard. + oi, err := wt.W.Workspace.GetStatusByPath(ctx, fmt.Sprintf("%s/test-dashboard-%s.lvdash.json", b.Config.Workspace.ResourcePath, uniqueID)) + require.NoError(t, err) + assert.EqualValues(t, workspace.ObjectTypeDashboard, oi.ObjectType) + + // Load the dashboard by its ID and confirm its display name. + dashboard, err := wt.W.Lakeview.GetByDashboardId(ctx, oi.ResourceId) + require.NoError(t, err) + assert.Equal(t, fmt.Sprintf("test-dashboard-%s", uniqueID), dashboard.DisplayName) + + // Make an out of band modification to the dashboard and confirm that it is detected. + _, err = wt.W.Lakeview.Update(ctx, dashboards.UpdateDashboardRequest{ + DashboardId: oi.ResourceId, + SerializedDashboard: dashboard.SerializedDashboard, + }) + require.NoError(t, err) + + // Try to redeploy the bundle and confirm that the out of band modification is detected. + stdout, _, err := deployBundleWithArgs(t, ctx, root) + require.Error(t, err) + assert.Contains(t, stdout, `Error: dashboard "file_reference" has been modified remotely`+"\n") + + // Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored. + _, stderr, err := deployBundleWithArgs(t, ctx, root, "--force") + require.NoError(t, err) + assert.Contains(t, stderr, `Deployment complete!`+"\n") +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index b8c81a8d2..8f1a866f6 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal" "github.com/databricks/cli/libs/cmdio" @@ -66,6 +67,19 @@ func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, err return stdout.Bytes(), err } +func mustValidateBundle(t *testing.T, ctx context.Context, path string) []byte { + data, err := validateBundle(t, ctx, path) + require.NoError(t, err) + return data +} + +func unmarshalConfig(t *testing.T, data []byte) *bundle.Bundle { + bundle := &bundle.Bundle{} + err := json.Unmarshal(data, &bundle.Config) + require.NoError(t, err) + return bundle +} + func deployBundle(t *testing.T, ctx context.Context, path string) error { ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve") @@ -73,6 +87,14 @@ func deployBundle(t *testing.T, ctx context.Context, path string) error { return err } +func deployBundleWithArgs(t *testing.T, ctx context.Context, path string, args ...string) (string, string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) + args = append([]string{"bundle", "deploy"}, args...) + c := internal.NewCobraTestRunnerWithContext(t, ctx, args...) + stdout, stderr, err := c.Run() + return stdout.String(), stderr.String(), err +} + func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error { ctx = env.Set(ctx, "BUNDLE_ROOT", path) args := []string{"bundle", "deploy", "--force-lock"} diff --git a/internal/dashboard_assumptions_test.go b/internal/dashboard_assumptions_test.go new file mode 100644 index 000000000..912e046b5 --- /dev/null +++ b/internal/dashboard_assumptions_test.go @@ -0,0 +1,110 @@ +package internal + +import ( + "encoding/base64" + "testing" + + "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/merge" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Verify that importing a dashboard through the Workspace API retains the identity of the underying resource, +// as well as properties exclusively accessible through the dashboards API. +func TestAccDashboardAssumptions_WorkspaceImport(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + t.Parallel() + + dashboardName := "New Dashboard" + dashboardPayload := []byte(`{"pages":[{"name":"2506f97a","displayName":"New Page"}]}`) + warehouseId := acc.GetEnvOrSkipTest(t, "TEST_DEFAULT_WAREHOUSE_ID") + + dir := wt.TemporaryWorkspaceDir("dashboard-assumptions-") + + dashboard, err := wt.W.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ + DisplayName: dashboardName, + ParentPath: dir, + SerializedDashboard: string(dashboardPayload), + WarehouseId: warehouseId, + }) + require.NoError(t, err) + t.Logf("Dashboard ID (per Lakeview API): %s", dashboard.DashboardId) + + // Overwrite the dashboard via the workspace API. + { + err := wt.W.Workspace.Import(ctx, workspace.Import{ + Format: workspace.ImportFormatAuto, + Path: dashboard.Path, + Content: base64.StdEncoding.EncodeToString(dashboardPayload), + Overwrite: true, + }) + require.NoError(t, err) + } + + // Cross-check consistency with the workspace object. + { + obj, err := wt.W.Workspace.GetStatusByPath(ctx, dashboard.Path) + require.NoError(t, err) + + // Confirm that the resource ID included in the response is equal to the dashboard ID. + require.Equal(t, dashboard.DashboardId, obj.ResourceId) + t.Logf("Dashboard ID (per workspace object status): %s", obj.ResourceId) + } + + // Try to overwrite the dashboard via the Lakeview API (and expect failure). + { + _, err := wt.W.Lakeview.Create(ctx, dashboards.CreateDashboardRequest{ + DisplayName: dashboardName, + ParentPath: dir, + SerializedDashboard: string(dashboardPayload), + }) + require.ErrorIs(t, err, apierr.ErrResourceAlreadyExists) + } + + // Retrieve the dashboard object and confirm that only select fields were updated by the import. + { + previousDashboard := dashboard + currentDashboard, err := wt.W.Lakeview.Get(ctx, dashboards.GetDashboardRequest{ + DashboardId: dashboard.DashboardId, + }) + require.NoError(t, err) + + // Convert the dashboard object to a [dyn.Value] to make comparison easier. + previous, err := convert.FromTyped(previousDashboard, dyn.NilValue) + require.NoError(t, err) + current, err := convert.FromTyped(currentDashboard, dyn.NilValue) + require.NoError(t, err) + + // Collect updated paths. + var updatedFieldPaths []string + _, err = merge.Override(previous, current, merge.OverrideVisitor{ + VisitDelete: func(basePath dyn.Path, left dyn.Value) error { + assert.Fail(t, "unexpected delete operation") + return nil + }, + VisitInsert: func(basePath dyn.Path, right dyn.Value) (dyn.Value, error) { + assert.Fail(t, "unexpected insert operation") + return right, nil + }, + VisitUpdate: func(basePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) { + updatedFieldPaths = append(updatedFieldPaths, basePath.String()) + return right, nil + }, + }) + require.NoError(t, err) + + // Confirm that only the expected fields have been updated. + assert.ElementsMatch(t, []string{ + "etag", + "update_time", + }, updatedFieldPaths) + } +} diff --git a/internal/fs_mkdir_test.go b/internal/fs_mkdir_test.go index dd75c7c32..9191f6143 100644 --- a/internal/fs_mkdir_test.go +++ b/internal/fs_mkdir_test.go @@ -112,8 +112,8 @@ func TestAccFsMkdirWhenFileExistsAtPath(t *testing.T) { // assert mkdir fails _, _, err = RequireErrorRun(t, "fs", "mkdir", path.Join(tmpDir, "hello")) - // Different cloud providers return different errors. - regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$`) + // Different cloud providers or cloud configurations return different errors. + regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$|(^|: )"The specified path already exists.".*$`) assert.Regexp(t, regex, err.Error()) }) diff --git a/internal/helpers.go b/internal/helpers.go index 419fa419c..3bf387757 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -20,6 +20,7 @@ import ( "time" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/acc" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/cmd" @@ -519,7 +520,7 @@ func TemporaryRepo(t *testing.T, w *databricks.WorkspaceClient) string { repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("integration-test-repo-")) t.Logf("Creating repo:%s", repoPath) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Url: "https://github.com/databricks/cli", Provider: "github", Path: repoPath, @@ -591,13 +592,10 @@ func setupWsfsExtensionsFiler(t *testing.T) (filer.Filer, string) { } func setupDbfsFiler(t *testing.T) (filer.Filer, string) { - t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + _, wt := acc.WorkspaceTest(t) - w, err := databricks.NewWorkspaceClient() - require.NoError(t, err) - - tmpDir := TemporaryDbfsDir(t, w) - f, err := filer.NewDbfsClient(w, tmpDir) + tmpDir := TemporaryDbfsDir(t, wt.W) + f, err := filer.NewDbfsClient(wt.W, tmpDir) require.NoError(t, err) return f, path.Join("dbfs:/", tmpDir) diff --git a/internal/init_test.go b/internal/init_test.go index d1a89f7b7..a6241d629 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -12,7 +12,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -126,7 +126,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }{ { funcName: "{{short_name}}", - expected: auth.GetShortUserName(me), + expected: iamutil.GetShortUserName(me), }, { funcName: "{{user_name}}", @@ -138,7 +138,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }, { funcName: "{{is_service_principal}}", - expected: strconv.FormatBool(auth.IsServicePrincipal(me.UserName)), + expected: strconv.FormatBool(iamutil.IsServicePrincipal(me)), }, { funcName: "{{smallest_node_type}}", diff --git a/internal/locker_test.go b/internal/locker_test.go index 21e08f732..3ae783d1b 100644 --- a/internal/locker_test.go +++ b/internal/locker_test.go @@ -29,7 +29,7 @@ func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databr assert.NoError(t, err) remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix)) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: remoteProjectRoot, Url: EmptyRepoUrl, Provider: "gitHub", diff --git a/internal/repos_test.go b/internal/repos_test.go index de0d926ad..1ad0e8775 100644 --- a/internal/repos_test.go +++ b/internal/repos_test.go @@ -34,7 +34,7 @@ func synthesizeTemporaryRepoPath(t *testing.T, w *databricks.WorkspaceClient, ct func createTemporaryRepo(t *testing.T, w *databricks.WorkspaceClient, ctx context.Context) (int64, string) { repoPath := synthesizeTemporaryRepoPath(t, w, ctx) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", diff --git a/internal/sync_test.go b/internal/sync_test.go index 4021e6490..6f8b1827b 100644 --- a/internal/sync_test.go +++ b/internal/sync_test.go @@ -38,7 +38,7 @@ func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Contex require.NoError(t, err) repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-")) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", diff --git a/libs/cmdio/render.go b/libs/cmdio/render.go index 4114db5ca..72d95978a 100644 --- a/libs/cmdio/render.go +++ b/libs/cmdio/render.go @@ -13,6 +13,7 @@ import ( "text/template" "time" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/flags" "github.com/databricks/databricks-sdk-go/listing" "github.com/fatih/color" @@ -285,70 +286,72 @@ func RenderIteratorJson[T any](ctx context.Context, i listing.Iterator[T]) error return renderWithTemplate(newIteratorRenderer(i), ctx, c.outputFormat, c.out, c.headerTemplate, c.template) } +var renderFuncMap = template.FuncMap{ + // we render colored output if stdout is TTY, otherwise we render text. + // in the future we'll check if we can explicitly check for stderr being + // a TTY + "header": color.BlueString, + "red": color.RedString, + "green": color.GreenString, + "blue": color.BlueString, + "yellow": color.YellowString, + "magenta": color.MagentaString, + "cyan": color.CyanString, + "bold": func(format string, a ...interface{}) string { + return color.New(color.Bold).Sprintf(format, a...) + }, + "italic": func(format string, a ...interface{}) string { + return color.New(color.Italic).Sprintf(format, a...) + }, + "replace": strings.ReplaceAll, + "join": strings.Join, + "bool": func(v bool) string { + if v { + return color.GreenString("YES") + } + return color.RedString("NO") + }, + "pretty_json": func(in string) (string, error) { + var tmp any + err := json.Unmarshal([]byte(in), &tmp) + if err != nil { + return "", err + } + b, err := fancyJSON(tmp) + if err != nil { + return "", err + } + return string(b), nil + }, + "pretty_date": func(t time.Time) string { + return t.Format("2006-01-02T15:04:05Z") + }, + "b64_encode": func(in string) (string, error) { + var out bytes.Buffer + enc := base64.NewEncoder(base64.StdEncoding, &out) + _, err := enc.Write([]byte(in)) + if err != nil { + return "", err + } + err = enc.Close() + if err != nil { + return "", err + } + return out.String(), nil + }, + "b64_decode": func(in string) (string, error) { + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(in)) + out, err := io.ReadAll(dec) + if err != nil { + return "", err + } + return string(out), nil + }, +} + func renderUsingTemplate(ctx context.Context, r templateRenderer, w io.Writer, headerTmpl, tmpl string) error { tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - base := template.New("command").Funcs(template.FuncMap{ - // we render colored output if stdout is TTY, otherwise we render text. - // in the future we'll check if we can explicitly check for stderr being - // a TTY - "header": color.BlueString, - "red": color.RedString, - "green": color.GreenString, - "blue": color.BlueString, - "yellow": color.YellowString, - "magenta": color.MagentaString, - "cyan": color.CyanString, - "bold": func(format string, a ...interface{}) string { - return color.New(color.Bold).Sprintf(format, a...) - }, - "italic": func(format string, a ...interface{}) string { - return color.New(color.Italic).Sprintf(format, a...) - }, - "replace": strings.ReplaceAll, - "join": strings.Join, - "bool": func(v bool) string { - if v { - return color.GreenString("YES") - } - return color.RedString("NO") - }, - "pretty_json": func(in string) (string, error) { - var tmp any - err := json.Unmarshal([]byte(in), &tmp) - if err != nil { - return "", err - } - b, err := fancyJSON(tmp) - if err != nil { - return "", err - } - return string(b), nil - }, - "pretty_date": func(t time.Time) string { - return t.Format("2006-01-02T15:04:05Z") - }, - "b64_encode": func(in string) (string, error) { - var out bytes.Buffer - enc := base64.NewEncoder(base64.StdEncoding, &out) - _, err := enc.Write([]byte(in)) - if err != nil { - return "", err - } - err = enc.Close() - if err != nil { - return "", err - } - return out.String(), nil - }, - "b64_decode": func(in string) (string, error) { - dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(in)) - out, err := io.ReadAll(dec) - if err != nil { - return "", err - } - return string(out), nil - }, - }) + base := template.New("command").Funcs(renderFuncMap) if headerTmpl != "" { headerT, err := base.Parse(headerTmpl) if err != nil { @@ -388,3 +391,77 @@ func fancyJSON(v any) ([]byte, error) { return jsoncolor.MarshalIndentWithFormatter(v, "", " ", f) } + +const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + +func RenderDiagnosticsToErrorOut(ctx context.Context, diags diag.Diagnostics) error { + c := fromContext(ctx) + return RenderDiagnostics(c.err, diags) +} + +func RenderDiagnostics(out io.Writer, diags diag.Diagnostics) error { + errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) + warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) + recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate)) + + // Print errors and warnings. + for _, d := range diags { + var t *template.Template + switch d.Severity { + case diag.Error: + t = errorT + case diag.Warning: + t = warningT + case diag.Recommendation: + t = recommendationT + } + + // Render the diagnostic with the appropriate template. + err := t.Execute(out, d) + if err != nil { + return fmt.Errorf("failed to render template: %w", err) + } + } + + return nil +} diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index 93334c067..254ecbd7d 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -1,6 +1,7 @@ package diag import ( + "errors" "fmt" "github.com/databricks/cli/libs/dyn" @@ -24,6 +25,9 @@ type Diagnostic struct { // Paths are paths to the values in the configuration tree that the diagnostic is associated with. // It may be nil if there are no associated paths. Paths []dyn.Path + + // A diagnostic ID. Only used for select diagnostic messages. + ID ID } // Errorf creates a new error diagnostic. @@ -69,7 +73,7 @@ func Infof(format string, args ...any) Diagnostics { } } -// Diagsnostics holds zero or more instances of [Diagnostic]. +// Diagnostics holds zero or more instances of [Diagnostic]. type Diagnostics []Diagnostic // Append adds a new diagnostic to the end of the list. @@ -96,7 +100,14 @@ func (ds Diagnostics) HasError() bool { func (ds Diagnostics) Error() error { for _, d := range ds { if d.Severity == Error { - return fmt.Errorf(d.Summary) + message := d.Detail + if message == "" { + message = d.Summary + } + if d.ID != "" { + message = string(d.ID) + ": " + message + } + return errors.New(message) } } return nil diff --git a/libs/diag/id.go b/libs/diag/id.go new file mode 100644 index 000000000..44af3c932 --- /dev/null +++ b/libs/diag/id.go @@ -0,0 +1,16 @@ +package diag + +type ID string + +// For select diagnostic messages we use IDs to identify them +// for support or tooling purposes. +// It is a non-goal to have an exhaustive list of IDs. +const ( + // We have many subtly different permission errors. + // These are numbered for easy reference and tooling support. + PathPermissionDenied ID = "EPERM1" + ResourcePermissionDenied ID = "EPERM2" + CannotChangePathPermissions ID = "EPERM3" + RunAsDenied ID = "EPERM4" + PermissionNotIncluded ID = "EPERM5" +) diff --git a/libs/diag/severity.go b/libs/diag/severity.go index d25c12806..0e88085f5 100644 --- a/libs/diag/severity.go +++ b/libs/diag/severity.go @@ -6,4 +6,5 @@ const ( Error Severity = iota Warning Info + Recommendation ) diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index bc80a150a..106add35d 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -398,6 +398,34 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d return dyn.NewValue(out, src.Locations()), diags } -func (n normalizeOptions) normalizeInterface(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { +func (n normalizeOptions) normalizeInterface(_ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { + // Deal with every [dyn.Kind] here to ensure completeness. + switch src.Kind() { + case dyn.KindMap: + // Fall through + case dyn.KindSequence: + // Fall through + case dyn.KindString: + // Fall through + case dyn.KindBool: + // Fall through + case dyn.KindInt: + // Fall through + case dyn.KindFloat: + // Fall through + case dyn.KindTime: + // Conversion of a time value to an interface{}. + // The [dyn.Value.AsAny] equivalent for this kind is the [time.Time] struct. + // If we convert to a typed representation and back again, we cannot distinguish + // a [time.Time] struct from any other struct. + // + // Therefore, we normalize the time value to a string. + return dyn.NewValue(src.MustTime().String(), src.Locations()), nil + case dyn.KindNil: + // Fall through + default: + return dyn.InvalidValue, diag.Errorf("unsupported kind: %s", src.Kind()) + } + return src, nil } diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 4b2a3c189..ab0a1cec1 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -858,23 +858,7 @@ func TestNormalizeAnchors(t *testing.T) { }, vout.AsAny()) } -func TestNormalizeBoolToAny(t *testing.T) { - var typ any - vin := dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeIntToAny(t *testing.T) { - var typ any - vin := dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}) - vout, err := Normalize(&typ, vin) - assert.Len(t, err, 0) - assert.Equal(t, dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) -} - -func TestNormalizeSliceToAny(t *testing.T) { +func TestNormalizeAnyFromSlice(t *testing.T) { var typ any v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) v2 := dyn.NewValue(2, []dyn.Location{{File: "file", Line: 1, Column: 1}}) @@ -883,3 +867,35 @@ func TestNormalizeSliceToAny(t *testing.T) { assert.Len(t, err, 0) assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) } + +func TestNormalizeAnyFromString(t *testing.T) { + var typ any + vin := dyn.NewValue("string", []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vout, err := Normalize(&typ, vin) + assert.Len(t, err, 0) + assert.Equal(t, dyn.NewValue("string", []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) +} + +func TestNormalizeAnyFromBool(t *testing.T) { + var typ any + vin := dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vout, err := Normalize(&typ, vin) + assert.Len(t, err, 0) + assert.Equal(t, dyn.NewValue(false, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) +} + +func TestNormalizeAnyFromInt(t *testing.T) { + var typ any + vin := dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vout, err := Normalize(&typ, vin) + assert.Len(t, err, 0) + assert.Equal(t, dyn.NewValue(10, []dyn.Location{{File: "file", Line: 1, Column: 1}}), vout) +} + +func TestNormalizeAnyFromTime(t *testing.T) { + var typ any + vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []dyn.Location{{File: "file", Line: 1, Column: 1}}) + vout, err := Normalize(&typ, vin) + assert.Empty(t, err) + assert.Equal(t, dyn.NewValue("2024-08-29", vin.Locations()), vout) +} diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 839d0111a..e2aa5d263 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -221,10 +221,10 @@ func toTypedBool(dst reflect.Value, src dyn.Value) error { case dyn.KindString: // See https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/decode.go#L684-L693. switch src.MustString() { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", "true": dst.SetBool(true) return nil - case "n", "N", "no", "No", "NO", "off", "Off", "OFF": + case "n", "N", "no", "No", "NO", "off", "Off", "OFF", "false": dst.SetBool(false) return nil } @@ -246,6 +246,19 @@ func toTypedInt(dst reflect.Value, src dyn.Value) error { case dyn.KindInt: dst.SetInt(src.MustInt()) return nil + case dyn.KindFloat: + v := src.MustFloat() + if v == float64(int64(v)) { + // If the destination is smaller than int64, but the value to set is bigger + // then destination overflows and is set to -1 + dst.SetInt(int64(src.MustFloat())) + return nil + } + + return TypeError{ + value: src, + msg: fmt.Sprintf("expected an int, found a %s", src.Kind()), + } case dyn.KindString: if i64, err := strconv.ParseInt(src.MustString(), 10, 64); err == nil { dst.SetInt(i64) diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go index 37d85539c..78221c299 100644 --- a/libs/dyn/convert/to_typed_test.go +++ b/libs/dyn/convert/to_typed_test.go @@ -341,14 +341,14 @@ func TestToTypedBoolFromString(t *testing.T) { var out bool // True-ish - for _, v := range []string{"y", "yes", "on"} { + for _, v := range []string{"y", "yes", "on", "true"} { err := ToTyped(&out, dyn.V(v)) require.NoError(t, err) assert.Equal(t, true, out) } // False-ish - for _, v := range []string{"n", "no", "off"} { + for _, v := range []string{"n", "no", "off", "false"} { err := ToTyped(&out, dyn.V(v)) require.NoError(t, err) assert.Equal(t, false, out) @@ -428,6 +428,19 @@ func TestToTypedIntFromStringVariableReference(t *testing.T) { assert.Equal(t, int(0), out) } +func TestToTypedIntFromFloat(t *testing.T) { + var out int + err := ToTyped(&out, dyn.V(1.0)) + require.NoError(t, err) + assert.Equal(t, int(1), out) +} + +func TestToTypedIntFromFloatError(t *testing.T) { + var out int + err := ToTyped(&out, dyn.V(1.2)) + require.ErrorContains(t, err, "expected an int, found a float") +} + func TestToTypedFloat32(t *testing.T) { var out float32 err := ToTyped(&out, dyn.V(float32(1.0))) diff --git a/libs/dyn/dynassert/assert.go b/libs/dyn/dynassert/assert.go index dc6676ca2..f667b08c7 100644 --- a/libs/dyn/dynassert/assert.go +++ b/libs/dyn/dynassert/assert.go @@ -111,3 +111,7 @@ func PanicsWithError(t assert.TestingT, errString string, f func(), msgAndArgs . func NotPanics(t assert.TestingT, f func(), msgAndArgs ...interface{}) bool { return assert.NotPanics(t, f, msgAndArgs...) } + +func JSONEq(t assert.TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { + return assert.JSONEq(t, expected, actual, msgAndArgs...) +} diff --git a/libs/dyn/dynassert/dump.go b/libs/dyn/dynassert/dump.go new file mode 100644 index 000000000..be0c72425 --- /dev/null +++ b/libs/dyn/dynassert/dump.go @@ -0,0 +1,60 @@ +package dynassert + +import ( + "fmt" + "strings" + + "github.com/databricks/cli/libs/dyn" +) + +// Dump returns the Go code to recreate the given value. +func Dump(v dyn.Value) string { + var sb strings.Builder + dump(v, &sb) + return sb.String() +} + +func dump(v dyn.Value, sb *strings.Builder) { + sb.WriteString("dyn.NewValue(\n") + + switch v.Kind() { + case dyn.KindMap: + sb.WriteString("map[string]dyn.Value{") + m := v.MustMap() + for _, p := range m.Pairs() { + sb.WriteString(fmt.Sprintf("\n%q: ", p.Key.MustString())) + dump(p.Value, sb) + sb.WriteByte(',') + } + sb.WriteString("\n},\n") + case dyn.KindSequence: + sb.WriteString("[]dyn.Value{\n") + for _, e := range v.MustSequence() { + dump(e, sb) + sb.WriteByte(',') + } + sb.WriteString("},\n") + case dyn.KindString: + sb.WriteString(fmt.Sprintf("%q,\n", v.MustString())) + case dyn.KindBool: + sb.WriteString(fmt.Sprintf("%t,\n", v.MustBool())) + case dyn.KindInt: + sb.WriteString(fmt.Sprintf("%d,\n", v.MustInt())) + case dyn.KindFloat: + sb.WriteString(fmt.Sprintf("%f,\n", v.MustFloat())) + case dyn.KindTime: + sb.WriteString(fmt.Sprintf("dyn.NewTime(%q),\n", v.MustTime().String())) + case dyn.KindNil: + sb.WriteString("nil,\n") + default: + panic(fmt.Sprintf("unhandled kind: %v", v.Kind())) + } + + // Add location + sb.WriteString("[]dyn.Location{") + for _, l := range v.Locations() { + sb.WriteString(fmt.Sprintf("{File: %q, Line: %d, Column: %d},", l.File, l.Line, l.Column)) + } + sb.WriteString("},\n") + sb.WriteString(")") +} diff --git a/libs/dyn/jsonloader/json.go b/libs/dyn/jsonloader/json.go new file mode 100644 index 000000000..cbf539263 --- /dev/null +++ b/libs/dyn/jsonloader/json.go @@ -0,0 +1,107 @@ +package jsonloader + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/databricks/cli/libs/dyn" +) + +func LoadJSON(data []byte, source string) (dyn.Value, error) { + offset := BuildLineOffsets(data) + offset.SetSource(source) + + reader := bytes.NewReader(data) + decoder := json.NewDecoder(reader) + + // Start decoding from the top-level value + value, err := decodeValue(decoder, &offset) + if err != nil { + if err == io.EOF { + err = fmt.Errorf("unexpected end of JSON input") + } + return dyn.InvalidValue, fmt.Errorf("error decoding JSON at %s: %v", value.Location(), err) + } + return value, nil +} + +func decodeValue(decoder *json.Decoder, o *Offset) (dyn.Value, error) { + // Read the next JSON token + token, err := decoder.Token() + if err != nil { + return dyn.InvalidValue, err + } + + // Get the current byte offset and the location. + // We will later use this location to store the location of the value in the file + // For objects and arrays, we will store the location of the opening '{' or '[' + // For primitive types, we will store the location of the value itself (end of the value) + // We can't reliably calculate the beginning of the value for primitive types because + // the decoder doesn't provide the offset of the beginning of the value and the value might or might not be quoted. + offset := decoder.InputOffset() + location := o.GetPosition(offset) + + switch tok := token.(type) { + case json.Delim: + if tok == '{' { + location = o.GetPosition(offset - 1) + // Decode JSON object + obj := dyn.NewMapping() + for decoder.More() { + // Decode the key + keyToken, err := decoder.Token() + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + key, ok := keyToken.(string) + if !ok { + return invalidValueWithLocation(decoder, o), fmt.Errorf("expected string for object key") + } + + // Get the offset of the key by subtracting the length of the key and the '"' character + keyOffset := decoder.InputOffset() - int64(len(key)+1) + keyVal := dyn.NewValue(key, []dyn.Location{o.GetPosition(keyOffset)}) + + // Decode the value recursively + val, err := decodeValue(decoder, o) + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + + obj.Set(keyVal, val) + } + // Consume the closing '}' + if _, err := decoder.Token(); err != nil { + return invalidValueWithLocation(decoder, o), err + } + return dyn.NewValue(obj, []dyn.Location{location}), nil + } else if tok == '[' { + location = o.GetPosition(offset - 1) + // Decode JSON array + var arr []dyn.Value + for decoder.More() { + val, err := decodeValue(decoder, o) + if err != nil { + return invalidValueWithLocation(decoder, o), err + } + arr = append(arr, val) + } + // Consume the closing ']' + if _, err := decoder.Token(); err != nil { + return invalidValueWithLocation(decoder, o), err + } + return dyn.NewValue(arr, []dyn.Location{location}), nil + } + default: + return dyn.NewValue(tok, []dyn.Location{location}), nil + } + + return invalidValueWithLocation(decoder, o), fmt.Errorf("unexpected token: %v", token) +} + +func invalidValueWithLocation(decoder *json.Decoder, o *Offset) dyn.Value { + location := o.GetPosition(decoder.InputOffset()) + return dyn.InvalidValue.WithLocations([]dyn.Location{location}) +} diff --git a/libs/dyn/jsonloader/json_test.go b/libs/dyn/jsonloader/json_test.go new file mode 100644 index 000000000..7fdc4d7f5 --- /dev/null +++ b/libs/dyn/jsonloader/json_test.go @@ -0,0 +1,93 @@ +package jsonloader + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +const jsonData = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "webhook_notifications": { + "on_start": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": 0, + "max_concurrent_runs": 1, + "tasks": [ + { + "task_key": "xxx", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonLoader(t *testing.T) { + v, err := LoadJSON([]byte(jsonData), "(inline)") + dynassert.NoError(t, err) + + var r jobs.ResetJob + err = convert.ToTyped(&r, v) + dynassert.NoError(t, err) +} + +const malformedMap = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "wrong", + } +} +` + +func TestJsonLoaderMalformedMap(t *testing.T) { + _, err := LoadJSON([]byte(malformedMap), "(inline)") + dynassert.ErrorContains(t, err, "error decoding JSON at (inline):6:16: invalid character ',' after object key") +} + +const malformedArray = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx", + "tasks": [1, "asd",] + } +}` + +func TestJsonLoaderMalformedArray(t *testing.T) { + _, err := LoadJSON([]byte(malformedArray), "path/to/file.json") + dynassert.ErrorContains(t, err, "error decoding JSON at path/to/file.json:6:28: invalid character ']' looking for beginning of value") +} + +const eofData = ` +{ + "job_id": 123, + "new_settings": { + "name": "xxx",` + +func TestJsonLoaderEOF(t *testing.T) { + _, err := LoadJSON([]byte(eofData), "path/to/file.json") + dynassert.ErrorContains(t, err, "unexpected end of JSON input") +} diff --git a/libs/dyn/jsonloader/locations.go b/libs/dyn/jsonloader/locations.go new file mode 100644 index 000000000..a692c7d03 --- /dev/null +++ b/libs/dyn/jsonloader/locations.go @@ -0,0 +1,53 @@ +package jsonloader + +import ( + "sort" + + "github.com/databricks/cli/libs/dyn" +) + +type LineOffset struct { + Line int + Start int64 +} + +type Offset struct { + offsets []LineOffset + source string +} + +// buildLineOffsets scans the input data and records the starting byte offset of each line. +func BuildLineOffsets(data []byte) Offset { + offsets := []LineOffset{{Line: 1, Start: 0}} + line := 1 + for i, b := range data { + if b == '\n' { + line++ + offsets = append(offsets, LineOffset{Line: line, Start: int64(i + 1)}) + } + } + return Offset{offsets: offsets} +} + +// GetPosition maps a byte offset to its corresponding line and column numbers. +func (o Offset) GetPosition(offset int64) dyn.Location { + // Binary search to find the line + idx := sort.Search(len(o.offsets), func(i int) bool { + return o.offsets[i].Start > offset + }) - 1 + + if idx < 0 { + idx = 0 + } + + lineOffset := o.offsets[idx] + return dyn.Location{ + File: o.source, + Line: lineOffset.Line, + Column: int(offset-lineOffset.Start) + 1, + } +} + +func (o *Offset) SetSource(source string) { + o.source = source +} diff --git a/libs/dyn/jsonsaver/encoder.go b/libs/dyn/jsonsaver/encoder.go new file mode 100644 index 000000000..66997e96d --- /dev/null +++ b/libs/dyn/jsonsaver/encoder.go @@ -0,0 +1,39 @@ +package jsonsaver + +import ( + "bytes" + "encoding/json" +) + +// The encoder type encapsulates a [json.Encoder] and its target buffer. +// Escaping of HTML characters in the output is disabled. +type encoder struct { + *json.Encoder + *bytes.Buffer +} + +func newEncoder() encoder { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + + // By default, json.Encoder escapes HTML characters, converting symbols like '<' to '\u003c'. + // This behavior helps prevent XSS attacks when JSON is embedded within HTML. + // However, we disable this feature since we're not dealing with HTML context. + // Keeping the escapes enabled would result in unnecessary differences when processing JSON payloads + // that already contain escaped characters. + enc.SetEscapeHTML(false) + return encoder{enc, &buf} +} + +func marshalNoEscape(v any) ([]byte, error) { + enc := newEncoder() + err := enc.Encode(v) + return enc.Bytes(), err +} + +func marshalIndentNoEscape(v any, prefix, indent string) ([]byte, error) { + enc := newEncoder() + enc.SetIndent(prefix, indent) + err := enc.Encode(v) + return enc.Bytes(), err +} diff --git a/libs/dyn/jsonsaver/encoder_test.go b/libs/dyn/jsonsaver/encoder_test.go new file mode 100644 index 000000000..d1b7d0177 --- /dev/null +++ b/libs/dyn/jsonsaver/encoder_test.go @@ -0,0 +1,41 @@ +package jsonsaver + +import ( + "testing" + + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestEncoder_MarshalNoEscape(t *testing.T) { + out, err := marshalNoEscape("1 < 2") + if !assert.NoError(t, err) { + return + } + + // Confirm the output. + assert.JSONEq(t, `"1 < 2"`, string(out)) + + // Confirm that HTML escaping is disabled. + assert.NotContains(t, string(out), "\\u003c") + + // Confirm that the encoder writes a trailing newline. + assert.Contains(t, string(out), "\n") +} + +func TestEncoder_MarshalIndentNoEscape(t *testing.T) { + out, err := marshalIndentNoEscape([]string{"1 < 2", "2 < 3"}, "", " ") + if !assert.NoError(t, err) { + return + } + + // Confirm the output. + assert.JSONEq(t, `["1 < 2", "2 < 3"]`, string(out)) + + // Confirm that HTML escaping is disabled. + assert.NotContains(t, string(out), "\\u003c") + + // Confirm that the encoder performs indenting and writes a trailing newline. + assert.Contains(t, string(out), "[\n") + assert.Contains(t, string(out), " \"1 < 2\",\n") + assert.Contains(t, string(out), "]\n") +} diff --git a/libs/dyn/jsonsaver/marshal.go b/libs/dyn/jsonsaver/marshal.go new file mode 100644 index 000000000..a78a68f29 --- /dev/null +++ b/libs/dyn/jsonsaver/marshal.go @@ -0,0 +1,89 @@ +package jsonsaver + +import ( + "bytes" + "fmt" + + "github.com/databricks/cli/libs/dyn" +) + +// Marshal is a version of [json.Marshal] for [dyn.Value]. +// +// Objects in the output retain the order of keys as they appear in the underlying [dyn.Value]. +// The output does not escape HTML characters in strings. +func Marshal(v dyn.Value) ([]byte, error) { + return marshalNoEscape(wrap{v}) +} + +// MarshalIndent is a version of [json.MarshalIndent] for [dyn.Value]. +// +// Objects in the output retain the order of keys as they appear in the underlying [dyn.Value]. +// The output does not escape HTML characters in strings. +func MarshalIndent(v dyn.Value, prefix, indent string) ([]byte, error) { + return marshalIndentNoEscape(wrap{v}, prefix, indent) +} + +// Wrapper type for [dyn.Value] to expose the [json.Marshaler] interface. +type wrap struct { + v dyn.Value +} + +// MarshalJSON implements the [json.Marshaler] interface for the [dyn.Value] wrapper type. +func (w wrap) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + if err := marshalValue(&buf, w.v); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// marshalValue recursively writes JSON for a [dyn.Value] to the buffer. +func marshalValue(buf *bytes.Buffer, v dyn.Value) error { + switch v.Kind() { + case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil: + out, err := marshalNoEscape(v.AsAny()) + if err != nil { + return err + } + + // The encoder writes a trailing newline, so we need to remove it + // to avoid adding extra newlines when embedding this JSON. + out = out[:len(out)-1] + buf.Write(out) + case dyn.KindMap: + buf.WriteByte('{') + for i, pair := range v.MustMap().Pairs() { + if i > 0 { + buf.WriteByte(',') + } + // Require keys to be strings. + if pair.Key.Kind() != dyn.KindString { + return fmt.Errorf("map key must be a string, got %s", pair.Key.Kind()) + } + // Marshal the key + if err := marshalValue(buf, pair.Key); err != nil { + return err + } + buf.WriteByte(':') + // Marshal the value + if err := marshalValue(buf, pair.Value); err != nil { + return err + } + } + buf.WriteByte('}') + case dyn.KindSequence: + buf.WriteByte('[') + for i, item := range v.MustSequence() { + if i > 0 { + buf.WriteByte(',') + } + if err := marshalValue(buf, item); err != nil { + return err + } + } + buf.WriteByte(']') + default: + return fmt.Errorf("unsupported kind: %d", v.Kind()) + } + return nil +} diff --git a/libs/dyn/jsonsaver/marshal_test.go b/libs/dyn/jsonsaver/marshal_test.go new file mode 100644 index 000000000..0b6a34283 --- /dev/null +++ b/libs/dyn/jsonsaver/marshal_test.go @@ -0,0 +1,100 @@ +package jsonsaver + +import ( + "testing" + + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +func TestMarshal_String(t *testing.T) { + b, err := Marshal(dyn.V("string")) + if assert.NoError(t, err) { + assert.JSONEq(t, `"string"`, string(b)) + } +} + +func TestMarshal_Bool(t *testing.T) { + b, err := Marshal(dyn.V(true)) + if assert.NoError(t, err) { + assert.JSONEq(t, `true`, string(b)) + } +} + +func TestMarshal_Int(t *testing.T) { + b, err := Marshal(dyn.V(42)) + if assert.NoError(t, err) { + assert.JSONEq(t, `42`, string(b)) + } +} + +func TestMarshal_Float(t *testing.T) { + b, err := Marshal(dyn.V(42.1)) + if assert.NoError(t, err) { + assert.JSONEq(t, `42.1`, string(b)) + } +} + +func TestMarshal_Time(t *testing.T) { + b, err := Marshal(dyn.V(dyn.MustTime("2021-01-01T00:00:00Z"))) + if assert.NoError(t, err) { + assert.JSONEq(t, `"2021-01-01T00:00:00Z"`, string(b)) + } +} + +func TestMarshal_Map(t *testing.T) { + m := dyn.NewMapping() + m.Set(dyn.V("key1"), dyn.V("value1")) + m.Set(dyn.V("key2"), dyn.V("value2")) + + b, err := Marshal(dyn.V(m)) + if assert.NoError(t, err) { + assert.JSONEq(t, `{"key1":"value1","key2":"value2"}`, string(b)) + } +} + +func TestMarshal_Sequence(t *testing.T) { + var s []dyn.Value + s = append(s, dyn.V("value1")) + s = append(s, dyn.V("value2")) + + b, err := Marshal(dyn.V(s)) + if assert.NoError(t, err) { + assert.JSONEq(t, `["value1","value2"]`, string(b)) + } +} + +func TestMarshal_Complex(t *testing.T) { + map1 := dyn.NewMapping() + map1.Set(dyn.V("str1"), dyn.V("value1")) + map1.Set(dyn.V("str2"), dyn.V("value2")) + + seq1 := []dyn.Value{} + seq1 = append(seq1, dyn.V("value1")) + seq1 = append(seq1, dyn.V("value2")) + + root := dyn.NewMapping() + root.Set(dyn.V("map1"), dyn.V(map1)) + root.Set(dyn.V("seq1"), dyn.V(seq1)) + + // Marshal without indent. + b, err := Marshal(dyn.V(root)) + if assert.NoError(t, err) { + assert.Equal(t, `{"map1":{"str1":"value1","str2":"value2"},"seq1":["value1","value2"]}`+"\n", string(b)) + } + + // Marshal with indent. + b, err = MarshalIndent(dyn.V(root), "", " ") + if assert.NoError(t, err) { + assert.Equal(t, `{ + "map1": { + "str1": "value1", + "str2": "value2" + }, + "seq1": [ + "value1", + "value2" + ] +}`+"\n", string(b)) + } +} diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index c3e8d0810..b4aaf0a74 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -105,6 +105,9 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro switch st { case "!!str": // OK + case "!!null": + // A literal unquoted "null" is treated as a null value by the YAML parser. + // However, when used as a key, it is treated as the string "null". case "!!merge": if merge != nil { panic("merge node already set") @@ -115,10 +118,11 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro return dyn.InvalidValue, errorf(loc, "invalid key tag: %v", st) } - k, err := d.load(key) - if err != nil { - return dyn.InvalidValue, err - } + k := dyn.NewValue(key.Value, []dyn.Location{{ + File: d.path, + Line: key.Line, + Column: key.Column, + }}) v, err := d.load(val) if err != nil { @@ -173,6 +177,14 @@ func (d *loader) loadMapping(node *yaml.Node, loc dyn.Location) (dyn.Value, erro return dyn.NewValue(out, []dyn.Location{loc}), nil } +func newIntValue(i64 int64, loc dyn.Location) dyn.Value { + // Use regular int type instead of int64 if possible. + if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { + return dyn.NewValue(int(i64), []dyn.Location{loc}) + } + return dyn.NewValue(i64, []dyn.Location{loc}) +} + func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error) { st := node.ShortTag() switch st { @@ -188,18 +200,44 @@ func (d *loader) loadScalar(node *yaml.Node, loc dyn.Location) (dyn.Value, error return dyn.InvalidValue, errorf(loc, "invalid bool value: %v", node.Value) } case "!!int": - i64, err := strconv.ParseInt(node.Value, 10, 64) - if err != nil { - return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value) + // Try to parse the an integer value in base 10. + // We trim leading zeros to avoid octal parsing of the "0" prefix. + // See "testdata/spec_example_2.19.yml" for background. + i64, err := strconv.ParseInt(strings.TrimLeft(node.Value, "0"), 10, 64) + if err == nil { + return newIntValue(i64, loc), nil } - // Use regular int type instead of int64 if possible. - if i64 >= math.MinInt32 && i64 <= math.MaxInt32 { - return dyn.NewValue(int(i64), []dyn.Location{loc}), nil + // Let the [ParseInt] function figure out the base. + i64, err = strconv.ParseInt(node.Value, 0, 64) + if err == nil { + return newIntValue(i64, loc), nil } - return dyn.NewValue(i64, []dyn.Location{loc}), nil + return dyn.InvalidValue, errorf(loc, "invalid int value: %v", node.Value) case "!!float": f64, err := strconv.ParseFloat(node.Value, 64) if err != nil { + // Deal with infinity prefixes. + v := strings.ToLower(node.Value) + switch { + case strings.HasPrefix(v, "+"): + v = strings.TrimPrefix(v, "+") + f64 = math.Inf(1) + case strings.HasPrefix(v, "-"): + v = strings.TrimPrefix(v, "-") + f64 = math.Inf(-1) + default: + // No prefix. + f64 = math.Inf(1) + } + + // Deal with infinity and NaN values. + switch v { + case ".inf": + return dyn.NewValue(f64, []dyn.Location{loc}), nil + case ".nan": + return dyn.NewValue(math.NaN(), []dyn.Location{loc}), nil + } + return dyn.InvalidValue, errorf(loc, "invalid float value: %v", node.Value) } return dyn.NewValue(f64, []dyn.Location{loc}), nil diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.1.yml b/libs/dyn/yamlloader/testdata/spec_example_2.1.yml new file mode 100644 index 000000000..c9e26274c --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.1.yml @@ -0,0 +1,5 @@ +# Example 2.1 Sequence of Scalars (ball players) + +- Mark McGwire +- Sammy Sosa +- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.10.yml b/libs/dyn/yamlloader/testdata/spec_example_2.10.yml new file mode 100644 index 000000000..a3459dedf --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.10.yml @@ -0,0 +1,10 @@ +# Example 2.10 Node for “Sammy Sosa” appears twice in this document + +--- +hr: +- Mark McGwire +# Following node labeled SS +- &SS Sammy Sosa +rbi: +- *SS # Subsequent occurrence +- Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.11.yml b/libs/dyn/yamlloader/testdata/spec_example_2.11.yml new file mode 100644 index 000000000..e3c5c115e --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.11.yml @@ -0,0 +1,10 @@ +# Example 2.11 Mapping between Sequences + +? - Detroit Tigers + - Chicago cubs +: - 2001-07-23 + +? [ New York Yankees, + Atlanta Braves ] +: [ 2001-07-02, 2001-08-12, + 2001-08-14 ] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.12.yml b/libs/dyn/yamlloader/testdata/spec_example_2.12.yml new file mode 100644 index 000000000..eb4a526f6 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.12.yml @@ -0,0 +1,10 @@ +# Example 2.12 Compact Nested Mapping + +--- +# Products purchased +- item : Super Hoop + quantity: 1 +- item : Basketball + quantity: 4 +- item : Big Shoes + quantity: 1 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.13.yml b/libs/dyn/yamlloader/testdata/spec_example_2.13.yml new file mode 100644 index 000000000..e55abff10 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.13.yml @@ -0,0 +1,6 @@ +# Example 2.13 In literals, newlines are preserved + +# ASCII Art +--- | + \//||\/|| + // || ||__ diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.14.yml b/libs/dyn/yamlloader/testdata/spec_example_2.14.yml new file mode 100644 index 000000000..439fca30c --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.14.yml @@ -0,0 +1,6 @@ +# Example 2.14 In the folded scalars, newlines become spaces + +--- > + Mark McGwire's + year was crippled + by a knee injury. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.15.yml b/libs/dyn/yamlloader/testdata/spec_example_2.15.yml new file mode 100644 index 000000000..266e7ce47 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.15.yml @@ -0,0 +1,10 @@ +# Example 2.15 Folded newlines are preserved for “more indented” and blank lines + +--- > + Sammy Sosa completed another + fine season with great stats. + + 63 Home Runs + 0.288 Batting Average + + What a year! diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.16.yml b/libs/dyn/yamlloader/testdata/spec_example_2.16.yml new file mode 100644 index 000000000..6db6b0878 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.16.yml @@ -0,0 +1,9 @@ +# Example 2.16 Indentation determines scope + +name: Mark McGwire +accomplishment: > + Mark set a major league + home run record in 1998. +stats: | + 65 Home Runs + 0.278 Batting Average diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.17.yml b/libs/dyn/yamlloader/testdata/spec_example_2.17.yml new file mode 100644 index 000000000..af0777aba --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.17.yml @@ -0,0 +1,9 @@ +# Example 2.17 Quoted Scalars + +unicode: "Sosa did fine.\u263A" +control: "\b1998\t1999\t2000\n" +hex esc: "\x0d\x0a is \r\n" + +single: '"Howdy!" he cried.' +quoted: ' # Not a ''comment''.' +tie-fighter: '|\-*-/|' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.18.yml b/libs/dyn/yamlloader/testdata/spec_example_2.18.yml new file mode 100644 index 000000000..741bcd8c0 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.18.yml @@ -0,0 +1,8 @@ +# Example 2.18 Multi-line Flow Scalars + +plain: + This unquoted scalar + spans many lines. + +quoted: "So does this + quoted scalar.\n" diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.19.yml b/libs/dyn/yamlloader/testdata/spec_example_2.19.yml new file mode 100644 index 000000000..6ed95e092 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.19.yml @@ -0,0 +1,15 @@ +# Example 2.19 Integers + +canonical: 12345 +decimal: +12345 +octal: 0o14 +hexadecimal: 0xC + +# Note: this example is not part of the spec but added for completeness. +# +# Octal numbers: +# - YAML 1.1: prefix is "0" +# - YAML 1.2: prefix is "0o" +# The "gopkg.in/yaml.v3" package accepts both for backwards compat. +# We accept only the YAML 1.2 prefix "0o". +octal11: 012345 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.2.yml b/libs/dyn/yamlloader/testdata/spec_example_2.2.yml new file mode 100644 index 000000000..29c16105e --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.2.yml @@ -0,0 +1,5 @@ +# Example 2.2 Mapping Scalars to Scalars (player statistics) + +hr: 65 # Home runs +avg: 0.278 # Batting average +rbi: 147 # Runs Batted In diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.20.yml b/libs/dyn/yamlloader/testdata/spec_example_2.20.yml new file mode 100644 index 000000000..77a79a0c3 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.20.yml @@ -0,0 +1,7 @@ +# Example 2.20 Floating Point + +canonical: 1.23015e+3 +exponential: 12.3015e+02 +fixed: 1230.15 +negative infinity: -.inf +not a number: .nan diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.21.yml b/libs/dyn/yamlloader/testdata/spec_example_2.21.yml new file mode 100644 index 000000000..cdb423c56 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.21.yml @@ -0,0 +1,5 @@ +# Example 2.21 Miscellaneous + +null: +booleans: [ true, false ] +string: '012345' diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.22.yml b/libs/dyn/yamlloader/testdata/spec_example_2.22.yml new file mode 100644 index 000000000..bef2addfa --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.22.yml @@ -0,0 +1,6 @@ +# Example 2.22 Timestamps + +canonical: 2001-12-15T02:59:43.1Z +iso8601: 2001-12-14t21:59:43.10-05:00 +spaced: 2001-12-14 21:59:43.10 -5 +date: 2002-12-14 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.23.yml b/libs/dyn/yamlloader/testdata/spec_example_2.23.yml new file mode 100644 index 000000000..56e9898e2 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.23.yml @@ -0,0 +1,15 @@ +# Example 2.23 Various Explicit Tags + +--- +not-date: !!str 2002-04-28 + +picture: !!binary | + R0lGODlhDAAMAIQAAP//9/X + 17unp5WZmZgAAAOfn515eXv + Pz7Y6OjuDg4J+fn5OTk6enp + 56enmleECcgggoBADs= + +application specific tag: !something | + The semantics of the tag + above may be different for + different documents. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.24.yml b/libs/dyn/yamlloader/testdata/spec_example_2.24.yml new file mode 100644 index 000000000..f7c11d0fb --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.24.yml @@ -0,0 +1,16 @@ +# Example 2.24 Global Tags + +%TAG ! tag:clarkevans.com,2002: +--- !shape + # Use the ! handle for presenting + # tag:clarkevans.com,2002:circle +- !circle + center: &ORIGIN {x: 73, y: 129} + radius: 7 +- !line + start: *ORIGIN + finish: { x: 89, y: 102 } +- !label + start: *ORIGIN + color: 0xFFEEBB + text: Pretty vector drawing. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.25.yml b/libs/dyn/yamlloader/testdata/spec_example_2.25.yml new file mode 100644 index 000000000..73bac8627 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.25.yml @@ -0,0 +1,9 @@ +# Example 2.25 Unordered Sets + +# Sets are represented as a +# Mapping where each key is +# associated with a null value +--- !!set +? Mark McGwire +? Sammy Sosa +? Ken Griffey diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.26.yml b/libs/dyn/yamlloader/testdata/spec_example_2.26.yml new file mode 100644 index 000000000..00863a6b8 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.26.yml @@ -0,0 +1,9 @@ +# Example 2.26 Ordered Mappings + +# Ordered maps are represented as +# A sequence of mappings, with +# each mapping having one key +--- !!omap +- Mark McGwire: 65 +- Sammy Sosa: 63 +- Ken Griffey: 58 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.27.yml b/libs/dyn/yamlloader/testdata/spec_example_2.27.yml new file mode 100644 index 000000000..fc9b460c2 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.27.yml @@ -0,0 +1,31 @@ +# Example 2.27 Invoice + +--- ! +invoice: 34843 +date : 2001-01-23 +bill-to: &id001 + given : Chris + family : Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city : Royal Oak + state : MI + postal : 48046 +ship-to: *id001 +product: +- sku : BL394D + quantity : 4 + description : Basketball + price : 450.00 +- sku : BL4438H + quantity : 1 + description : Super Hoop + price : 2392.00 +tax : 251.42 +total: 4443.52 +comments: + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.28.yml b/libs/dyn/yamlloader/testdata/spec_example_2.28.yml new file mode 100644 index 000000000..353694729 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.28.yml @@ -0,0 +1,28 @@ +# Example 2.28 Log File + +--- +Time: 2001-11-23 15:01:42 -5 +User: ed +Warning: + This is an error message + for the log file +--- +Time: 2001-11-23 15:02:31 -5 +User: ed +Warning: + A slightly different error + message. +--- +Date: 2001-11-23 15:03:17 -5 +User: ed +Fatal: + Unknown variable "bar" +Stack: +- file: TopClass.py + line: 23 + code: | + x = MoreObject("345\n") +- file: MoreClass.py + line: 58 + code: |- + foo = bar diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.3.yml b/libs/dyn/yamlloader/testdata/spec_example_2.3.yml new file mode 100644 index 000000000..70cbe07d3 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.3.yml @@ -0,0 +1,10 @@ +# Example 2.3 Mapping Scalars to Sequences (ball clubs in each league) + +american: +- Boston Red Sox +- Detroit Tigers +- New York Yankees +national: +- New York Mets +- Chicago Cubs +- Atlanta Braves diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.4.yml b/libs/dyn/yamlloader/testdata/spec_example_2.4.yml new file mode 100644 index 000000000..cce28625f --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.4.yml @@ -0,0 +1,10 @@ +# Example 2.4 Sequence of Mappings (players’ statistics) + +- + name: Mark McGwire + hr: 65 + avg: 0.278 +- + name: Sammy Sosa + hr: 63 + avg: 0.288 diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.5.yml b/libs/dyn/yamlloader/testdata/spec_example_2.5.yml new file mode 100644 index 000000000..a585faee7 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.5.yml @@ -0,0 +1,5 @@ +# Example 2.5 Sequence of Sequences + +- [name , hr, avg ] +- [Mark McGwire, 65, 0.278] +- [Sammy Sosa , 63, 0.288] diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.6.yml b/libs/dyn/yamlloader/testdata/spec_example_2.6.yml new file mode 100644 index 000000000..cc137e5d5 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.6.yml @@ -0,0 +1,7 @@ +# Example 2.6 Mapping of Mappings + +Mark McGwire: {hr: 65, avg: 0.278} +Sammy Sosa: { + hr: 63, + avg: 0.288, + } diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.7.yml b/libs/dyn/yamlloader/testdata/spec_example_2.7.yml new file mode 100644 index 000000000..35c2541d6 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.7.yml @@ -0,0 +1,12 @@ +# Example 2.7 Two Documents in a Stream (each with a leading comment) + +# Ranking of 1998 home runs +--- +- Mark McGwire +- Sammy Sosa +- Ken Griffey + +# Team ranking +--- +- Chicago Cubs +- St Louis Cardinals diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.8.yml b/libs/dyn/yamlloader/testdata/spec_example_2.8.yml new file mode 100644 index 000000000..ae6e8bf27 --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.8.yml @@ -0,0 +1,12 @@ +# Example 2.8 Play by Play Feed from a Game + +--- +time: 20:03:20 +player: Sammy Sosa +action: strike (miss) +... +--- +time: 20:03:47 +player: Sammy Sosa +action: grand slam +... diff --git a/libs/dyn/yamlloader/testdata/spec_example_2.9.yml b/libs/dyn/yamlloader/testdata/spec_example_2.9.yml new file mode 100644 index 000000000..75217b25e --- /dev/null +++ b/libs/dyn/yamlloader/testdata/spec_example_2.9.yml @@ -0,0 +1,10 @@ +# Example 2.9 Single Document with Two Comments + +--- +hr: # 1998 hr ranking +- Mark McGwire +- Sammy Sosa +# 1998 rbi ranking +rbi: +- Sammy Sosa +- Ken Griffey diff --git a/libs/dyn/yamlloader/yaml_spec_test.go b/libs/dyn/yamlloader/yaml_spec_test.go new file mode 100644 index 000000000..2a5ae817f --- /dev/null +++ b/libs/dyn/yamlloader/yaml_spec_test.go @@ -0,0 +1,821 @@ +package yamlloader_test + +import ( + "bytes" + "math" + "os" + "testing" + + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/stretchr/testify/require" +) + +const NL = "\n" + +func loadExample(t *testing.T, file string) dyn.Value { + input, err := os.ReadFile(file) + require.NoError(t, err) + self, err := yamlloader.LoadYAML(file, bytes.NewBuffer(input)) + require.NoError(t, err) + return self +} + +func TestYAMLSpecExample_2_1(t *testing.T) { + file := "testdata/spec_example_2.1.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 3, Column: 3}}), + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 4, Column: 3}}), + dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 5, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_2(t *testing.T) { + file := "testdata/spec_example_2.2.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 3, Column: 6}}), + "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 4, Column: 6}}), + "rbi": dyn.NewValue(147, []dyn.Location{{File: file, Line: 5, Column: 6}}), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_3(t *testing.T) { + file := "testdata/spec_example_2.3.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "american": dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Boston Red Sox", []dyn.Location{{File: file, Line: 4, Column: 3}}), + dyn.NewValue("Detroit Tigers", []dyn.Location{{File: file, Line: 5, Column: 3}}), + dyn.NewValue("New York Yankees", []dyn.Location{{File: file, Line: 6, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 1}}, + ), + "national": dyn.NewValue( + []dyn.Value{ + dyn.NewValue("New York Mets", []dyn.Location{{File: file, Line: 8, Column: 3}}), + dyn.NewValue("Chicago Cubs", []dyn.Location{{File: file, Line: 9, Column: 3}}), + dyn.NewValue("Atlanta Braves", []dyn.Location{{File: file, Line: 10, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 1}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_4(t *testing.T) { + file := "testdata/spec_example_2.4.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + map[string]dyn.Value{ + "name": dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 4, Column: 9}}), + "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 5, Column: 9}}), + "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 6, Column: 9}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "name": dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 8, Column: 9}}), + "hr": dyn.NewValue(63, []dyn.Location{{File: file, Line: 9, Column: 9}}), + "avg": dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 10, Column: 9}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_5(t *testing.T) { + file := "testdata/spec_example_2.5.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + []dyn.Value{ + dyn.NewValue("name", []dyn.Location{{File: file, Line: 3, Column: 4}}), + dyn.NewValue("hr", []dyn.Location{{File: file, Line: 3, Column: 18}}), + dyn.NewValue("avg", []dyn.Location{{File: file, Line: 3, Column: 22}}), + }, + []dyn.Location{{File: file, Line: 3, Column: 3}}, + ), + dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 4, Column: 4}}), + dyn.NewValue(65, []dyn.Location{{File: file, Line: 4, Column: 18}}), + dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 4, Column: 22}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 3}}, + ), + dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 5, Column: 4}}), + dyn.NewValue(63, []dyn.Location{{File: file, Line: 5, Column: 18}}), + dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 5, Column: 22}}), + }, + []dyn.Location{{File: file, Line: 5, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_6(t *testing.T) { + file := "testdata/spec_example_2.6.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "Mark McGwire": dyn.NewValue( + map[string]dyn.Value{ + "hr": dyn.NewValue(65, []dyn.Location{{File: file, Line: 3, Column: 20}}), + "avg": dyn.NewValue(0.278, []dyn.Location{{File: file, Line: 3, Column: 29}}), + }, + []dyn.Location{{File: file, Line: 3, Column: 15}}, + ), + "Sammy Sosa": dyn.NewValue( + map[string]dyn.Value{ + "hr": dyn.NewValue(63, []dyn.Location{{File: file, Line: 5, Column: 9}}), + "avg": dyn.NewValue(0.288, []dyn.Location{{File: file, Line: 6, Column: 10}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 13}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_7(t *testing.T) { + file := "testdata/spec_example_2.7.yml" + self := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + "Mark McGwire", + []dyn.Location{{File: file, Line: 5, Column: 3}}, + ), + dyn.NewValue( + "Sammy Sosa", + []dyn.Location{{File: file, Line: 6, Column: 3}}, + ), + dyn.NewValue( + "Ken Griffey", + []dyn.Location{{File: file, Line: 7, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 5, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_8(t *testing.T) { + file := "testdata/spec_example_2.8.yml" + self := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "time": dyn.NewValue("20:03:20", []dyn.Location{{File: file, Line: 4, Column: 7}}), + "player": dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 5, Column: 9}}), + "action": dyn.NewValue("strike (miss)", []dyn.Location{{File: file, Line: 6, Column: 9}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_9(t *testing.T) { + file := "testdata/spec_example_2.9.yml" + self := loadExample(t, file) + + // Note: we do not support multiple documents in a single YAML file. + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "hr": dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 5, Column: 3}}), + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 6, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 5, Column: 1}}, + ), + "rbi": dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 9, Column: 3}}), + dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 10, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 9, Column: 1}}, + ), + }, + []dyn.Location{{File: file, Line: 4, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_10(t *testing.T) { + file := "testdata/spec_example_2.10.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "hr": dyn.NewValue( + []dyn.Value{ + dyn.NewValue("Mark McGwire", []dyn.Location{{File: file, Line: 5, Column: 3}}), + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 7, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 5, Column: 1}}, + ), + "rbi": dyn.NewValue( + []dyn.Value{ + // The location for an anchored value refers to the anchor, not the reference. + // This is the same location as the anchor that appears in the "hr" mapping. + dyn.NewValue("Sammy Sosa", []dyn.Location{{File: file, Line: 7, Column: 3}}), + dyn.NewValue("Ken Griffey", []dyn.Location{{File: file, Line: 10, Column: 3}}), + }, + []dyn.Location{{File: file, Line: 9, Column: 1}}, + ), + }, + []dyn.Location{{File: file, Line: 4, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_11(t *testing.T) { + file := "testdata/spec_example_2.11.yml" + input, err := os.ReadFile(file) + require.NoError(t, err) + + // Note: non-string mapping keys are not supported by "gopkg.in/yaml.v3". + _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) + assert.ErrorContains(t, err, `: key is not a scalar`) +} + +func TestYAMLSpecExample_2_12(t *testing.T) { + file := "testdata/spec_example_2.12.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + map[string]dyn.Value{ + "item": dyn.NewValue("Super Hoop", []dyn.Location{{File: file, Line: 5, Column: 13}}), + "quantity": dyn.NewValue(1, []dyn.Location{{File: file, Line: 6, Column: 13}}), + }, + []dyn.Location{{File: file, Line: 5, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "item": dyn.NewValue("Basketball", []dyn.Location{{File: file, Line: 7, Column: 13}}), + "quantity": dyn.NewValue(4, []dyn.Location{{File: file, Line: 8, Column: 13}}), + }, + []dyn.Location{{File: file, Line: 7, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "item": dyn.NewValue("Big Shoes", []dyn.Location{{File: file, Line: 9, Column: 13}}), + "quantity": dyn.NewValue(1, []dyn.Location{{File: file, Line: 10, Column: 13}}), + }, + []dyn.Location{{File: file, Line: 9, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 5, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_13(t *testing.T) { + file := "testdata/spec_example_2.13.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + ``+ + `\//||\/||`+NL+ + "// || ||__"+NL, + []dyn.Location{{File: file, Line: 4, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_14(t *testing.T) { + file := "testdata/spec_example_2.14.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + `Mark McGwire's year was crippled by a knee injury.`+NL, + []dyn.Location{{File: file, Line: 3, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_15(t *testing.T) { + file := "testdata/spec_example_2.15.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + ``+ + `Sammy Sosa completed another fine season with great stats.`+NL+ + NL+ + ` 63 Home Runs`+NL+ + ` 0.288 Batting Average`+NL+ + NL+ + `What a year!`+NL, + []dyn.Location{{File: file, Line: 3, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_16(t *testing.T) { + file := "testdata/spec_example_2.16.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "name": dyn.NewValue( + "Mark McGwire", + []dyn.Location{{File: file, Line: 3, Column: 7}}, + ), + "accomplishment": dyn.NewValue( + `Mark set a major league home run record in 1998.`+NL, + []dyn.Location{{File: file, Line: 4, Column: 17}}, + ), + "stats": dyn.NewValue( + ``+ + `65 Home Runs`+NL+ + `0.278 Batting Average`+NL, + []dyn.Location{{File: file, Line: 7, Column: 8}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_17(t *testing.T) { + file := "testdata/spec_example_2.17.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "unicode": dyn.NewValue( + `Sosa did fine.`+"\u263A", + []dyn.Location{{File: file, Line: 3, Column: 10}}, + ), + "control": dyn.NewValue( + "\b1998\t1999\t2000\n", + []dyn.Location{{File: file, Line: 4, Column: 10}}, + ), + "hex esc": dyn.NewValue( + "\x0d\x0a is \r\n", + []dyn.Location{{File: file, Line: 5, Column: 10}}, + ), + "single": dyn.NewValue( + `"Howdy!" he cried.`, + []dyn.Location{{File: file, Line: 7, Column: 9}}, + ), + "quoted": dyn.NewValue( + ` # Not a 'comment'.`, + []dyn.Location{{File: file, Line: 8, Column: 9}}, + ), + "tie-fighter": dyn.NewValue( + `|\-*-/|`, + []dyn.Location{{File: file, Line: 9, Column: 14}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_18(t *testing.T) { + file := "testdata/spec_example_2.18.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "plain": dyn.NewValue( + `This unquoted scalar spans many lines.`, + []dyn.Location{{File: file, Line: 4, Column: 3}}, + ), + "quoted": dyn.NewValue( + `So does this quoted scalar.`+NL, + []dyn.Location{{File: file, Line: 7, Column: 9}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_19(t *testing.T) { + file := "testdata/spec_example_2.19.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "canonical": dyn.NewValue( + 12345, + []dyn.Location{{File: file, Line: 3, Column: 12}}, + ), + "decimal": dyn.NewValue( + 12345, + []dyn.Location{{File: file, Line: 4, Column: 10}}, + ), + "octal": dyn.NewValue( + 12, + []dyn.Location{{File: file, Line: 5, Column: 8}}, + ), + "hexadecimal": dyn.NewValue( + 12, + []dyn.Location{{File: file, Line: 6, Column: 14}}, + ), + "octal11": dyn.NewValue( + 12345, + []dyn.Location{{File: file, Line: 15, Column: 10}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_20(t *testing.T) { + file := "testdata/spec_example_2.20.yml" + self := loadExample(t, file) + + // Equality assertion doesn't work with NaNs. + // See https://github.com/stretchr/testify/issues/624. + // + // Remove the NaN entry. + self, _ = dyn.Walk(self, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if f, ok := v.AsFloat(); ok && math.IsNaN(f) { + return dyn.InvalidValue, dyn.ErrDrop + } + return v, nil + }) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "canonical": dyn.NewValue( + 1230.15, + []dyn.Location{{File: file, Line: 3, Column: 12}}, + ), + "exponential": dyn.NewValue( + 1230.15, + []dyn.Location{{File: file, Line: 4, Column: 14}}, + ), + "fixed": dyn.NewValue( + 1230.15, + []dyn.Location{{File: file, Line: 5, Column: 8}}, + ), + "negative infinity": dyn.NewValue( + math.Inf(-1), + []dyn.Location{{File: file, Line: 6, Column: 20}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_21(t *testing.T) { + file := "testdata/spec_example_2.21.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "null": dyn.NewValue( + nil, + []dyn.Location{{File: file, Line: 3, Column: 6}}, + ), + "booleans": dyn.NewValue( + []dyn.Value{ + dyn.NewValue(true, []dyn.Location{{File: file, Line: 4, Column: 13}}), + dyn.NewValue(false, []dyn.Location{{File: file, Line: 4, Column: 19}}), + }, + []dyn.Location{{File: file, Line: 4, Column: 11}}, + ), + "string": dyn.NewValue( + "012345", + []dyn.Location{{File: file, Line: 5, Column: 9}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_22(t *testing.T) { + file := "testdata/spec_example_2.22.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "canonical": dyn.NewValue( + dyn.MustTime("2001-12-15T02:59:43.1Z"), + []dyn.Location{{File: file, Line: 3, Column: 12}}, + ), + "iso8601": dyn.NewValue( + dyn.MustTime("2001-12-14t21:59:43.10-05:00"), + []dyn.Location{{File: file, Line: 4, Column: 10}}, + ), + "spaced": dyn.NewValue( + // This is parsed as a string, not a timestamp, + // both by "gopkg.in/yaml.v3" and by our implementation. + "2001-12-14 21:59:43.10 -5", + []dyn.Location{{File: file, Line: 5, Column: 9}}, + ), + "date": dyn.NewValue( + dyn.MustTime("2002-12-14"), + []dyn.Location{{File: file, Line: 6, Column: 7}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 1}}, + ), self) +} + +func TestYAMLSpecExample_2_23(t *testing.T) { + file := "testdata/spec_example_2.23.yml" + input, err := os.ReadFile(file) + require.NoError(t, err) + + // Note: the !!binary tag is not supported by us. + + _, err = yamlloader.LoadYAML(file, bytes.NewBuffer(input)) + assert.ErrorContains(t, err, `: unknown tag: !!binary`) +} + +func TestYAMLSpecExample_2_24(t *testing.T) { + file := "testdata/spec_example_2.24.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + map[string]dyn.Value{ + "center": dyn.NewValue( + map[string]dyn.Value{ + "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), + "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 11}}, + ), + "radius": dyn.NewValue(7, []dyn.Location{{File: file, Line: 9, Column: 11}}), + }, + []dyn.Location{{File: file, Line: 7, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "start": dyn.NewValue( + map[string]dyn.Value{ + "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), + "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 11}}, + ), + "finish": dyn.NewValue( + map[string]dyn.Value{ + "x": dyn.NewValue(89, []dyn.Location{{File: file, Line: 12, Column: 16}}), + "y": dyn.NewValue(102, []dyn.Location{{File: file, Line: 12, Column: 23}}), + }, + []dyn.Location{{File: file, Line: 12, Column: 11}}, + ), + }, + []dyn.Location{{File: file, Line: 10, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "start": dyn.NewValue( + map[string]dyn.Value{ + "x": dyn.NewValue(73, []dyn.Location{{File: file, Line: 8, Column: 23}}), + "y": dyn.NewValue(129, []dyn.Location{{File: file, Line: 8, Column: 30}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 11}}, + ), + "color": dyn.NewValue(16772795, []dyn.Location{{File: file, Line: 15, Column: 10}}), + "text": dyn.NewValue("Pretty vector drawing.", []dyn.Location{{File: file, Line: 16, Column: 9}}), + }, + []dyn.Location{{File: file, Line: 13, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 4, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_25(t *testing.T) { + file := "testdata/spec_example_2.25.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "Mark McGwire": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 8, Column: 1}}), + "Sammy Sosa": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 9, Column: 1}}), + "Ken Griffey": dyn.NewValue(nil, []dyn.Location{{File: file, Line: 10, Column: 1}}), + }, + []dyn.Location{{File: file, Line: 6, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_26(t *testing.T) { + file := "testdata/spec_example_2.26.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + map[string]dyn.Value{ + "Mark McGwire": dyn.NewValue(65, []dyn.Location{{File: file, Line: 7, Column: 17}}), + }, + []dyn.Location{{File: file, Line: 7, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "Sammy Sosa": dyn.NewValue(63, []dyn.Location{{File: file, Line: 8, Column: 15}}), + }, + []dyn.Location{{File: file, Line: 8, Column: 3}}, + ), + dyn.NewValue( + map[string]dyn.Value{ + "Ken Griffey": dyn.NewValue(58, []dyn.Location{{File: file, Line: 9, Column: 16}}), + }, + []dyn.Location{{File: file, Line: 9, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 6, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_27(t *testing.T) { + file := "testdata/spec_example_2.27.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "invoice": dyn.NewValue( + 34843, + []dyn.Location{{File: file, Line: 4, Column: 10}}, + ), + "date": dyn.NewValue( + dyn.MustTime("2001-01-23"), + []dyn.Location{{File: file, Line: 5, Column: 10}}, + ), + "bill-to": dyn.NewValue( + map[string]dyn.Value{ + "given": dyn.NewValue( + "Chris", + []dyn.Location{{File: file, Line: 7, Column: 12}}, + ), + "family": dyn.NewValue( + "Dumars", + []dyn.Location{{File: file, Line: 8, Column: 12}}, + ), + "address": dyn.NewValue( + map[string]dyn.Value{ + "lines": dyn.NewValue( + "458 Walkman Dr.\nSuite #292\n", + []dyn.Location{{File: file, Line: 10, Column: 12}}, + ), + "city": dyn.NewValue( + "Royal Oak", + []dyn.Location{{File: file, Line: 13, Column: 15}}, + ), + "state": dyn.NewValue( + "MI", + []dyn.Location{{File: file, Line: 14, Column: 15}}, + ), + "postal": dyn.NewValue( + 48046, + []dyn.Location{{File: file, Line: 15, Column: 15}}, + ), + }, + []dyn.Location{{File: file, Line: 10, Column: 5}}, + ), + }, + []dyn.Location{{File: file, Line: 6, Column: 10}}, + ), + "ship-to": dyn.NewValue( + map[string]dyn.Value{ + "given": dyn.NewValue( + "Chris", + []dyn.Location{{File: file, Line: 7, Column: 12}}, + ), + "family": dyn.NewValue( + "Dumars", + []dyn.Location{{File: file, Line: 8, Column: 12}}, + ), + "address": dyn.NewValue( + map[string]dyn.Value{ + "lines": dyn.NewValue( + "458 Walkman Dr.\nSuite #292\n", + []dyn.Location{{File: file, Line: 10, Column: 12}}, + ), + "city": dyn.NewValue( + "Royal Oak", + []dyn.Location{{File: file, Line: 13, Column: 15}}, + ), + "state": dyn.NewValue( + "MI", + []dyn.Location{{File: file, Line: 14, Column: 15}}, + ), + "postal": dyn.NewValue( + 48046, + []dyn.Location{{File: file, Line: 15, Column: 15}}, + ), + }, + []dyn.Location{{File: file, Line: 10, Column: 5}}, + ), + }, + []dyn.Location{{File: file, Line: 6, Column: 10}}, + ), + "product": dyn.NewValue( + []dyn.Value{ + dyn.NewValue( + map[string]dyn.Value{ + "sku": dyn.NewValue( + "BL394D", + []dyn.Location{{File: file, Line: 18, Column: 17}}, + ), + "quantity": dyn.NewValue( + 4, + []dyn.Location{{File: file, Line: 19, Column: 17}}, + ), + "description": dyn.NewValue( + "Basketball", + []dyn.Location{{File: file, Line: 20, Column: 17}}, + ), + "price": dyn.NewValue( + 450.0, + []dyn.Location{{File: file, Line: 21, Column: 17}}, + ), + }, + []dyn.Location{{File: file, Line: 18, Column: 3}}, + ), dyn.NewValue( + map[string]dyn.Value{ + "sku": dyn.NewValue( + "BL4438H", + []dyn.Location{{File: file, Line: 22, Column: 17}}, + ), + "quantity": dyn.NewValue( + 1, + []dyn.Location{{File: file, Line: 23, Column: 17}}, + ), + "description": dyn.NewValue( + "Super Hoop", + []dyn.Location{{File: file, Line: 24, Column: 17}}, + ), + "price": dyn.NewValue( + 2392.0, + []dyn.Location{{File: file, Line: 25, Column: 17}}, + ), + }, + []dyn.Location{{File: file, Line: 22, Column: 3}}, + )}, + []dyn.Location{{File: file, Line: 18, Column: 1}}, + ), + "tax": dyn.NewValue( + 251.42, + []dyn.Location{{File: file, Line: 26, Column: 8}}, + ), + "total": dyn.NewValue( + 4443.52, + []dyn.Location{{File: file, Line: 27, Column: 8}}, + ), + "comments": dyn.NewValue( + "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338.", + []dyn.Location{{File: file, Line: 29, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 3, Column: 5}}, + ), self) +} + +func TestYAMLSpecExample_2_28(t *testing.T) { + file := "testdata/spec_example_2.28.yml" + self := loadExample(t, file) + + assert.Equal(t, dyn.NewValue( + map[string]dyn.Value{ + "Time": dyn.NewValue( + "2001-11-23 15:01:42 -5", + []dyn.Location{{File: file, Line: 4, Column: 7}}, + ), + "User": dyn.NewValue( + "ed", + []dyn.Location{{File: file, Line: 5, Column: 7}}, + ), + "Warning": dyn.NewValue( + "This is an error message for the log file", + []dyn.Location{{File: file, Line: 7, Column: 3}}, + ), + }, + []dyn.Location{{File: file, Line: 4, Column: 1}}, + ), self) +} diff --git a/libs/filer/filer.go b/libs/filer/filer.go index c1c747c54..fcfbcea07 100644 --- a/libs/filer/filer.go +++ b/libs/filer/filer.go @@ -103,6 +103,18 @@ func (err CannotDeleteRootError) Is(other error) bool { return other == fs.ErrInvalid } +type PermissionError struct { + path string +} + +func (err PermissionError) Error() string { + return fmt.Sprintf("access denied: %s", err.path) +} + +func (err PermissionError) Is(other error) bool { + return other == fs.ErrPermission +} + // Filer is used to access files in a workspace. // It has implementations for accessing files in WSFS and in DBFS. type Filer interface { diff --git a/libs/filer/workspace_files_client.go b/libs/filer/workspace_files_client.go index d8ab5a6bb..4bb03aea5 100644 --- a/libs/filer/workspace_files_client.go +++ b/libs/filer/workspace_files_client.go @@ -178,6 +178,9 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io // Create parent directory. err = w.workspaceClient.Workspace.MkdirsByPath(ctx, path.Dir(absPath)) if err != nil { + if errors.As(err, &aerr) && aerr.StatusCode == http.StatusForbidden { + return PermissionError{absPath} + } return fmt.Errorf("unable to mkdir to write file %s: %w", absPath, err) } @@ -203,6 +206,11 @@ func (w *workspaceFilesClient) Write(ctx context.Context, name string, reader io return FileAlreadyExistsError{absPath} } + // This API returns StatusForbidden when you have read access but don't have write access to a file + if aerr.StatusCode == http.StatusForbidden { + return PermissionError{absPath} + } + return err } @@ -295,11 +303,11 @@ func (w *workspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.D return nil, err } - // This API returns a 404 if the specified path does not exist. + // NOTE: This API returns a 404 if the specified path does not exist, + // but can also do so if we don't have read access. if aerr.StatusCode == http.StatusNotFound { return nil, NoSuchDirectoryError{path.Dir(absPath)} } - return nil, err } diff --git a/libs/flags/json_flag.go b/libs/flags/json_flag.go index 8dbc3b2d9..0324f1145 100644 --- a/libs/flags/json_flag.go +++ b/libs/flags/json_flag.go @@ -4,10 +4,17 @@ import ( "encoding/json" "fmt" "os" + "reflect" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/jsonloader" + "github.com/databricks/databricks-sdk-go/marshal" ) type JsonFlag struct { - raw []byte + raw []byte + source string } func (j *JsonFlag) String() string { @@ -19,21 +26,65 @@ func (j *JsonFlag) Set(v string) error { // Load request from file if it starts with '@' (like curl). if v[0] != '@' { j.raw = []byte(v) + j.source = "(inline)" return nil } - buf, err := os.ReadFile(v[1:]) + filePath := v[1:] + buf, err := os.ReadFile(filePath) + j.source = filePath if err != nil { - return fmt.Errorf("read %s: %w", v, err) + return fmt.Errorf("read %s: %w", filePath, err) } j.raw = buf return nil } -func (j *JsonFlag) Unmarshal(v any) error { +func (j *JsonFlag) Unmarshal(v any) diag.Diagnostics { if j.raw == nil { return nil } - return json.Unmarshal(j.raw, v) + + dv, err := jsonloader.LoadJSON(j.raw, j.source) + if err != nil { + return diag.FromErr(err) + } + + // First normalize the input data. + // It will convert all the values to the correct types. + // For example string literals for booleans and integers will be converted to the correct types. + nv, diags := convert.Normalize(v, dv) + if diags.HasError() { + return diags + } + + // Then marshal the normalized data to the output. + // It will serialize all set data with the correct types. + data, err := json.Marshal(nv.AsAny()) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + + kind := reflect.ValueOf(v).Kind() + if kind == reflect.Ptr { + kind = reflect.ValueOf(v).Elem().Kind() + } + + if kind == reflect.Struct { + // Finally unmarshal the normalized data to the output. + // It will fill in the ForceSendFields field if the struct contains it. + err = marshal.Unmarshal(data, v) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + } else { + // If the output is not a struct, just unmarshal the data to the output. + err = json.Unmarshal(data, v) + if err != nil { + return diags.Extend(diag.FromErr(err)) + } + } + + return diags } func (j *JsonFlag) Type() string { diff --git a/libs/flags/json_flag_test.go b/libs/flags/json_flag_test.go index 2a8170fe6..77530086a 100644 --- a/libs/flags/json_flag_test.go +++ b/libs/flags/json_flag_test.go @@ -6,6 +6,9 @@ import ( "path" "testing" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -14,10 +17,11 @@ func TestJsonFlagEmpty(t *testing.T) { var body JsonFlag var request any - err := body.Unmarshal(&request) + diags := body.Unmarshal(&request) assert.Equal(t, "JSON (0 bytes)", body.String()) - assert.NoError(t, err) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) assert.Nil(t, request) } @@ -28,8 +32,9 @@ func TestJsonFlagInline(t *testing.T) { assert.NoError(t, err) var request any - err = body.Unmarshal(&request) - assert.NoError(t, err) + diags := body.Unmarshal(&request) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) assert.Equal(t, "JSON (14 bytes)", body.String()) assert.Equal(t, map[string]any{"foo": "bar"}, request) @@ -42,8 +47,8 @@ func TestJsonFlagError(t *testing.T) { assert.NoError(t, err) var request any - err = body.Unmarshal(&request) - assert.EqualError(t, err, "unexpected end of JSON input") + diags := body.Unmarshal(&request) + assert.EqualError(t, diags.Error(), "error decoding JSON at (inline):1:8: unexpected end of JSON input") assert.Equal(t, "JSON (7 bytes)", body.String()) } @@ -52,7 +57,7 @@ func TestJsonFlagFile(t *testing.T) { var request any var fpath string - var payload = []byte(`"hello world"`) + var payload = []byte(`{"foo": "bar"}`) { f, err := os.Create(path.Join(t.TempDir(), "file")) @@ -65,8 +70,217 @@ func TestJsonFlagFile(t *testing.T) { err := body.Set(fmt.Sprintf("@%s", fpath)) require.NoError(t, err) - err = body.Unmarshal(&request) + diags := body.Unmarshal(&request) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, map[string]any{"foo": "bar"}, request) +} + +const jsonData = ` +{ + "job_id": 123, + "new_settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_canceled_runs": false + }, + "timeout_seconds": 0, + "max_concurrent_runs": 1, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalForRequest(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(jsonData) require.NoError(t, err) - assert.Equal(t, "hello world", request) + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, int64(123), r.JobId) + assert.Equal(t, "new job", r.NewSettings.Name) + assert.Equal(t, 0, r.NewSettings.TimeoutSeconds) + assert.Equal(t, 1, r.NewSettings.MaxConcurrentRuns) + assert.Equal(t, 1, len(r.NewSettings.Tasks)) + assert.Equal(t, "new task", r.NewSettings.Tasks[0].TaskKey) + assert.Equal(t, 0, r.NewSettings.Tasks[0].TimeoutSeconds) + assert.Equal(t, 0, r.NewSettings.Tasks[0].MaxRetries) + assert.Equal(t, 0, r.NewSettings.Tasks[0].MinRetryIntervalMillis) + assert.Equal(t, true, r.NewSettings.Tasks[0].RetryOnTimeout) +} + +const incorrectJsonData = `{ + "job_id": 123, + "settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": {}, + "max_concurrent_runs": {}, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalRequestMismatch(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(incorrectJsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "unknown field: settings", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 3, + Column: 6, + }, + }, + Paths: []dyn.Path{{}}, + }) +} + +const wrontTypeJsonData = `{ + "job_id": 123, + "new_settings": { + "name": "new job", + "email_notifications": { + "on_start": [], + "on_success": [], + "on_failure": [] + }, + "notification_settings": { + "no_alert_for_skipped_runs": true, + "no_alert_for_canceled_runs": true + }, + "timeout_seconds": "wrong_type", + "max_concurrent_runs": {}, + "tasks": [ + { + "task_key": "new task", + "email_notifications": {}, + "notification_settings": {}, + "timeout_seconds": 0, + "max_retries": 0, + "min_retry_interval_millis": 0, + "retry_on_timeout": "true" + } + ] + } +} +` + +func TestJsonUnmarshalWrongTypeReportsCorrectLocation(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(`{ + "job_id": [1, 2, 3] +} +`) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "expected int, found sequence", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 2, + Column: 15, + }, + }, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("job_id"))}, + }) +} + +func TestJsonUnmarshalArrayInsteadOfIntReportsCorrectLocation(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(wrontTypeJsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.NotEmpty(t, diags) + + assert.Contains(t, diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "cannot parse \"wrong_type\" as an integer", + Locations: []dyn.Location{ + { + File: "(inline)", + Line: 14, + Column: 40, + }, + }, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("new_settings"), dyn.Key("timeout_seconds"))}, + }) +} + +func TestJsonUnmarshalForRequestWithForceSendFields(t *testing.T) { + var body JsonFlag + + var r jobs.ResetJob + err := body.Set(jsonData) + require.NoError(t, err) + + diags := body.Unmarshal(&r) + assert.NoError(t, diags.Error()) + assert.Empty(t, diags) + + assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForSkippedRuns) + assert.Equal(t, false, r.NewSettings.NotificationSettings.NoAlertForCanceledRuns) + assert.NotContains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForSkippedRuns") + assert.Contains(t, r.NewSettings.NotificationSettings.ForceSendFields, "NoAlertForCanceledRuns") } diff --git a/libs/git/repository.go b/libs/git/repository.go index 6940ddac8..0bbd57865 100644 --- a/libs/git/repository.go +++ b/libs/git/repository.go @@ -23,8 +23,21 @@ type Repository struct { // directory where we process .gitignore files. real bool - // root is the absolute path to the repository root. - root vfs.Path + // rootDir is the path to the root of the repository checkout. + // This can be either the main repository checkout or a worktree checkout. + // For more information about worktrees, see: https://git-scm.com/docs/git-worktree#_description. + rootDir vfs.Path + + // gitDir is the equivalent of $GIT_DIR and points to the + // `.git` directory of a repository or a worktree directory. + // See https://git-scm.com/docs/git-worktree#_details for more information. + gitDir vfs.Path + + // gitCommonDir is the equivalent of $GIT_COMMON_DIR and points to the + // `.git` directory of the main working tree (common between worktrees). + // This is equivalent to [gitDir] if this is the main working tree. + // See https://git-scm.com/docs/git-worktree#_details for more information. + gitCommonDir vfs.Path // ignore contains a list of ignore patterns indexed by the // path prefix relative to the repository root. @@ -44,12 +57,11 @@ type Repository struct { // Root returns the absolute path to the repository root. func (r *Repository) Root() string { - return r.root.Native() + return r.rootDir.Native() } func (r *Repository) CurrentBranch() (string, error) { - // load .git/HEAD - ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD")) + ref, err := LoadReferenceFile(r.gitDir, "HEAD") if err != nil { return "", err } @@ -65,8 +77,7 @@ func (r *Repository) CurrentBranch() (string, error) { } func (r *Repository) LatestCommit() (string, error) { - // load .git/HEAD - ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD")) + ref, err := LoadReferenceFile(r.gitDir, "HEAD") if err != nil { return "", err } @@ -80,12 +91,12 @@ func (r *Repository) LatestCommit() (string, error) { return ref.Content, nil } - // read reference from .git/HEAD + // Read reference from $GIT_DIR/HEAD branchHeadPath, err := ref.ResolvePath() if err != nil { return "", err } - branchHeadRef, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, branchHeadPath)) + branchHeadRef, err := LoadReferenceFile(r.gitCommonDir, branchHeadPath) if err != nil { return "", err } @@ -125,7 +136,7 @@ func (r *Repository) loadConfig() error { if err != nil { return fmt.Errorf("unable to load user specific gitconfig: %w", err) } - err = config.loadFile(r.root, ".git/config") + err = config.loadFile(r.gitCommonDir, "config") if err != nil { return fmt.Errorf("unable to load repository specific gitconfig: %w", err) } @@ -133,12 +144,6 @@ func (r *Repository) loadConfig() error { return nil } -// newIgnoreFile constructs a new [ignoreRules] implementation backed by -// a file using the specified path relative to the repository root. -func (r *Repository) newIgnoreFile(relativeIgnoreFilePath string) ignoreRules { - return newIgnoreFile(r.root, relativeIgnoreFilePath) -} - // getIgnoreRules returns a slice of [ignoreRules] that apply // for the specified prefix. The prefix must be cleaned by the caller. // It lazily initializes an entry for the specified prefix if it @@ -149,7 +154,7 @@ func (r *Repository) getIgnoreRules(prefix string) []ignoreRules { return fs } - r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(path.Join(prefix, gitIgnoreFileName))) + r.ignore[prefix] = append(r.ignore[prefix], newIgnoreFile(r.rootDir, path.Join(prefix, gitIgnoreFileName))) return r.ignore[prefix] } @@ -205,21 +210,30 @@ func (r *Repository) Ignore(relPath string) (bool, error) { func NewRepository(path vfs.Path) (*Repository, error) { real := true - rootPath, err := vfs.FindLeafInTree(path, GitDirectoryName) + rootDir, err := vfs.FindLeafInTree(path, GitDirectoryName) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return nil, err } // Cannot find `.git` directory. - // Treat the specified path as a potential repository root. + // Treat the specified path as a potential repository root checkout. real = false - rootPath = path + rootDir = path + } + + // Derive $GIT_DIR and $GIT_COMMON_DIR paths if this is a real repository. + // If it isn't a real repository, they'll point to the (non-existent) `.git` directory. + gitDir, gitCommonDir, err := resolveGitDirs(rootDir) + if err != nil { + return nil, err } repo := &Repository{ - real: real, - root: rootPath, - ignore: make(map[string][]ignoreRules), + real: real, + rootDir: rootDir, + gitDir: gitDir, + gitCommonDir: gitCommonDir, + ignore: make(map[string][]ignoreRules), } err = repo.loadConfig() @@ -252,10 +266,10 @@ func NewRepository(path vfs.Path) (*Repository, error) { newStringIgnoreRules([]string{ ".git", }), - // Load repository-wide excludes file. - repo.newIgnoreFile(".git/info/excludes"), + // Load repository-wide exclude file. + newIgnoreFile(repo.gitCommonDir, "info/exclude"), // Load root gitignore file. - repo.newIgnoreFile(".gitignore"), + newIgnoreFile(repo.rootDir, ".gitignore"), } return repo, nil diff --git a/libs/git/view.go b/libs/git/view.go index 90eed0bb8..2d2e39a60 100644 --- a/libs/git/view.go +++ b/libs/git/view.go @@ -80,7 +80,7 @@ func NewView(root vfs.Path) (*View, error) { // Target path must be relative to the repository root path. target := root.Native() - prefix := repo.root.Native() + prefix := repo.rootDir.Native() if !strings.HasPrefix(target, prefix) { return nil, fmt.Errorf("path %q is not within repository root %q", root.Native(), prefix) } diff --git a/libs/git/worktree.go b/libs/git/worktree.go new file mode 100644 index 000000000..964c1c95b --- /dev/null +++ b/libs/git/worktree.go @@ -0,0 +1,123 @@ +package git + +import ( + "bufio" + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/vfs" +) + +func readLines(root vfs.Path, name string) ([]string, error) { + file, err := root.Open(name) + if err != nil { + return nil, err + } + + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + return lines, scanner.Err() +} + +// readGitDir reads the value of the `.git` file in a worktree. +func readGitDir(root vfs.Path) (string, error) { + lines, err := readLines(root, GitDirectoryName) + if err != nil { + return "", err + } + + var gitDir string + for _, line := range lines { + parts := strings.SplitN(line, ": ", 2) + if len(parts) != 2 { + continue + } + + if parts[0] == "gitdir" { + gitDir = strings.TrimSpace(parts[1]) + } + } + + if gitDir == "" { + return "", fmt.Errorf(`expected %q to contain a line with "gitdir: [...]"`, filepath.Join(root.Native(), GitDirectoryName)) + } + + return gitDir, nil +} + +// readGitCommonDir reads the value of the `commondir` file in the `.git` directory of a worktree. +// This file typically contains "../.." to point to $GIT_COMMON_DIR. +func readGitCommonDir(gitDir vfs.Path) (string, error) { + lines, err := readLines(gitDir, "commondir") + if err != nil { + return "", err + } + + if len(lines) == 0 { + return "", errors.New("file is empty") + } + + return strings.TrimSpace(lines[0]), nil +} + +// resolveGitDirs resolves the paths for $GIT_DIR and $GIT_COMMON_DIR. +// The path argument is the root of the checkout where (supposedly) a `.git` file or directory exists. +func resolveGitDirs(root vfs.Path) (vfs.Path, vfs.Path, error) { + fileInfo, err := root.Stat(GitDirectoryName) + if err != nil { + // If the `.git` file or directory does not exist, then this is not a git repository. + // Return paths that we know don't exist, so we do not need to perform nil checks in the caller. + if errors.Is(err, fs.ErrNotExist) { + gitDir := vfs.MustNew(filepath.Join(root.Native(), GitDirectoryName)) + return gitDir, gitDir, nil + } + return nil, nil, err + } + + // If the path is a directory, then it is the main working tree. + // Both $GIT_DIR and $GIT_COMMON_DIR point to the same directory. + if fileInfo.IsDir() { + gitDir := vfs.MustNew(filepath.Join(root.Native(), GitDirectoryName)) + return gitDir, gitDir, nil + } + + // If the path is not a directory, then it is a worktree. + // Read value for $GIT_DIR. + gitDirValue, err := readGitDir(root) + if err != nil { + return nil, nil, err + } + + // Resolve $GIT_DIR. + var gitDir vfs.Path + if filepath.IsAbs(gitDirValue) { + gitDir = vfs.MustNew(gitDirValue) + } else { + gitDir = vfs.MustNew(filepath.Join(root.Native(), gitDirValue)) + } + + // Read value for $GIT_COMMON_DIR. + gitCommonDirValue, err := readGitCommonDir(gitDir) + if err != nil { + return nil, nil, fmt.Errorf(`expected "commondir" file in worktree git folder at %q: %w`, gitDir.Native(), err) + } + + // Resolve $GIT_COMMON_DIR. + var gitCommonDir vfs.Path + if filepath.IsAbs(gitCommonDirValue) { + gitCommonDir = vfs.MustNew(gitCommonDirValue) + } else { + gitCommonDir = vfs.MustNew(filepath.Join(gitDir.Native(), gitCommonDirValue)) + } + + return gitDir, gitCommonDir, nil +} diff --git a/libs/git/worktree_test.go b/libs/git/worktree_test.go new file mode 100644 index 000000000..3d620c483 --- /dev/null +++ b/libs/git/worktree_test.go @@ -0,0 +1,108 @@ +package git + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupWorktree(t *testing.T) string { + var err error + + tmpDir := t.TempDir() + + // Checkout path + err = os.MkdirAll(filepath.Join(tmpDir, "my_worktree"), os.ModePerm) + require.NoError(t, err) + + // Main $GIT_COMMON_DIR + err = os.MkdirAll(filepath.Join(tmpDir, ".git"), os.ModePerm) + require.NoError(t, err) + + // Worktree $GIT_DIR + err = os.MkdirAll(filepath.Join(tmpDir, ".git/worktrees/my_worktree"), os.ModePerm) + require.NoError(t, err) + + return tmpDir +} + +func writeGitDir(t *testing.T, dir, content string) { + err := os.WriteFile(filepath.Join(dir, "my_worktree/.git"), []byte(content), os.ModePerm) + require.NoError(t, err) +} + +func writeGitCommonDir(t *testing.T, dir, content string) { + err := os.WriteFile(filepath.Join(dir, ".git/worktrees/my_worktree/commondir"), []byte(content), os.ModePerm) + require.NoError(t, err) +} + +func verifyCorrectDirs(t *testing.T, dir string) { + gitDir, gitCommonDir, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree"))) + require.NoError(t, err) + assert.Equal(t, filepath.Join(dir, ".git/worktrees/my_worktree"), gitDir.Native()) + assert.Equal(t, filepath.Join(dir, ".git"), gitCommonDir.Native()) +} + +func TestWorktreeResolveGitDir(t *testing.T) { + dir := setupWorktree(t) + writeGitCommonDir(t, dir, "../..") + + t.Run("relative", func(t *testing.T) { + writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree")) + verifyCorrectDirs(t, dir) + }) + + t.Run("absolute", func(t *testing.T) { + writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", filepath.Join(dir, ".git/worktrees/my_worktree"))) + verifyCorrectDirs(t, dir) + }) + + t.Run("additional spaces", func(t *testing.T) { + writeGitDir(t, dir, fmt.Sprintf("gitdir: %s \n\n\n", "../.git/worktrees/my_worktree")) + verifyCorrectDirs(t, dir) + }) + + t.Run("empty", func(t *testing.T) { + writeGitDir(t, dir, "") + + _, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree"))) + assert.ErrorContains(t, err, ` to contain a line with "gitdir: [...]"`) + }) +} + +func TestWorktreeResolveCommonDir(t *testing.T) { + dir := setupWorktree(t) + writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree")) + + t.Run("relative", func(t *testing.T) { + writeGitCommonDir(t, dir, "../..") + verifyCorrectDirs(t, dir) + }) + + t.Run("absolute", func(t *testing.T) { + writeGitCommonDir(t, dir, filepath.Join(dir, ".git")) + verifyCorrectDirs(t, dir) + }) + + t.Run("additional spaces", func(t *testing.T) { + writeGitCommonDir(t, dir, " ../.. \n\n\n") + verifyCorrectDirs(t, dir) + }) + + t.Run("empty", func(t *testing.T) { + writeGitCommonDir(t, dir, "") + + _, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree"))) + assert.ErrorContains(t, err, `expected "commondir" file in worktree git folder at `) + }) + + t.Run("missing", func(t *testing.T) { + _, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree"))) + assert.ErrorContains(t, err, `expected "commondir" file in worktree git folder at `) + }) +} diff --git a/libs/auth/service_principal.go b/libs/iamutil/service_principal.go similarity index 61% rename from libs/auth/service_principal.go rename to libs/iamutil/service_principal.go index 5f1854e3a..7b65f1f55 100644 --- a/libs/auth/service_principal.go +++ b/libs/iamutil/service_principal.go @@ -1,15 +1,16 @@ -package auth +package iamutil import ( + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/google/uuid" ) -// Determines whether a given user name is a service principal. +// Determines whether a given user is a service principal. // This function uses a heuristic: if the user name is a UUID, then we assume // it's a service principal. Unfortunately, the service principal listing API is too // slow for our purposes. And the "users" and "service principals get" APIs // only allow access by workspace admins. -func IsServicePrincipal(userName string) bool { - _, err := uuid.Parse(userName) +func IsServicePrincipal(user *iam.User) bool { + _, err := uuid.Parse(user.UserName) return err == nil } diff --git a/libs/auth/service_principal_test.go b/libs/iamutil/service_principal_test.go similarity index 57% rename from libs/auth/service_principal_test.go rename to libs/iamutil/service_principal_test.go index 95e8ab5cb..07e076692 100644 --- a/libs/auth/service_principal_test.go +++ b/libs/iamutil/service_principal_test.go @@ -1,19 +1,24 @@ -package auth +package iamutil import ( "testing" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) func TestIsServicePrincipal_ValidUUID(t *testing.T) { - userId := "8b948b2e-d2b5-4b9e-8274-11b596f3b652" - isSP := IsServicePrincipal(userId) + user := &iam.User{ + UserName: "8b948b2e-d2b5-4b9e-8274-11b596f3b652", + } + isSP := IsServicePrincipal(user) assert.True(t, isSP, "Expected user ID to be recognized as a service principal") } func TestIsServicePrincipal_InvalidUUID(t *testing.T) { - userId := "invalid" - isSP := IsServicePrincipal(userId) + user := &iam.User{ + UserName: "invalid", + } + isSP := IsServicePrincipal(user) assert.False(t, isSP, "Expected user ID to not be recognized as a service principal") } diff --git a/libs/auth/user.go b/libs/iamutil/user.go similarity index 87% rename from libs/auth/user.go rename to libs/iamutil/user.go index c6aa974f3..53704dab6 100644 --- a/libs/auth/user.go +++ b/libs/iamutil/user.go @@ -1,4 +1,4 @@ -package auth +package iamutil import ( "strings" @@ -12,7 +12,7 @@ import ( // including dots, which are not supported in e.g. experiment names. func GetShortUserName(user *iam.User) string { name := user.UserName - if IsServicePrincipal(user.UserName) && user.DisplayName != "" { + if IsServicePrincipal(user) && user.DisplayName != "" { name = user.DisplayName } local, _, _ := strings.Cut(name, "@") diff --git a/libs/auth/user_test.go b/libs/iamutil/user_test.go similarity index 99% rename from libs/auth/user_test.go rename to libs/iamutil/user_test.go index 24b61464b..8aa863e6e 100644 --- a/libs/auth/user_test.go +++ b/libs/iamutil/user_test.go @@ -1,4 +1,4 @@ -package auth +package iamutil import ( "testing" diff --git a/libs/process/opts.go b/libs/process/opts.go index 9516e49ba..dd0667516 100644 --- a/libs/process/opts.go +++ b/libs/process/opts.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os/exec" + "sync" ) type execOption func(context.Context, *exec.Cmd) error @@ -69,10 +70,24 @@ func WithStdoutWriter(dst io.Writer) execOption { } } +// safeWriter is a writer that is safe to use concurrently. +// It serializes writes to the underlying writer. +type safeWriter struct { + w io.Writer + m sync.Mutex +} + +func (s *safeWriter) Write(p []byte) (n int, err error) { + s.m.Lock() + defer s.m.Unlock() + return s.w.Write(p) +} + func WithCombinedOutput(buf *bytes.Buffer) execOption { + sw := &safeWriter{w: buf} return func(_ context.Context, c *exec.Cmd) error { - c.Stdout = io.MultiWriter(buf, c.Stdout) - c.Stderr = io.MultiWriter(buf, c.Stderr) + c.Stdout = io.MultiWriter(sw, c.Stdout) + c.Stderr = io.MultiWriter(sw, c.Stderr) return nil } } diff --git a/libs/set/set.go b/libs/set/set.go index 4798ed092..4b6bc8766 100644 --- a/libs/set/set.go +++ b/libs/set/set.go @@ -14,6 +14,11 @@ type Set[T any] struct { data map[string]T } +// Values returns a slice of the set's values +func (s *Set[T]) Values() []T { + return maps.Values(s.data) +} + // NewSetFromF initialise a new set with initial values and a hash function // to define uniqueness of value func NewSetFromF[T any](values []T, f hashFunc[T]) *Set[T] { @@ -69,6 +74,11 @@ func (s *Set[T]) Has(item T) bool { return ok } +// Size returns the number of elements in the set +func (s *Set[T]) Size() int { + return len(s.data) +} + // Returns an iterable slice of values from set func (s *Set[T]) Iter() []T { return maps.Values(s.data) diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 88c73cc47..f25cbee4f 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -11,7 +11,7 @@ import ( "text/template" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/iam" @@ -119,7 +119,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return "", err } } - return auth.GetShortUserName(cachedUser), nil + return iamutil.GetShortUserName(cachedUser), nil }, // Get the default workspace catalog. If there is no default, or if // Unity Catalog is not enabled, return an empty string. @@ -128,8 +128,8 @@ func loadHelpers(ctx context.Context) template.FuncMap { metastore, err := w.Metastores.Current(ctx) if err != nil { var aerr *apierr.APIError - if errors.As(err, &aerr) && aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" { - // Workspace doesn't have a metastore assigned, ignore error + if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST") { + // Ignore: access denied or workspace doesn't have a metastore assigned empty_default := "" cachedCatalog = &empty_default return "", nil @@ -151,7 +151,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return false, err } } - result := auth.IsServicePrincipal(cachedUser.UserName) + result := iamutil.IsServicePrincipal(cachedUser) cachedIsServicePrincipal = &result return result, nil }, diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 50e5ad97c..1c6b8607e 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -3,7 +3,7 @@ resources: pipelines: {{.project_name}}_pipeline: name: {{.project_name}}_pipeline - {{- if eq default_catalog ""}} + {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name {{- else}}