Compare commits

...

6 Commits

Author SHA1 Message Date
Pieter Noordhuis fa25b92ba1
Add `libs/dyn/jsonsaver` (#1862)
## Changes

This package can be used to marshal a `dyn.Value` as JSON and retain the
ordering of keys in a mapping. Unlike the default behavior of
`json.Marshal,` the output does not encode HTML characters.

Otherwise, this is no different from using `JSON.Marshal` with
`v.AsAny().`

## Tests

Unit tests.
2024-10-29 15:32:33 +00:00
hectorcast-db 20fd401550
[Internal] Add test instructions for external contributors (#1863)
## Changes
Add test instructions for external contributors

## Tests
See Go Changes
https://github.com/databricks/databricks-sdk-go/pull/1073
2024-10-29 13:42:36 +00:00
Andrew Nester f018daf413
Use SetPermissions instead of UpdatePermissions when setting folder permissions based on top-level ones (#1822)
## Changes
Changed to use SetPermissions() to configure the permissions which
remove other permissions on deployment folders.

## Tests
Added unit test
2024-10-29 12:06:38 +00:00
Pieter Noordhuis 1896b09350
Add bundle generate variant for dashboards (#1847)
## Changes

This change adds the `databricks bundle generate dashboard` command.

The command requires one of three flags:
* `--existing-id` to generate configuration for an existing dashboard by
its ID.
* `--existing-path` to generate configuration for an existing dashboard
by its path in the workspace file system.
* `--resource` to generate the `.lvdash.json` dashboard file for a
dashboard that's already defined in the bundle. This option does not
impact the YAML configuration.

A typical workflow could look like this:
1. Use the command with `--existing-id` or `--existing-path` for a
starting point
2. Run `bundle deploy` to deploy a copy of the dashboard
3. Run `bundle open` to open this copy in your browser
4. Navigate to the draft mode and make modifications
5. Run `bundle generate dashboard` with `--resource` to update the local
`.lvdash.json` file with the remote modifications

## Tests

* Unit tests.
* Manual walkthrough as documented in the [Dashboard for NYC Taxi Trip
Analysis
example](https://github.com/databricks/bundle-examples/tree/main/knowledge_base/dashboard_nyc_taxi).
2024-10-29 11:51:59 +00:00
Pieter Noordhuis 11f75fd320
Add support for AI/BI dashboards (#1743)
## Changes

This change adds support for modeling [AI/BI dashboards][docs] in DABs.


[Example bundle configuration][example] is located in the
`bundle-examples` repository.

[docs]: https://docs.databricks.com/en/dashboards/index.html#dashboards
[example]:
https://github.com/databricks/bundle-examples/tree/main/knowledge_base/dashboard_nyc_taxi

## Tests

* Added unit tests for self-contained parts
* Integration test for e2e dashboard deployment and remote change
modification
2024-10-29 09:11:08 +00:00
dependabot[bot] 9c96f006c4
Bump github.com/fatih/color from 1.17.0 to 1.18.0 (#1861)
Bumps [github.com/fatih/color](https://github.com/fatih/color) from
1.17.0 to 1.18.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/fatih/color/releases">github.com/fatih/color's
releases</a>.</em></p>
<blockquote>
<h2>v1.18.0</h2>
<h2>What's Changed</h2>
<ul>
<li>Add RGB API support by <a
href="https://github.com/fatih"><code>@​fatih</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/225">fatih/color#225</a></li>
<li>Bump GitHub workflow actions by <a
href="https://github.com/deining"><code>@​deining</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/235">fatih/color#235</a></li>
<li>Bump golang.org/x/sys from 0.18.0 to 0.24.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/236">fatih/color#236</a></li>
<li>Bump golang.org/x/sys from 0.24.0 to 0.25.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/237">fatih/color#237</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a href="https://github.com/deining"><code>@​deining</code></a> made
their first contribution in <a
href="https://redirect.github.com/fatih/color/pull/235">fatih/color#235</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/fatih/color/compare/v1.17.0...v1.18.0">https://github.com/fatih/color/compare/v1.17.0...v1.18.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="1c8d870660"><code>1c8d870</code></a>
Update README.md</li>
<li><a
href="546c2d0f9a"><code>546c2d0</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/225">#225</a> from
fatih/add-rgb-api</li>
<li><a
href="1ff0f97908"><code>1ff0f97</code></a>
Apply suggestions from code review</li>
<li><a
href="5723903daa"><code>5723903</code></a>
Add RGB API support</li>
<li><a
href="f203fbcecb"><code>f203fbc</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/237">#237</a> from
fatih/dependabot/go_modules/golang.org/x/sys-0.25.0</li>
<li><a
href="60aa7fb483"><code>60aa7fb</code></a>
Bump golang.org/x/sys from 0.24.0 to 0.25.0</li>
<li><a
href="741c2f4087"><code>741c2f4</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/236">#236</a> from
fatih/dependabot/go_modules/golang.org/x/sys-0.24.0</li>
<li><a
href="0d24b42a27"><code>0d24b42</code></a>
Bump golang.org/x/sys from 0.18.0 to 0.24.0</li>
<li><a
href="cb154c0218"><code>cb154c0</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/235">#235</a> from
deining/fix-typo</li>
<li><a
href="9b9653e8ce"><code>9b9653e</code></a>
Bump GitHub workflow actions</li>
<li>See full diff in <a
href="https://github.com/fatih/color/compare/v1.17.0...v1.18.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/fatih/color&package-manager=go_modules&previous-version=1.17.0&new-version=1.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-10-29 08:57:10 +00:00
48 changed files with 2509 additions and 41 deletions

114
.github/workflows/external-message.yml vendored Normal file
View File

@ -0,0 +1,114 @@
name: PR Comment
# WARNING:
# THIS WORKFLOW ALWAYS RUNS FOR EXTERNAL CONTRIBUTORS WITHOUT ANY APPROVAL.
# THIS WORKFLOW RUNS FROM MAIN BRANCH, NOT FROM THE PR BRANCH.
# DO NOT PULL THE PR OR EXECUTE ANY CODE FROM THE PR.
on:
pull_request_target:
types: [opened, reopened, synchronize]
branches:
- main
jobs:
comment-on-pr:
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
# NOTE: The following checks may not be accurate depending on Org or Repo settings.
- name: Check user and potential secret access
id: check-secrets-access
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
USER_LOGIN="${{ github.event.pull_request.user.login }}"
REPO_OWNER="${{ github.repository_owner }}"
REPO_NAME="${{ github.event.repository.name }}"
echo "Pull request opened by: $USER_LOGIN"
# Check if PR is from a fork
IS_FORK=$([[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]] && echo "true" || echo "false")
HAS_ACCESS="false"
# Check user's permission level on the repository
USER_PERMISSION=$(gh api repos/$REPO_OWNER/$REPO_NAME/collaborators/$USER_LOGIN/permission --jq '.permission')
if [[ "$USER_PERMISSION" == "admin" || "$USER_PERMISSION" == "write" ]]; then
HAS_ACCESS="true"
elif [[ "$USER_PERMISSION" == "read" ]]; then
# For read access, we need to check if the user has been explicitly granted secret access
# This information is not directly available via API, so we'll make an assumption
# that read access does not imply secret access
HAS_ACCESS="false"
fi
# Check if repo owner is an organization
IS_ORG=$(gh api users/$REPO_OWNER --jq '.type == "Organization"')
if [[ "$IS_ORG" == "true" && "$HAS_ACCESS" == "false" ]]; then
# Check if user is a member of any team with write or admin access to the repo
TEAMS_WITH_ACCESS=$(gh api repos/$REPO_OWNER/$REPO_NAME/teams --jq '.[] | select(.permission == "push" or .permission == "admin") | .slug')
for team in $TEAMS_WITH_ACCESS; do
IS_TEAM_MEMBER=$(gh api orgs/$REPO_OWNER/teams/$team/memberships/$USER_LOGIN --silent && echo "true" || echo "false")
if [[ "$IS_TEAM_MEMBER" == "true" ]]; then
HAS_ACCESS="true"
break
fi
done
fi
# If it's a fork, set HAS_ACCESS to false regardless of other checks
if [[ "$IS_FORK" == "true" ]]; then
HAS_ACCESS="false"
fi
echo "has_secrets_access=$HAS_ACCESS" >> $GITHUB_OUTPUT
if [[ "$HAS_ACCESS" == "true" ]]; then
echo "User $USER_LOGIN likely has access to secrets"
else
echo "User $USER_LOGIN likely does not have access to secrets"
fi
- uses: actions/checkout@v4
- name: Delete old comments
if: steps.check-secrets-access.outputs.has_secrets_access != 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Delete previous comment if it exists
previous_comment_ids=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
--jq '.[] | select(.body | startswith("<!-- INTEGRATION_TESTS -->")) | .id')
echo "Previous comment IDs: $previous_comment_ids"
# Iterate over each comment ID and delete the comment
if [ ! -z "$previous_comment_ids" ]; then
echo "$previous_comment_ids" | while read -r comment_id; do
echo "Deleting comment with ID: $comment_id"
gh api "repos/${{ github.repository }}/issues/comments/$comment_id" -X DELETE
done
fi
- name: Comment on PR
if: steps.check-secrets-access.outputs.has_secrets_access != 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
run: |
gh pr comment ${{ github.event.pull_request.number }} --body \
"<!-- INTEGRATION_TESTS -->
Run integration tests manually:
[go/deco-tests-run/cli](https://go/deco-tests-run/cli)
Inputs:
* PR number: ${{github.event.pull_request.number}}
* Commit SHA: \`${{ env.COMMIT_SHA }}\`
Checks will be approved automatically on success.
"

View File

@ -9,9 +9,26 @@ on:
jobs:
trigger-tests:
if: github.event_name == 'pull_request'
check-token:
runs-on: ubuntu-latest
outputs:
has_token: ${{ steps.set-token-status.outputs.has_token }}
steps:
- name: Check if GITHUB_TOKEN is set
id: set-token-status
run: |
if [ -z "${{ secrets.GITHUB_TOKEN }}" ]; then
echo "GITHUB_TOKEN is empty. User has no access to tokens."
echo "::set-output name=has_token::false"
else
echo "GITHUB_TOKEN is set. User has no access to tokens."
echo "::set-output name=has_token::true"
fi
trigger-tests:
runs-on: ubuntu-latest
needs: check-token
if: github.event_name == 'pull_request' && needs.check-token.outputs.has_token == 'true'
environment: "test-trigger-is"
steps:

View File

@ -0,0 +1,18 @@
package generate
import (
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/dashboards"
)
func ConvertDashboardToValue(dashboard *dashboards.Dashboard, filePath string) (dyn.Value, error) {
// The majority of fields of the dashboard struct are read-only.
// We copy the relevant fields manually.
dv := map[string]dyn.Value{
"display_name": dyn.NewValue(dashboard.DisplayName, []dyn.Location{{Line: 1}}),
"warehouse_id": dyn.NewValue(dashboard.WarehouseId, []dyn.Location{{Line: 2}}),
"file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}),
}
return dyn.V(dv), nil
}

View File

@ -212,6 +212,15 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
}
}
// Dashboards: Prefix
for key, dashboard := range r.Dashboards {
if dashboard == nil || dashboard.CreateDashboardRequest == nil {
diags = diags.Extend(diag.Errorf("dashboard %s s is not defined", key))
continue
}
dashboard.DisplayName = prefix + dashboard.DisplayName
}
return diags
}

View File

@ -0,0 +1,70 @@
package mutator
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
)
type configureDashboardDefaults struct{}
func ConfigureDashboardDefaults() bundle.Mutator {
return &configureDashboardDefaults{}
}
func (m *configureDashboardDefaults) Name() string {
return "ConfigureDashboardDefaults"
}
func (m *configureDashboardDefaults) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
var diags diag.Diagnostics
pattern := dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("dashboards"),
dyn.AnyKey(),
)
// Configure defaults for all dashboards.
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
var err error
v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("parent_path")), dyn.V(b.Config.Workspace.ResourcePath))
if err != nil {
return dyn.InvalidValue, err
}
v, err = setIfNotExists(v, dyn.NewPath(dyn.Key("embed_credentials")), dyn.V(false))
if err != nil {
return dyn.InvalidValue, err
}
return v, nil
})
})
diags = diags.Extend(diag.FromErr(err))
return diags
}
func setIfNotExists(v dyn.Value, path dyn.Path, defaultValue dyn.Value) (dyn.Value, error) {
// Get the field at the specified path (if set).
_, err := dyn.GetByPath(v, path)
switch {
case dyn.IsNoSuchKeyError(err):
// OK, we'll set the default value.
break
case dyn.IsCannotTraverseNilError(err):
// Cannot traverse the value, skip it.
return v, nil
case err == nil:
// The field is set, skip it.
return v, nil
default:
// Return the error.
return v, err
}
// Set the field at the specified path.
return dyn.SetByPath(v, path, defaultValue)
}

View File

@ -0,0 +1,130 @@
package mutator_test
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConfigureDashboardDefaultsParentPath(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
ResourcePath: "/foo/bar",
},
Resources: config.Resources{
Dashboards: map[string]*resources.Dashboard{
"d1": {
// Empty string is skipped.
// See below for how it is set.
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
ParentPath: "",
},
},
"d2": {
// Non-empty string is skipped.
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
ParentPath: "already-set",
},
},
"d3": {
// No parent path set.
},
"d4": nil,
},
},
},
}
// We can't set an empty string in the typed configuration.
// Do it on the dyn.Value directly.
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
return dyn.Set(v, "resources.dashboards.d1.parent_path", dyn.V(""))
})
diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults())
require.NoError(t, diags.Error())
var v dyn.Value
var err error
// Set to empty string; unchanged.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.parent_path")
if assert.NoError(t, err) {
assert.Equal(t, "", v.MustString())
}
// Set to "already-set"; unchanged.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.parent_path")
if assert.NoError(t, err) {
assert.Equal(t, "already-set", v.MustString())
}
// Not set; now set to the workspace resource path.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.parent_path")
if assert.NoError(t, err) {
assert.Equal(t, "/foo/bar", v.MustString())
}
// No valid dashboard; no change.
_, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.parent_path")
assert.True(t, dyn.IsCannotTraverseNilError(err))
}
func TestConfigureDashboardDefaultsEmbedCredentials(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Dashboards: map[string]*resources.Dashboard{
"d1": {
EmbedCredentials: true,
},
"d2": {
EmbedCredentials: false,
},
"d3": {
// No parent path set.
},
"d4": nil,
},
},
},
}
diags := bundle.Apply(context.Background(), b, mutator.ConfigureDashboardDefaults())
require.NoError(t, diags.Error())
var v dyn.Value
var err error
// Set to true; still true.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d1.embed_credentials")
if assert.NoError(t, err) {
assert.Equal(t, true, v.MustBool())
}
// Set to false; still false.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d2.embed_credentials")
if assert.NoError(t, err) {
assert.Equal(t, false, v.MustBool())
}
// Not set; now false.
v, err = dyn.Get(b.Config.Value(), "resources.dashboards.d3.embed_credentials")
if assert.NoError(t, err) {
assert.Equal(t, false, v.MustBool())
}
// No valid dashboard; no change.
_, err = dyn.Get(b.Config.Value(), "resources.dashboards.d4.embed_credentials")
assert.True(t, dyn.IsCannotTraverseNilError(err))
}

View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
@ -85,6 +86,14 @@ func TestInitializeURLs(t *testing.T) {
},
},
},
Dashboards: map[string]*resources.Dashboard{
"dashboard1": {
ID: "01ef8d56871e1d50ae30ce7375e42478",
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "My special dashboard",
},
},
},
},
},
}
@ -99,6 +108,7 @@ func TestInitializeURLs(t *testing.T) {
"qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456",
"schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456",
"cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456",
"dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456",
}
initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/")

View File

@ -14,6 +14,7 @@ import (
sdkconfig "github.com/databricks/databricks-sdk-go/config"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/ml"
@ -123,6 +124,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
Clusters: map[string]*resources.Cluster{
"cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}},
},
Dashboards: map[string]*resources.Dashboard{
"dashboard1": {
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "dashboard1",
},
},
},
},
},
// Use AWS implementation for testing.
@ -184,6 +192,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
// Clusters
assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName)
// Dashboards
assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName)
}
func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) {

View File

@ -110,6 +110,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
))
}
// Dashboards do not support run_as in the API.
if len(b.Config.Resources.Dashboards) > 0 {
diags = diags.Extend(reportRunAsNotSupported(
"dashboards",
b.Config.GetLocation("resources.dashboards"),
b.Config.Workspace.CurrentUser.UserName,
identity,
))
}
return diags
}

View File

@ -33,6 +33,7 @@ func allResourceTypes(t *testing.T) []string {
// also update this check when adding a new resource
require.Equal(t, []string{
"clusters",
"dashboards",
"experiments",
"jobs",
"model_serving_endpoints",
@ -188,6 +189,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) {
Config: *r,
}
diags := bundle.Apply(context.Background(), b, SetRunAs())
require.Error(t, diags.Error())
assert.Contains(t, diags.Error().Error(), "do not support a setting a run_as user that is different from the owner.\n"+
"Current identity: alice. Run as identity: bob.\n"+
"See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.", rt)

View File

@ -162,6 +162,20 @@ func (t *translateContext) translateNoOp(literal, localFullPath, localRelPath, r
return localRelPath, nil
}
func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
info, err := t.b.SyncRoot.Stat(localRelPath)
if errors.Is(err, fs.ErrNotExist) {
return "", fmt.Errorf("file %s not found", literal)
}
if err != nil {
return "", fmt.Errorf("unable to determine if %s is a file: %w", localFullPath, err)
}
if info.IsDir() {
return "", fmt.Errorf("expected %s to be a file but found a directory", literal)
}
return localFullPath, nil
}
func (t *translateContext) translateNoOpWithPrefix(literal, localFullPath, localRelPath, remotePath string) (string, error) {
if !strings.HasPrefix(localRelPath, ".") {
localRelPath = "." + string(filepath.Separator) + localRelPath
@ -215,6 +229,7 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
t.applyJobTranslations,
t.applyPipelineTranslations,
t.applyArtifactTranslations,
t.applyDashboardTranslations,
} {
v, err = fn(v)
if err != nil {

View File

@ -0,0 +1,28 @@
package mutator
import (
"fmt"
"github.com/databricks/cli/libs/dyn"
)
func (t *translateContext) applyDashboardTranslations(v dyn.Value) (dyn.Value, error) {
// Convert the `file_path` field to a local absolute path.
// We load the file at this path and use its contents for the dashboard contents.
pattern := dyn.NewPattern(
dyn.Key("resources"),
dyn.Key("dashboards"),
dyn.AnyKey(),
dyn.Key("file_path"),
)
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
key := p[2].Key()
dir, err := v.Location().Directory()
if err != nil {
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for dashboard %s: %w", key, err)
}
return t.rewriteRelativeTo(p, v, t.retainLocalAbsoluteFilePath, dir, "")
})
}

View File

@ -21,6 +21,7 @@ type Resources struct {
QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"`
Schemas map[string]*resources.Schema `json:"schemas,omitempty"`
Clusters map[string]*resources.Cluster `json:"clusters,omitempty"`
Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"`
}
type ConfigResource interface {
@ -77,6 +78,7 @@ func (r *Resources) AllResources() []ResourceGroup {
collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors),
collectResourceMap(descriptions["schemas"], r.Schemas),
collectResourceMap(descriptions["clusters"], r.Clusters),
collectResourceMap(descriptions["dashboards"], r.Dashboards),
}
}
@ -175,5 +177,11 @@ func SupportedResources() map[string]ResourceDescription {
SingularTitle: "Cluster",
PluralTitle: "Clusters",
},
"dashboards": {
SingularName: "dashboard",
PluralName: "dashboards",
SingularTitle: "Dashboard",
PluralTitle: "Dashboards",
},
}
}

View File

@ -0,0 +1,81 @@
package resources
import (
"context"
"fmt"
"net/url"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/dashboards"
)
type Dashboard struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`
*dashboards.CreateDashboardRequest
// =========================
// === Additional fields ===
// =========================
// SerializedDashboard holds the contents of the dashboard in serialized JSON form.
// We override the field's type from the SDK struct here to allow for inlining as YAML.
// If the value is a string, it is used as is.
// If it is not a string, its contents is marshalled as JSON.
SerializedDashboard any `json:"serialized_dashboard,omitempty"`
// EmbedCredentials is a flag to indicate if the publisher's credentials should
// be embedded in the published dashboard. These embedded credentials will be used
// to execute the published dashboard's queries.
//
// Defaults to false if not set.
EmbedCredentials bool `json:"embed_credentials,omitempty"`
// FilePath points to the local `.lvdash.json` file containing the dashboard definition.
FilePath string `json:"file_path,omitempty"`
}
func (r *Dashboard) UnmarshalJSON(b []byte) error {
return marshal.Unmarshal(b, r)
}
func (r Dashboard) MarshalJSON() ([]byte, error) {
return marshal.Marshal(r)
}
func (*Dashboard) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.Lakeview.Get(ctx, dashboards.GetDashboardRequest{
DashboardId: id,
})
if err != nil {
log.Debugf(ctx, "dashboard %s does not exist", id)
return false, err
}
return true, nil
}
func (*Dashboard) TerraformResourceName() string {
return "databricks_dashboard"
}
func (r *Dashboard) InitializeURL(baseURL url.URL) {
if r.ID == "" {
return
}
baseURL.Path = fmt.Sprintf("dashboardsv3/%s/published", r.ID)
r.URL = baseURL.String()
}
func (r *Dashboard) GetName() string {
return r.DisplayName
}
func (r *Dashboard) GetURL() string {
return r.URL
}

View File

@ -4,10 +4,9 @@ import (
"context"
"fmt"
"path"
"strings"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/libraries"
"github.com/databricks/cli/bundle/paths"
"github.com/databricks/cli/bundle/permissions"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/apierr"
@ -24,37 +23,12 @@ func (f *folderPermissions) Apply(ctx context.Context, b bundle.ReadOnlyBundle)
return nil
}
rootPath := b.Config().Workspace.RootPath
paths := []string{}
if !libraries.IsVolumesPath(rootPath) && !libraries.IsWorkspaceSharedPath(rootPath) {
paths = append(paths, rootPath)
}
if !strings.HasSuffix(rootPath, "/") {
rootPath += "/"
}
for _, p := range []string{
b.Config().Workspace.ArtifactPath,
b.Config().Workspace.FilePath,
b.Config().Workspace.StatePath,
b.Config().Workspace.ResourcePath,
} {
if libraries.IsWorkspaceSharedPath(p) || libraries.IsVolumesPath(p) {
continue
}
if strings.HasPrefix(p, rootPath) {
continue
}
paths = append(paths, p)
}
bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config().Workspace)
var diags diag.Diagnostics
g, ctx := errgroup.WithContext(ctx)
results := make([]diag.Diagnostics, len(paths))
for i, p := range paths {
results := make([]diag.Diagnostics, len(bundlePaths))
for i, p := range bundlePaths {
g.Go(func() error {
results[i] = checkFolderPermission(ctx, b, p)
return nil

View File

@ -0,0 +1,117 @@
package terraform
import (
"context"
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
tfjson "github.com/hashicorp/terraform-json"
)
type dashboardState struct {
Name string
ID string
ETag string
}
func collectDashboardsFromState(ctx context.Context, b *bundle.Bundle) ([]dashboardState, error) {
state, err := ParseResourcesState(ctx, b)
if err != nil && state == nil {
return nil, err
}
var dashboards []dashboardState
for _, resource := range state.Resources {
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
for _, instance := range resource.Instances {
switch resource.Type {
case "databricks_dashboard":
dashboards = append(dashboards, dashboardState{
Name: resource.Name,
ID: instance.Attributes.ID,
ETag: instance.Attributes.ETag,
})
}
}
}
return dashboards, nil
}
type checkDashboardsModifiedRemotely struct {
}
func (l *checkDashboardsModifiedRemotely) Name() string {
return "CheckDashboardsModifiedRemotely"
}
func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// This mutator is relevant only if the bundle includes dashboards.
if len(b.Config.Resources.Dashboards) == 0 {
return nil
}
// If the user has forced the deployment, skip this check.
if b.Config.Bundle.Force {
return nil
}
dashboards, err := collectDashboardsFromState(ctx, b)
if err != nil {
return diag.FromErr(err)
}
var diags diag.Diagnostics
for _, dashboard := range dashboards {
// Skip dashboards that are not defined in the bundle.
// These will be destroyed upon deployment.
if _, ok := b.Config.Resources.Dashboards[dashboard.Name]; !ok {
continue
}
path := dyn.MustPathFromString(fmt.Sprintf("resources.dashboards.%s", dashboard.Name))
loc := b.Config.GetLocation(path.String())
actual, err := b.WorkspaceClient().Lakeview.GetByDashboardId(ctx, dashboard.ID)
if err != nil {
diags = diags.Append(diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("failed to get dashboard %q", dashboard.Name),
Detail: err.Error(),
Paths: []dyn.Path{path},
Locations: []dyn.Location{loc},
})
continue
}
// If the ETag is the same, the dashboard has not been modified.
if actual.Etag == dashboard.ETag {
continue
}
diags = diags.Append(diag.Diagnostic{
Severity: diag.Error,
Summary: fmt.Sprintf("dashboard %q has been modified remotely", dashboard.Name),
Detail: "" +
"This dashboard has been modified remotely since the last bundle deployment.\n" +
"These modifications are untracked and will be overwritten on deploy.\n" +
"\n" +
"Make sure that the local dashboard definition matches what you intend to deploy\n" +
"before proceeding with the deployment.\n" +
"\n" +
"Run `databricks bundle deploy --force` to bypass this error." +
"",
Paths: []dyn.Path{path},
Locations: []dyn.Location{loc},
})
}
return diags
}
func CheckDashboardsModifiedRemotely() *checkDashboardsModifiedRemotely {
return &checkDashboardsModifiedRemotely{}
}

View File

@ -0,0 +1,191 @@
package terraform
import (
"context"
"fmt"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func mockDashboardBundle(t *testing.T) *bundle.Bundle {
dir := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: dir,
Config: config.Root{
Bundle: config.Bundle{
Target: "test",
},
Resources: config.Resources{
Dashboards: map[string]*resources.Dashboard{
"dash1": {
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "My Special Dashboard",
},
},
},
},
},
}
return b
}
func TestCheckDashboardsModifiedRemotely_NoDashboards(t *testing.T) {
dir := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: dir,
Config: config.Root{
Bundle: config.Bundle{
Target: "test",
},
Resources: config.Resources{},
},
}
diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely())
assert.Empty(t, diags)
}
func TestCheckDashboardsModifiedRemotely_FirstDeployment(t *testing.T) {
b := mockDashboardBundle(t)
diags := bundle.Apply(context.Background(), b, CheckDashboardsModifiedRemotely())
assert.Empty(t, diags)
}
func TestCheckDashboardsModifiedRemotely_ExistingStateNoChange(t *testing.T) {
ctx := context.Background()
b := mockDashboardBundle(t)
writeFakeDashboardState(t, ctx, b)
// Mock the call to the API.
m := mocks.NewMockWorkspaceClient(t)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().
GetByDashboardId(mock.Anything, "id1").
Return(&dashboards.Dashboard{
DisplayName: "My Special Dashboard",
Etag: "1000",
}, nil).
Once()
b.SetWorkpaceClient(m.WorkspaceClient)
// No changes, so no diags.
diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely())
assert.Empty(t, diags)
}
func TestCheckDashboardsModifiedRemotely_ExistingStateChange(t *testing.T) {
ctx := context.Background()
b := mockDashboardBundle(t)
writeFakeDashboardState(t, ctx, b)
// Mock the call to the API.
m := mocks.NewMockWorkspaceClient(t)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().
GetByDashboardId(mock.Anything, "id1").
Return(&dashboards.Dashboard{
DisplayName: "My Special Dashboard",
Etag: "1234",
}, nil).
Once()
b.SetWorkpaceClient(m.WorkspaceClient)
// The dashboard has changed, so expect an error.
diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely())
if assert.Len(t, diags, 1) {
assert.Equal(t, diag.Error, diags[0].Severity)
assert.Equal(t, `dashboard "dash1" has been modified remotely`, diags[0].Summary)
}
}
func TestCheckDashboardsModifiedRemotely_ExistingStateFailureToGet(t *testing.T) {
ctx := context.Background()
b := mockDashboardBundle(t)
writeFakeDashboardState(t, ctx, b)
// Mock the call to the API.
m := mocks.NewMockWorkspaceClient(t)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().
GetByDashboardId(mock.Anything, "id1").
Return(nil, fmt.Errorf("failure")).
Once()
b.SetWorkpaceClient(m.WorkspaceClient)
// Unable to get the dashboard, so expect an error.
diags := bundle.Apply(ctx, b, CheckDashboardsModifiedRemotely())
if assert.Len(t, diags, 1) {
assert.Equal(t, diag.Error, diags[0].Severity)
assert.Equal(t, `failed to get dashboard "dash1"`, diags[0].Summary)
}
}
func writeFakeDashboardState(t *testing.T, ctx context.Context, b *bundle.Bundle) {
tfDir, err := Dir(ctx, b)
require.NoError(t, err)
// Write fake state file.
testutil.WriteFile(t, `
{
"version": 4,
"terraform_version": "1.5.5",
"resources": [
{
"mode": "managed",
"type": "databricks_dashboard",
"name": "dash1",
"instances": [
{
"schema_version": 0,
"attributes": {
"etag": "1000",
"id": "id1"
}
}
]
},
{
"mode": "managed",
"type": "databricks_job",
"name": "job",
"instances": [
{
"schema_version": 0,
"attributes": {
"id": "1234"
}
}
]
},
{
"mode": "managed",
"type": "databricks_dashboard",
"name": "dash2",
"instances": [
{
"schema_version": 0,
"attributes": {
"etag": "1001",
"id": "id2"
}
}
]
}
]
}
`, filepath.Join(tfDir, TerraformStateFileName))
}

View File

@ -176,6 +176,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
}
cur.ID = instance.Attributes.ID
config.Resources.Clusters[resource.Name] = cur
case "databricks_dashboard":
if config.Resources.Dashboards == nil {
config.Resources.Dashboards = make(map[string]*resources.Dashboard)
}
cur := config.Resources.Dashboards[resource.Name]
if cur == nil {
cur = &resources.Dashboard{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Dashboards[resource.Name] = cur
case "databricks_permissions":
case "databricks_grants":
// Ignore; no need to pull these back into the configuration.
@ -230,6 +240,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
for _, src := range config.Resources.Dashboards {
if src.ModifiedStatus == "" && src.ID == "" {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
return nil
}

View File

@ -12,6 +12,7 @@ import (
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
@ -677,6 +678,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_dashboard",
Mode: "managed",
Name: "test_dashboard",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
},
}
err := TerraformToBundle(&tfState, &config)
@ -709,6 +718,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus)
assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -778,6 +790,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
},
},
},
Dashboards: map[string]*resources.Dashboard{
"test_dashboard": {
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "test_dashboard",
},
},
},
},
}
var tfState = resourcesState{
@ -813,6 +832,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus)
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -927,6 +949,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
},
},
},
Dashboards: map[string]*resources.Dashboard{
"test_dashboard": {
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "test_dashboard",
},
},
"test_dashboard_new": {
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "test_dashboard_new",
},
},
},
},
}
var tfState = resourcesState{
@ -1075,6 +1109,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_dashboard",
Mode: "managed",
Name: "test_dashboard",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_dashboard",
Mode: "managed",
Name: "test_dashboard_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
},
}
err := TerraformToBundle(&tfState, &config)
@ -1143,6 +1193,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus)
assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID)
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ModifiedStatus)
assert.Equal(t, "2", config.Resources.Dashboards["test_dashboard_old"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard_old"].ModifiedStatus)
assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}

View File

@ -60,6 +60,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...)
case dyn.Key("clusters"):
path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...)
case dyn.Key("dashboards"):
path = dyn.NewPath(dyn.Key("databricks_dashboard")).Append(path[2:]...)
default:
// Trigger "key not found" for unknown resource types.
return dyn.GetByPath(root, path)

View File

@ -32,6 +32,7 @@ func TestInterpolate(t *testing.T) {
"other_registered_model": "${resources.registered_models.other_registered_model.id}",
"other_schema": "${resources.schemas.other_schema.id}",
"other_cluster": "${resources.clusters.other_cluster.id}",
"other_dashboard": "${resources.dashboards.other_dashboard.id}",
},
Tasks: []jobs.Task{
{
@ -69,6 +70,7 @@ func TestInterpolate(t *testing.T) {
assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"])
assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"])
assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"])
assert.Equal(t, "${databricks_dashboard.other_dashboard.id}", j.Tags["other_dashboard"])
m := b.Config.Resources.Models["my_model"]
assert.Equal(t, "my_model", m.Model.Name)

View File

@ -0,0 +1,109 @@
package tfdyn
import (
"context"
"encoding/json"
"fmt"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/cli/libs/log"
)
const (
filePathFieldName = "file_path"
serializedDashboardFieldName = "serialized_dashboard"
)
// Marshal "serialized_dashboard" as JSON if it is set in the input but not in the output.
func marshalSerializedDashboard(vin dyn.Value, vout dyn.Value) (dyn.Value, error) {
// Skip if the "serialized_dashboard" field is already set.
if v := vout.Get(serializedDashboardFieldName); v.IsValid() {
return vout, nil
}
// Skip if the "serialized_dashboard" field on the input is not set.
v := vin.Get(serializedDashboardFieldName)
if !v.IsValid() {
return vout, nil
}
// Marshal the "serialized_dashboard" field as JSON.
data, err := json.Marshal(v.AsAny())
if err != nil {
return dyn.InvalidValue, fmt.Errorf("failed to marshal serialized_dashboard: %w", err)
}
// Set the "serialized_dashboard" field on the output.
return dyn.Set(vout, serializedDashboardFieldName, dyn.V(string(data)))
}
func convertDashboardResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
var err error
// Normalize the output value to the target schema.
vout, diags := convert.Normalize(schema.ResourceDashboard{}, vin)
for _, diag := range diags {
log.Debugf(ctx, "dashboard normalization diagnostic: %s", diag.Summary)
}
// Include "serialized_dashboard" field if "file_path" is set.
// Note: the Terraform resource supports "file_path" natively, but its
// change detection mechanism doesn't work as expected at the time of writing (Sep 30).
if path, ok := vout.Get(filePathFieldName).AsString(); ok {
vout, err = dyn.Set(vout, serializedDashboardFieldName, dyn.V(fmt.Sprintf("${file(%q)}", path)))
if err != nil {
return dyn.InvalidValue, fmt.Errorf("failed to set serialized_dashboard: %w", err)
}
// Drop the "file_path" field. It is mutually exclusive with "serialized_dashboard".
vout, err = dyn.Walk(vout, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
switch len(p) {
case 0:
return v, nil
case 1:
if p[0] == dyn.Key(filePathFieldName) {
return v, dyn.ErrDrop
}
}
// Skip everything else.
return v, dyn.ErrSkip
})
if err != nil {
return dyn.InvalidValue, fmt.Errorf("failed to drop file_path: %w", err)
}
}
// Marshal "serialized_dashboard" as JSON if it is set in the input but not in the output.
vout, err = marshalSerializedDashboard(vin, vout)
if err != nil {
return dyn.InvalidValue, err
}
return vout, nil
}
type dashboardConverter struct{}
func (dashboardConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
vout, err := convertDashboardResource(ctx, vin)
if err != nil {
return err
}
// Add the converted resource to the output.
out.Dashboard[key] = vout.AsAny()
// Configure permissions for this resource.
if permissions := convertPermissionsResource(ctx, vin); permissions != nil {
permissions.DashboardId = fmt.Sprintf("${databricks_dashboard.%s.id}", key)
out.Permissions["dashboard_"+key] = permissions
}
return nil
}
func init() {
registerConverter("dashboards", dashboardConverter{})
}

View File

@ -0,0 +1,153 @@
package tfdyn
import (
"context"
"testing"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertDashboard(t *testing.T) {
var src = resources.Dashboard{
CreateDashboardRequest: &dashboards.CreateDashboardRequest{
DisplayName: "my dashboard",
WarehouseId: "f00dcafe",
ParentPath: "/some/path",
},
EmbedCredentials: true,
Permissions: []resources.Permission{
{
Level: "CAN_VIEW",
UserName: "jane@doe.com",
},
},
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
require.NoError(t, err)
// Assert equality on the job
assert.Equal(t, map[string]any{
"display_name": "my dashboard",
"warehouse_id": "f00dcafe",
"parent_path": "/some/path",
"embed_credentials": true,
}, out.Dashboard["my_dashboard"])
// Assert equality on the permissions
assert.Equal(t, &schema.ResourcePermissions{
DashboardId: "${databricks_dashboard.my_dashboard.id}",
AccessControl: []schema.ResourcePermissionsAccessControl{
{
PermissionLevel: "CAN_VIEW",
UserName: "jane@doe.com",
},
},
}, out.Permissions["dashboard_my_dashboard"])
}
func TestConvertDashboardFilePath(t *testing.T) {
var src = resources.Dashboard{
FilePath: "some/path",
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
require.NoError(t, err)
// Assert that the "serialized_dashboard" is included.
assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
"serialized_dashboard": "${file(\"some/path\")}",
})
// Assert that the "file_path" doesn't carry over.
assert.NotSubset(t, out.Dashboard["my_dashboard"], map[string]any{
"file_path": "some/path",
})
}
func TestConvertDashboardFilePathQuoted(t *testing.T) {
var src = resources.Dashboard{
FilePath: `C:\foo\bar\baz\dashboard.lvdash.json`,
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
require.NoError(t, err)
// Assert that the "serialized_dashboard" is included.
assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
"serialized_dashboard": `${file("C:\\foo\\bar\\baz\\dashboard.lvdash.json")}`,
})
// Assert that the "file_path" doesn't carry over.
assert.NotSubset(t, out.Dashboard["my_dashboard"], map[string]any{
"file_path": `C:\foo\bar\baz\dashboard.lvdash.json`,
})
}
func TestConvertDashboardSerializedDashboardString(t *testing.T) {
var src = resources.Dashboard{
SerializedDashboard: `{ "json": true }`,
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
require.NoError(t, err)
// Assert that the "serialized_dashboard" is included.
assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
"serialized_dashboard": `{ "json": true }`,
})
}
func TestConvertDashboardSerializedDashboardAny(t *testing.T) {
var src = resources.Dashboard{
SerializedDashboard: map[string]any{
"pages": []map[string]any{
{
"displayName": "New Page",
"layout": []map[string]any{},
},
},
},
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = dashboardConverter{}.Convert(ctx, "my_dashboard", vin, out)
require.NoError(t, err)
// Assert that the "serialized_dashboard" is included.
assert.Subset(t, out.Dashboard["my_dashboard"], map[string]any{
"serialized_dashboard": `{"pages":[{"displayName":"New Page","layout":[]}]}`,
})
}

View File

@ -13,7 +13,7 @@ import (
// Partial representation of the Terraform state file format.
// We are only interested global version and serial numbers,
// plus resource types, names, modes, and ids.
// plus resource types, names, modes, IDs, and ETags (for dashboards).
type resourcesState struct {
Version int `json:"version"`
Resources []stateResource `json:"resources"`
@ -33,7 +33,8 @@ type stateResourceInstance struct {
}
type stateInstanceAttributes struct {
ID string `json:"id"`
ID string `json:"id"`
ETag string `json:"etag,omitempty"`
}
func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) {

View File

@ -0,0 +1,20 @@
package bundletest
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/require"
)
func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) {
diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
err := b.Config.Mutate(f)
require.NoError(t, err)
return nil
})
require.NoError(t, diags.Error())
}

View File

@ -38,6 +38,7 @@ func IsWorkspaceLibrary(library *compute.Library) bool {
}
// IsVolumesPath returns true if the specified path indicates that
// it should be interpreted as a Databricks Volumes path.
func IsVolumesPath(path string) bool {
return strings.HasPrefix(path, "/Volumes/")
}

39
bundle/paths/paths.go Normal file
View File

@ -0,0 +1,39 @@
package paths
import (
"strings"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/libraries"
)
func CollectUniqueWorkspacePathPrefixes(workspace config.Workspace) []string {
rootPath := workspace.RootPath
paths := []string{}
if !libraries.IsVolumesPath(rootPath) && !libraries.IsWorkspaceSharedPath(rootPath) {
paths = append(paths, rootPath)
}
if !strings.HasSuffix(rootPath, "/") {
rootPath += "/"
}
for _, p := range []string{
workspace.ArtifactPath,
workspace.FilePath,
workspace.StatePath,
workspace.ResourcePath,
} {
if libraries.IsWorkspaceSharedPath(p) || libraries.IsVolumesPath(p) {
continue
}
if strings.HasPrefix(p, rootPath) {
continue
}
paths = append(paths, p)
}
return paths
}

View File

@ -39,6 +39,10 @@ var levelsMap = map[string](map[string]string){
CAN_VIEW: "CAN_VIEW",
CAN_RUN: "CAN_QUERY",
},
"dashboards": {
CAN_MANAGE: "CAN_MANAGE",
CAN_VIEW: "CAN_READ",
},
}
type bundlePermissions struct{}

View File

@ -5,8 +5,10 @@ import (
"fmt"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/paths"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/service/workspace"
"golang.org/x/sync/errgroup"
)
type workspaceRootPermissions struct {
@ -52,16 +54,30 @@ func giveAccessForWorkspaceRoot(ctx context.Context, b *bundle.Bundle) error {
}
w := b.WorkspaceClient().Workspace
obj, err := w.GetStatusByPath(ctx, b.Config.Workspace.RootPath)
bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config.Workspace)
g, ctx := errgroup.WithContext(ctx)
for _, p := range bundlePaths {
g.Go(func() error {
return setPermissions(ctx, w, p, permissions)
})
}
return g.Wait()
}
func setPermissions(ctx context.Context, w workspace.WorkspaceInterface, path string, permissions []workspace.WorkspaceObjectAccessControlRequest) error {
obj, err := w.GetStatusByPath(ctx, path)
if err != nil {
return err
}
_, err = w.UpdatePermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{
_, err = w.SetPermissions(ctx, workspace.WorkspaceObjectPermissionsRequest{
WorkspaceObjectId: fmt.Sprint(obj.ObjectId),
WorkspaceObjectType: "directories",
AccessControlList: permissions,
})
return err
}

View File

@ -21,7 +21,11 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
RootPath: "/Users/foo@bar.com",
RootPath: "/Users/foo@bar.com",
ArtifactPath: "/Users/foo@bar.com/artifacts",
FilePath: "/Users/foo@bar.com/files",
StatePath: "/Users/foo@bar.com/state",
ResourcePath: "/Users/foo@bar.com/resources",
},
Permissions: []resources.Permission{
{Level: CAN_MANAGE, UserName: "TestUser"},
@ -59,7 +63,7 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com").Return(&workspace.ObjectInfo{
ObjectId: 1234,
}, nil)
workspaceApi.EXPECT().UpdatePermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
@ -72,3 +76,116 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
diags := bundle.Apply(context.Background(), b, bundle.Seq(ValidateSharedRootPermissions(), ApplyWorkspaceRootPermissions()))
require.Empty(t, diags)
}
func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
RootPath: "/Some/Root/Path",
ArtifactPath: "/Users/foo@bar.com/artifacts",
FilePath: "/Users/foo@bar.com/files",
StatePath: "/Users/foo@bar.com/state",
ResourcePath: "/Users/foo@bar.com/resources",
},
Permissions: []resources.Permission{
{Level: CAN_MANAGE, UserName: "TestUser"},
{Level: CAN_VIEW, GroupName: "TestGroup"},
{Level: CAN_RUN, ServicePrincipalName: "TestServicePrincipal"},
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}},
"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
"pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}},
},
Models: map[string]*resources.MlflowModel{
"model_1": {Model: &ml.Model{}},
"model_2": {Model: &ml.Model{}},
},
Experiments: map[string]*resources.MlflowExperiment{
"experiment_1": {Experiment: &ml.Experiment{}},
"experiment_2": {Experiment: &ml.Experiment{}},
},
ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
"endpoint_1": {CreateServingEndpoint: &serving.CreateServingEndpoint{}},
"endpoint_2": {CreateServingEndpoint: &serving.CreateServingEndpoint{}},
},
},
},
}
m := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(m.WorkspaceClient)
workspaceApi := m.GetMockWorkspaceAPI()
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Some/Root/Path").Return(&workspace.ObjectInfo{
ObjectId: 1,
}, nil)
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/artifacts").Return(&workspace.ObjectInfo{
ObjectId: 2,
}, nil)
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/files").Return(&workspace.ObjectInfo{
ObjectId: 3,
}, nil)
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/state").Return(&workspace.ObjectInfo{
ObjectId: 4,
}, nil)
workspaceApi.EXPECT().GetStatusByPath(mock.Anything, "/Users/foo@bar.com/resources").Return(&workspace.ObjectInfo{
ObjectId: 5,
}, nil)
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
{ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"},
},
WorkspaceObjectId: "1",
WorkspaceObjectType: "directories",
}).Return(nil, nil)
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
{ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"},
},
WorkspaceObjectId: "2",
WorkspaceObjectType: "directories",
}).Return(nil, nil)
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
{ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"},
},
WorkspaceObjectId: "3",
WorkspaceObjectType: "directories",
}).Return(nil, nil)
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
{ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"},
},
WorkspaceObjectId: "4",
WorkspaceObjectType: "directories",
}).Return(nil, nil)
workspaceApi.EXPECT().SetPermissions(mock.Anything, workspace.WorkspaceObjectPermissionsRequest{
AccessControlList: []workspace.WorkspaceObjectAccessControlRequest{
{UserName: "TestUser", PermissionLevel: "CAN_MANAGE"},
{GroupName: "TestGroup", PermissionLevel: "CAN_READ"},
{ServicePrincipalName: "TestServicePrincipal", PermissionLevel: "CAN_RUN"},
},
WorkspaceObjectId: "5",
WorkspaceObjectType: "directories",
}).Return(nil, nil)
diags := bundle.Apply(context.Background(), b, ApplyWorkspaceRootPermissions())
require.NoError(t, diags.Error())
}

View File

@ -152,6 +152,7 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
bundle.Defer(
bundle.Seq(
terraform.StatePull(),
terraform.CheckDashboardsModifiedRemotely(),
deploy.StatePull(),
mutator.ValidateGitDetails(),
artifacts.CleanUp(),

View File

@ -66,6 +66,7 @@ func Initialize() bundle.Mutator {
permissions.PermissionDiagnostics(),
mutator.SetRunAs(),
mutator.OverrideCompute(),
mutator.ConfigureDashboardDefaults(),
mutator.ProcessTargetMode(),
mutator.ApplyPresets(),
mutator.DefaultQueueing(),

View File

@ -180,6 +180,48 @@
}
]
},
"resources.Dashboard": {
"anyOf": [
{
"type": "object",
"properties": {
"display_name": {
"description": "The display name of the dashboard.",
"$ref": "#/$defs/string"
},
"embed_credentials": {
"$ref": "#/$defs/bool"
},
"file_path": {
"$ref": "#/$defs/string"
},
"parent_path": {
"description": "The workspace path of the folder containing the dashboard. Includes leading slash and no\ntrailing slash.\nThis field is excluded in List Dashboards responses.",
"$ref": "#/$defs/string"
},
"permissions": {
"$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission"
},
"serialized_dashboard": {
"description": "The contents of the dashboard in serialized string form.\nThis field is excluded in List Dashboards responses.\nUse the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get)\nto retrieve an example response, which includes the `serialized_dashboard` field.\nThis field provides the structure of the JSON string that represents the dashboard's\nlayout and components.",
"$ref": "#/$defs/interface"
},
"warehouse_id": {
"description": "The warehouse ID used to run the dashboard.",
"$ref": "#/$defs/string"
}
},
"additionalProperties": false,
"required": [
"display_name"
]
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"resources.Grant": {
"anyOf": [
{
@ -1054,6 +1096,9 @@
"clusters": {
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster"
},
"dashboards": {
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard"
},
"experiments": {
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment"
},
@ -5292,6 +5337,20 @@
}
]
},
"resources.Dashboard": {
"anyOf": [
{
"type": "object",
"additionalProperties": {
"$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Dashboard"
}
},
{
"type": "string",
"pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}"
}
]
},
"resources.Job": {
"anyOf": [
{

View File

@ -16,6 +16,7 @@ func newGenerateCommand() *cobra.Command {
cmd.AddCommand(generate.NewGenerateJobCommand())
cmd.AddCommand(generate.NewGeneratePipelineCommand())
cmd.AddCommand(generate.NewGenerateDashboardCommand())
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
return cmd
}

View File

@ -0,0 +1,467 @@
package generate
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config/generate"
"github.com/databricks/cli/bundle/deploy/terraform"
"github.com/databricks/cli/bundle/phases"
"github.com/databricks/cli/bundle/render"
"github.com/databricks/cli/bundle/resources"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/yamlsaver"
"github.com/databricks/cli/libs/textutil"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/spf13/cobra"
"golang.org/x/exp/maps"
"gopkg.in/yaml.v3"
)
type dashboard struct {
// Lookup flags for one-time generate.
existingPath string
existingID string
// Lookup flag for existing bundle resource.
resource string
// Where to write the configuration and dashboard representation.
resourceDir string
dashboardDir string
// Force overwrite of existing files.
force bool
// Watch for changes to the dashboard.
watch bool
// Relative path from the resource directory to the dashboard directory.
relativeDashboardDir string
}
func (d *dashboard) resolveID(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) {
switch {
case d.existingPath != "":
return d.resolveFromPath(ctx, b)
case d.existingID != "":
return d.resolveFromID(ctx, b)
}
return "", diag.Errorf("expected one of --dashboard-path, --dashboard-id")
}
func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) {
w := b.WorkspaceClient()
obj, err := w.Workspace.GetStatusByPath(ctx, d.existingPath)
if err != nil {
if apierr.IsMissing(err) {
return "", diag.Errorf("dashboard %q not found", path.Base(d.existingPath))
}
// Emit a more descriptive error message for legacy dashboards.
if errors.Is(err, apierr.ErrBadRequest) && strings.HasPrefix(err.Error(), "dbsqlDashboard ") {
return "", diag.Diagnostics{
{
Severity: diag.Error,
Summary: fmt.Sprintf("dashboard %q is a legacy dashboard", path.Base(d.existingPath)),
Detail: "" +
"Databricks Asset Bundles work exclusively with AI/BI dashboards.\n" +
"\n" +
"Instructions on how to convert a legacy dashboard to an AI/BI dashboard\n" +
"can be found at: https://docs.databricks.com/en/dashboards/clone-legacy-to-aibi.html.",
},
}
}
return "", diag.FromErr(err)
}
if obj.ObjectType != workspace.ObjectTypeDashboard {
found := strings.ToLower(obj.ObjectType.String())
return "", diag.Diagnostics{
{
Severity: diag.Error,
Summary: fmt.Sprintf("expected a dashboard, found a %s", found),
},
}
}
if obj.ResourceId == "" {
return "", diag.Diagnostics{
{
Severity: diag.Error,
Summary: "expected a non-empty dashboard resource ID",
},
}
}
return obj.ResourceId, nil
}
func (d *dashboard) resolveFromID(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics) {
w := b.WorkspaceClient()
obj, err := w.Lakeview.GetByDashboardId(ctx, d.existingID)
if err != nil {
if apierr.IsMissing(err) {
return "", diag.Errorf("dashboard with ID %s not found", d.existingID)
}
return "", diag.FromErr(err)
}
return obj.DashboardId, nil
}
func remarshalJSON(data []byte) ([]byte, error) {
var tmp any
var err error
err = json.Unmarshal(data, &tmp)
if err != nil {
return nil, err
}
// Remarshal the data to ensure its formatting is stable.
// The result will have alphabetically sorted keys and be indented.
// HTML escaping is disabled to retain characters such as &, <, and >.
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false)
err = enc.Encode(tmp)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (d *dashboard) saveSerializedDashboard(_ context.Context, b *bundle.Bundle, dashboard *dashboards.Dashboard, filename string) error {
// Unmarshal and remarshal the serialized dashboard to ensure it is formatted correctly.
// The result will have alphabetically sorted keys and be indented.
data, err := remarshalJSON([]byte(dashboard.SerializedDashboard))
if err != nil {
return err
}
// Make sure the output directory exists.
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
return err
}
// Clean the filename to ensure it is a valid path (and can be used on this OS).
filename = filepath.Clean(filename)
// Attempt to make the path relative to the bundle root.
rel, err := filepath.Rel(b.BundleRootPath, filename)
if err != nil {
rel = filename
}
// Verify that the file does not already exist.
info, err := os.Stat(filename)
if err == nil {
if info.IsDir() {
return fmt.Errorf("%s is a directory", rel)
}
if !d.force {
return fmt.Errorf("%s already exists. Use --force to overwrite", rel)
}
}
fmt.Printf("Writing dashboard to %q\n", rel)
return os.WriteFile(filename, data, 0644)
}
func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, dashboard *dashboards.Dashboard, key string) error {
// Save serialized dashboard definition to the dashboard directory.
dashboardBasename := fmt.Sprintf("%s.lvdash.json", key)
dashboardPath := filepath.Join(d.dashboardDir, dashboardBasename)
err := d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath)
if err != nil {
return err
}
// Synthesize resource configuration.
v, err := generate.ConvertDashboardToValue(dashboard, path.Join(d.relativeDashboardDir, dashboardBasename))
if err != nil {
return err
}
result := map[string]dyn.Value{
"resources": dyn.V(map[string]dyn.Value{
"dashboards": dyn.V(map[string]dyn.Value{
key: v,
}),
}),
}
// Make sure the output directory exists.
if err := os.MkdirAll(d.resourceDir, 0755); err != nil {
return err
}
// Save the configuration to the resource directory.
resourcePath := filepath.Join(d.resourceDir, fmt.Sprintf("%s.dashboard.yml", key))
saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{
"display_name": yaml.DoubleQuotedStyle,
})
// Attempt to make the path relative to the bundle root.
rel, err := filepath.Rel(b.BundleRootPath, resourcePath)
if err != nil {
rel = resourcePath
}
fmt.Printf("Writing configuration to %q\n", rel)
err = saver.SaveAsYAML(result, resourcePath, d.force)
if err != nil {
return err
}
return nil
}
func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboard *dashboards.Dashboard) diag.Diagnostics {
// Compute [time.Time] for the most recent update.
tref, err := time.Parse(time.RFC3339, dashboard.UpdateTime)
if err != nil {
return diag.FromErr(err)
}
for {
obj, err := w.Workspace.GetStatusByPath(ctx, dashboard.Path)
if err != nil {
return diag.FromErr(err)
}
// Compute [time.Time] from timestamp in millis since epoch.
tcur := time.Unix(0, obj.ModifiedAt*int64(time.Millisecond))
if tcur.After(tref) {
break
}
time.Sleep(1 * time.Second)
}
return nil
}
func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
resource, ok := b.Config.Resources.Dashboards[d.resource]
if !ok {
return diag.Errorf("dashboard resource %q is not defined", d.resource)
}
if resource.FilePath == "" {
return diag.Errorf("dashboard resource %q has no file path defined", d.resource)
}
// Resolve the dashboard ID from the resource.
dashboardID := resource.ID
// Overwrite the dashboard at the path referenced from the resource.
dashboardPath := resource.FilePath
w := b.WorkspaceClient()
// Start polling the underlying dashboard for changes.
var etag string
for {
dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID)
if err != nil {
return diag.FromErr(err)
}
if etag != dashboard.Etag {
err = d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath)
if err != nil {
return diag.FromErr(err)
}
}
// Abort if we are not watching for changes.
if !d.watch {
return nil
}
// Update the etag for the next iteration.
etag = dashboard.Etag
// Now poll the workspace API for changes.
// This is much more efficient than polling the dashboard API because it
// includes the entire serialized dashboard whereas we're only interested
// in the last modified time of the dashboard here.
waitForChanges(ctx, w, dashboard)
}
}
func (d *dashboard) generateForExisting(ctx context.Context, b *bundle.Bundle, dashboardID string) diag.Diagnostics {
w := b.WorkspaceClient()
dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID)
if err != nil {
return diag.FromErr(err)
}
key := textutil.NormalizeString(dashboard.DisplayName)
err = d.saveConfiguration(ctx, b, dashboard, key)
if err != nil {
return diag.FromErr(err)
}
return nil
}
func (d *dashboard) initialize(b *bundle.Bundle) diag.Diagnostics {
// Make the paths absolute if they aren't already.
if !filepath.IsAbs(d.resourceDir) {
d.resourceDir = filepath.Join(b.BundleRootPath, d.resourceDir)
}
if !filepath.IsAbs(d.dashboardDir) {
d.dashboardDir = filepath.Join(b.BundleRootPath, d.dashboardDir)
}
// Make sure we know how the dashboard path is relative to the resource path.
rel, err := filepath.Rel(d.resourceDir, d.dashboardDir)
if err != nil {
return diag.FromErr(err)
}
d.relativeDashboardDir = filepath.ToSlash(rel)
return nil
}
func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
diags := bundle.Apply(ctx, b, bundle.Seq(
phases.Initialize(),
terraform.Interpolate(),
terraform.Write(),
terraform.StatePull(),
terraform.Load(),
))
if diags.HasError() {
return diags
}
return d.updateDashboardForResource(ctx, b)
}
func (d *dashboard) runForExisting(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// Resolve the ID of the dashboard to generate configuration for.
dashboardID, diags := d.resolveID(ctx, b)
if diags.HasError() {
return diags
}
return d.generateForExisting(ctx, b, dashboardID)
}
func (d *dashboard) RunE(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
b, diags := root.MustConfigureBundle(cmd)
if diags.HasError() {
return diags.Error()
}
diags = d.initialize(b)
if diags.HasError() {
return diags.Error()
}
if d.resource != "" {
diags = d.runForResource(ctx, b)
} else {
diags = d.runForExisting(ctx, b)
}
renderOpts := render.RenderOptions{RenderSummaryTable: false}
err := render.RenderDiagnostics(cmd.OutOrStdout(), b, diags, renderOpts)
if err != nil {
return fmt.Errorf("failed to render output: %w", err)
}
if diags.HasError() {
return root.ErrAlreadyPrinted
}
return nil
}
// filterDashboards returns a filter that only includes dashboards.
func filterDashboards(ref resources.Reference) bool {
return ref.Description.SingularName == "dashboard"
}
// dashboardResourceCompletion executes to autocomplete the argument to the resource flag.
func dashboardResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
b, diags := root.MustConfigureBundle(cmd)
if err := diags.Error(); err != nil {
cobra.CompErrorln(err.Error())
return nil, cobra.ShellCompDirectiveError
}
if b == nil {
return nil, cobra.ShellCompDirectiveNoFileComp
}
return maps.Keys(resources.Completions(b, filterDashboards)), cobra.ShellCompDirectiveNoFileComp
}
func NewGenerateDashboardCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard",
Short: "Generate configuration for a dashboard",
}
d := &dashboard{}
// Lookup flags.
cmd.Flags().StringVar(&d.existingPath, "existing-path", "", `workspace path of the dashboard to generate configuration for`)
cmd.Flags().StringVar(&d.existingID, "existing-id", "", `ID of the dashboard to generate configuration for`)
cmd.Flags().StringVar(&d.resource, "resource", "", `resource key of dashboard to watch for changes`)
// Alias lookup flags that include the resource type name.
// Included for symmetry with the other generate commands, but we prefer the shorter flags.
cmd.Flags().StringVar(&d.existingPath, "existing-dashboard-path", "", `workspace path of the dashboard to generate configuration for`)
cmd.Flags().StringVar(&d.existingID, "existing-dashboard-id", "", `ID of the dashboard to generate configuration for`)
cmd.Flags().MarkHidden("existing-dashboard-path")
cmd.Flags().MarkHidden("existing-dashboard-id")
// Output flags.
cmd.Flags().StringVarP(&d.resourceDir, "resource-dir", "d", "./resources", `directory to write the configuration to`)
cmd.Flags().StringVarP(&d.dashboardDir, "dashboard-dir", "s", "./src", `directory to write the dashboard representation to`)
cmd.Flags().BoolVarP(&d.force, "force", "f", false, `force overwrite existing files in the output directory`)
// Exactly one of the lookup flags must be provided.
cmd.MarkFlagsOneRequired(
"existing-path",
"existing-id",
"resource",
)
// Watch flag. This is relevant only in combination with the resource flag.
cmd.Flags().BoolVar(&d.watch, "watch", false, `watch for changes to the dashboard and update the configuration`)
// Make sure the watch flag is only used with the existing-resource flag.
cmd.MarkFlagsMutuallyExclusive("watch", "existing-path")
cmd.MarkFlagsMutuallyExclusive("watch", "existing-id")
// Completion for the resource flag.
cmd.RegisterFlagCompletionFunc("resource", dashboardResourceCompletion)
cmd.RunE = d.RunE
return cmd
}

View File

@ -0,0 +1,182 @@
package generate
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) {
// Response to a GetStatus request on a path pointing to a legacy dashboard.
//
// < HTTP/2.0 400 Bad Request
// < {
// < "error_code": "BAD_REQUEST",
// < "message": "dbsqlDashboard is not user-facing."
// < }
d := dashboard{
existingPath: "/path/to/legacy dashboard",
}
m := mocks.NewMockWorkspaceClient(t)
w := m.GetMockWorkspaceAPI()
w.On("GetStatusByPath", mock.Anything, "/path/to/legacy dashboard").Return(nil, &apierr.APIError{
StatusCode: 400,
ErrorCode: "BAD_REQUEST",
Message: "dbsqlDashboard is not user-facing.",
})
ctx := context.Background()
b := &bundle.Bundle{}
b.SetWorkpaceClient(m.WorkspaceClient)
_, diags := d.resolveID(ctx, b)
require.Len(t, diags, 1)
assert.Equal(t, diags[0].Summary, "dashboard \"legacy dashboard\" is a legacy dashboard")
}
func TestDashboard_ExistingID_Nominal(t *testing.T) {
root := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: root,
}
m := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(m.WorkspaceClient)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(&dashboards.Dashboard{
DashboardId: "f00dcafe",
DisplayName: "This is a test dashboard",
SerializedDashboard: `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`,
WarehouseId: "w4r3h0us3",
}, nil)
ctx := bundle.Context(context.Background(), b)
cmd := NewGenerateDashboardCommand()
cmd.SetContext(ctx)
cmd.Flag("existing-id").Value.Set("f00dcafe")
err := cmd.RunE(cmd, []string{})
require.NoError(t, err)
// Assert the contents of the generated configuration
data, err := os.ReadFile(filepath.Join(root, "resources", "this_is_a_test_dashboard.dashboard.yml"))
require.NoError(t, err)
assert.Equal(t, `resources:
dashboards:
this_is_a_test_dashboard:
display_name: "This is a test dashboard"
warehouse_id: w4r3h0us3
file_path: ../src/this_is_a_test_dashboard.lvdash.json
`, string(data))
data, err = os.ReadFile(filepath.Join(root, "src", "this_is_a_test_dashboard.lvdash.json"))
require.NoError(t, err)
assert.JSONEq(t, `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, string(data))
}
func TestDashboard_ExistingID_NotFound(t *testing.T) {
root := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: root,
}
m := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(m.WorkspaceClient)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(nil, &apierr.APIError{
StatusCode: 404,
})
ctx := bundle.Context(context.Background(), b)
cmd := NewGenerateDashboardCommand()
cmd.SetContext(ctx)
cmd.Flag("existing-id").Value.Set("f00dcafe")
err := cmd.RunE(cmd, []string{})
require.Error(t, err)
}
func TestDashboard_ExistingPath_Nominal(t *testing.T) {
root := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: root,
}
m := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(m.WorkspaceClient)
workspaceAPI := m.GetMockWorkspaceAPI()
workspaceAPI.EXPECT().GetStatusByPath(mock.Anything, "/path/to/dashboard").Return(&workspace.ObjectInfo{
ObjectType: workspace.ObjectTypeDashboard,
ResourceId: "f00dcafe",
}, nil)
dashboardsAPI := m.GetMockLakeviewAPI()
dashboardsAPI.EXPECT().GetByDashboardId(mock.Anything, "f00dcafe").Return(&dashboards.Dashboard{
DashboardId: "f00dcafe",
DisplayName: "This is a test dashboard",
SerializedDashboard: `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`,
WarehouseId: "w4r3h0us3",
}, nil)
ctx := bundle.Context(context.Background(), b)
cmd := NewGenerateDashboardCommand()
cmd.SetContext(ctx)
cmd.Flag("existing-path").Value.Set("/path/to/dashboard")
err := cmd.RunE(cmd, []string{})
require.NoError(t, err)
// Assert the contents of the generated configuration
data, err := os.ReadFile(filepath.Join(root, "resources", "this_is_a_test_dashboard.dashboard.yml"))
require.NoError(t, err)
assert.Equal(t, `resources:
dashboards:
this_is_a_test_dashboard:
display_name: "This is a test dashboard"
warehouse_id: w4r3h0us3
file_path: ../src/this_is_a_test_dashboard.lvdash.json
`, string(data))
data, err = os.ReadFile(filepath.Join(root, "src", "this_is_a_test_dashboard.lvdash.json"))
require.NoError(t, err)
assert.JSONEq(t, `{"pages":[{"displayName":"New Page","layout":[],"name":"12345678"}]}`, string(data))
}
func TestDashboard_ExistingPath_NotFound(t *testing.T) {
root := t.TempDir()
b := &bundle.Bundle{
BundleRootPath: root,
}
m := mocks.NewMockWorkspaceClient(t)
b.SetWorkpaceClient(m.WorkspaceClient)
workspaceAPI := m.GetMockWorkspaceAPI()
workspaceAPI.EXPECT().GetStatusByPath(mock.Anything, "/path/to/dashboard").Return(nil, &apierr.APIError{
StatusCode: 404,
})
ctx := bundle.Context(context.Background(), b)
cmd := NewGenerateDashboardCommand()
cmd.SetContext(ctx)
cmd.Flag("existing-path").Value.Set("/path/to/dashboard")
err := cmd.RunE(cmd, []string{})
require.Error(t, err)
}

2
go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/Masterminds/semver/v3 v3.3.0 // MIT
github.com/briandowns/spinner v1.23.1 // Apache 2.0
github.com/databricks/databricks-sdk-go v0.49.0 // Apache 2.0
github.com/fatih/color v1.17.0 // MIT
github.com/fatih/color v1.18.0 // MIT
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
github.com/google/uuid v1.6.0 // BSD-3-Clause
github.com/hashicorp/go-version v1.7.0 // MPL 2.0

4
go.sum generated
View File

@ -44,8 +44,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=

View File

@ -0,0 +1,12 @@
{
"properties": {
"unique_id": {
"type": "string",
"description": "Unique ID for job name"
},
"warehouse_id": {
"type": "string",
"description": "The SQL warehouse ID to use for the dashboard"
}
}
}

View File

@ -0,0 +1,34 @@
{
"pages": [
{
"displayName": "New Page",
"layout": [
{
"position": {
"height": 2,
"width": 6,
"x": 0,
"y": 0
},
"widget": {
"name": "82eb9107",
"textbox_spec": "# I'm a title"
}
},
{
"position": {
"height": 2,
"width": 6,
"x": 0,
"y": 2
},
"widget": {
"name": "ffa6de4f",
"textbox_spec": "Text"
}
}
],
"name": "fdd21a3c"
}
]
}

View File

@ -0,0 +1,12 @@
bundle:
name: dashboards
workspace:
root_path: "~/.bundle/{{.unique_id}}"
resources:
dashboards:
file_reference:
display_name: test-dashboard-{{.unique_id}}
file_path: ./dashboard.lvdash.json
warehouse_id: {{.warehouse_id}}

View File

@ -0,0 +1,63 @@
package bundle
import (
"fmt"
"testing"
"github.com/databricks/cli/internal/acc"
"github.com/databricks/databricks-sdk-go/service/dashboards"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAccDashboards(t *testing.T) {
ctx, wt := acc.WorkspaceTest(t)
warehouseID := acc.GetEnvOrSkipTest(t, "TEST_DEFAULT_WAREHOUSE_ID")
uniqueID := uuid.New().String()
root, err := initTestTemplate(t, ctx, "dashboards", map[string]any{
"unique_id": uniqueID,
"warehouse_id": warehouseID,
})
require.NoError(t, err)
t.Cleanup(func() {
err = destroyBundle(t, ctx, root)
require.NoError(t, err)
})
err = deployBundle(t, ctx, root)
require.NoError(t, err)
// Load bundle configuration by running the validate command.
b := unmarshalConfig(t, mustValidateBundle(t, ctx, root))
// Assert that the dashboard exists at the expected path and is, indeed, a dashboard.
oi, err := wt.W.Workspace.GetStatusByPath(ctx, fmt.Sprintf("%s/test-dashboard-%s.lvdash.json", b.Config.Workspace.ResourcePath, uniqueID))
require.NoError(t, err)
assert.EqualValues(t, workspace.ObjectTypeDashboard, oi.ObjectType)
// Load the dashboard by its ID and confirm its display name.
dashboard, err := wt.W.Lakeview.GetByDashboardId(ctx, oi.ResourceId)
require.NoError(t, err)
assert.Equal(t, fmt.Sprintf("test-dashboard-%s", uniqueID), dashboard.DisplayName)
// Make an out of band modification to the dashboard and confirm that it is detected.
_, err = wt.W.Lakeview.Update(ctx, dashboards.UpdateDashboardRequest{
DashboardId: oi.ResourceId,
SerializedDashboard: dashboard.SerializedDashboard,
})
require.NoError(t, err)
// Try to redeploy the bundle and confirm that the out of band modification is detected.
stdout, _, err := deployBundleWithArgs(t, ctx, root)
require.Error(t, err)
assert.Contains(t, stdout, `Error: dashboard "file_reference" has been modified remotely`+"\n")
// Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored.
_, stderr, err := deployBundleWithArgs(t, ctx, root, "--force")
require.NoError(t, err)
assert.Contains(t, stderr, `Deployment complete!`+"\n")
}

View File

@ -11,6 +11,7 @@ import (
"strings"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/internal"
"github.com/databricks/cli/libs/cmdio"
@ -66,6 +67,19 @@ func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, err
return stdout.Bytes(), err
}
func mustValidateBundle(t *testing.T, ctx context.Context, path string) []byte {
data, err := validateBundle(t, ctx, path)
require.NoError(t, err)
return data
}
func unmarshalConfig(t *testing.T, data []byte) *bundle.Bundle {
bundle := &bundle.Bundle{}
err := json.Unmarshal(data, &bundle.Config)
require.NoError(t, err)
return bundle
}
func deployBundle(t *testing.T, ctx context.Context, path string) error {
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve")
@ -73,6 +87,14 @@ func deployBundle(t *testing.T, ctx context.Context, path string) error {
return err
}
func deployBundleWithArgs(t *testing.T, ctx context.Context, path string, args ...string) (string, string, error) {
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
args = append([]string{"bundle", "deploy"}, args...)
c := internal.NewCobraTestRunnerWithContext(t, ctx, args...)
stdout, stderr, err := c.Run()
return stdout.String(), stderr.String(), err
}
func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error {
ctx = env.Set(ctx, "BUNDLE_ROOT", path)
args := []string{"bundle", "deploy", "--force-lock"}

View File

@ -111,3 +111,7 @@ func PanicsWithError(t assert.TestingT, errString string, f func(), msgAndArgs .
func NotPanics(t assert.TestingT, f func(), msgAndArgs ...interface{}) bool {
return assert.NotPanics(t, f, msgAndArgs...)
}
func JSONEq(t assert.TestingT, expected string, actual string, msgAndArgs ...interface{}) bool {
return assert.JSONEq(t, expected, actual, msgAndArgs...)
}

View File

@ -0,0 +1,39 @@
package jsonsaver
import (
"bytes"
"encoding/json"
)
// The encoder type encapsulates a [json.Encoder] and its target buffer.
// Escaping of HTML characters in the output is disabled.
type encoder struct {
*json.Encoder
*bytes.Buffer
}
func newEncoder() encoder {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
// By default, json.Encoder escapes HTML characters, converting symbols like '<' to '\u003c'.
// This behavior helps prevent XSS attacks when JSON is embedded within HTML.
// However, we disable this feature since we're not dealing with HTML context.
// Keeping the escapes enabled would result in unnecessary differences when processing JSON payloads
// that already contain escaped characters.
enc.SetEscapeHTML(false)
return encoder{enc, &buf}
}
func marshalNoEscape(v any) ([]byte, error) {
enc := newEncoder()
err := enc.Encode(v)
return enc.Bytes(), err
}
func marshalIndentNoEscape(v any, prefix, indent string) ([]byte, error) {
enc := newEncoder()
enc.SetIndent(prefix, indent)
err := enc.Encode(v)
return enc.Bytes(), err
}

View File

@ -0,0 +1,41 @@
package jsonsaver
import (
"testing"
assert "github.com/databricks/cli/libs/dyn/dynassert"
)
func TestEncoder_MarshalNoEscape(t *testing.T) {
out, err := marshalNoEscape("1 < 2")
if !assert.NoError(t, err) {
return
}
// Confirm the output.
assert.JSONEq(t, `"1 < 2"`, string(out))
// Confirm that HTML escaping is disabled.
assert.NotContains(t, string(out), "\\u003c")
// Confirm that the encoder writes a trailing newline.
assert.Contains(t, string(out), "\n")
}
func TestEncoder_MarshalIndentNoEscape(t *testing.T) {
out, err := marshalIndentNoEscape([]string{"1 < 2", "2 < 3"}, "", " ")
if !assert.NoError(t, err) {
return
}
// Confirm the output.
assert.JSONEq(t, `["1 < 2", "2 < 3"]`, string(out))
// Confirm that HTML escaping is disabled.
assert.NotContains(t, string(out), "\\u003c")
// Confirm that the encoder performs indenting and writes a trailing newline.
assert.Contains(t, string(out), "[\n")
assert.Contains(t, string(out), " \"1 < 2\",\n")
assert.Contains(t, string(out), "]\n")
}

View File

@ -0,0 +1,89 @@
package jsonsaver
import (
"bytes"
"fmt"
"github.com/databricks/cli/libs/dyn"
)
// Marshal is a version of [json.Marshal] for [dyn.Value].
//
// Objects in the output retain the order of keys as they appear in the underlying [dyn.Value].
// The output does not escape HTML characters in strings.
func Marshal(v dyn.Value) ([]byte, error) {
return marshalNoEscape(wrap{v})
}
// MarshalIndent is a version of [json.MarshalIndent] for [dyn.Value].
//
// Objects in the output retain the order of keys as they appear in the underlying [dyn.Value].
// The output does not escape HTML characters in strings.
func MarshalIndent(v dyn.Value, prefix, indent string) ([]byte, error) {
return marshalIndentNoEscape(wrap{v}, prefix, indent)
}
// Wrapper type for [dyn.Value] to expose the [json.Marshaler] interface.
type wrap struct {
v dyn.Value
}
// MarshalJSON implements the [json.Marshaler] interface for the [dyn.Value] wrapper type.
func (w wrap) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
if err := marshalValue(&buf, w.v); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// marshalValue recursively writes JSON for a [dyn.Value] to the buffer.
func marshalValue(buf *bytes.Buffer, v dyn.Value) error {
switch v.Kind() {
case dyn.KindString, dyn.KindBool, dyn.KindInt, dyn.KindFloat, dyn.KindTime, dyn.KindNil:
out, err := marshalNoEscape(v.AsAny())
if err != nil {
return err
}
// The encoder writes a trailing newline, so we need to remove it
// to avoid adding extra newlines when embedding this JSON.
out = out[:len(out)-1]
buf.Write(out)
case dyn.KindMap:
buf.WriteByte('{')
for i, pair := range v.MustMap().Pairs() {
if i > 0 {
buf.WriteByte(',')
}
// Require keys to be strings.
if pair.Key.Kind() != dyn.KindString {
return fmt.Errorf("map key must be a string, got %s", pair.Key.Kind())
}
// Marshal the key
if err := marshalValue(buf, pair.Key); err != nil {
return err
}
buf.WriteByte(':')
// Marshal the value
if err := marshalValue(buf, pair.Value); err != nil {
return err
}
}
buf.WriteByte('}')
case dyn.KindSequence:
buf.WriteByte('[')
for i, item := range v.MustSequence() {
if i > 0 {
buf.WriteByte(',')
}
if err := marshalValue(buf, item); err != nil {
return err
}
}
buf.WriteByte(']')
default:
return fmt.Errorf("unsupported kind: %d", v.Kind())
}
return nil
}

View File

@ -0,0 +1,100 @@
package jsonsaver
import (
"testing"
"github.com/databricks/cli/libs/dyn"
assert "github.com/databricks/cli/libs/dyn/dynassert"
)
func TestMarshal_String(t *testing.T) {
b, err := Marshal(dyn.V("string"))
if assert.NoError(t, err) {
assert.JSONEq(t, `"string"`, string(b))
}
}
func TestMarshal_Bool(t *testing.T) {
b, err := Marshal(dyn.V(true))
if assert.NoError(t, err) {
assert.JSONEq(t, `true`, string(b))
}
}
func TestMarshal_Int(t *testing.T) {
b, err := Marshal(dyn.V(42))
if assert.NoError(t, err) {
assert.JSONEq(t, `42`, string(b))
}
}
func TestMarshal_Float(t *testing.T) {
b, err := Marshal(dyn.V(42.1))
if assert.NoError(t, err) {
assert.JSONEq(t, `42.1`, string(b))
}
}
func TestMarshal_Time(t *testing.T) {
b, err := Marshal(dyn.V(dyn.MustTime("2021-01-01T00:00:00Z")))
if assert.NoError(t, err) {
assert.JSONEq(t, `"2021-01-01T00:00:00Z"`, string(b))
}
}
func TestMarshal_Map(t *testing.T) {
m := dyn.NewMapping()
m.Set(dyn.V("key1"), dyn.V("value1"))
m.Set(dyn.V("key2"), dyn.V("value2"))
b, err := Marshal(dyn.V(m))
if assert.NoError(t, err) {
assert.JSONEq(t, `{"key1":"value1","key2":"value2"}`, string(b))
}
}
func TestMarshal_Sequence(t *testing.T) {
var s []dyn.Value
s = append(s, dyn.V("value1"))
s = append(s, dyn.V("value2"))
b, err := Marshal(dyn.V(s))
if assert.NoError(t, err) {
assert.JSONEq(t, `["value1","value2"]`, string(b))
}
}
func TestMarshal_Complex(t *testing.T) {
map1 := dyn.NewMapping()
map1.Set(dyn.V("str1"), dyn.V("value1"))
map1.Set(dyn.V("str2"), dyn.V("value2"))
seq1 := []dyn.Value{}
seq1 = append(seq1, dyn.V("value1"))
seq1 = append(seq1, dyn.V("value2"))
root := dyn.NewMapping()
root.Set(dyn.V("map1"), dyn.V(map1))
root.Set(dyn.V("seq1"), dyn.V(seq1))
// Marshal without indent.
b, err := Marshal(dyn.V(root))
if assert.NoError(t, err) {
assert.Equal(t, `{"map1":{"str1":"value1","str2":"value2"},"seq1":["value1","value2"]}`+"\n", string(b))
}
// Marshal with indent.
b, err = MarshalIndent(dyn.V(root), "", " ")
if assert.NoError(t, err) {
assert.Equal(t, `{
"map1": {
"str1": "value1",
"str2": "value2"
},
"seq1": [
"value1",
"value2"
]
}`+"\n", string(b))
}
}