steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into denik/wheel-patch

This commit is contained in:
Denis Bilenko 2025-03-05 16:08:20 +01:00 committed by GitHub
parent 7488b5ce98 a24a7f5738
commit 9568ce19ac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
32 changed files with 139 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/push.yml vendored
View File

@ -145,7 +145,10 @@ jobs:
go run main.go bundle schema > schema.json
# Add markdownDescription keyword to ajv
echo "module.exports=function(a){a.addKeyword('markdownDescription')}" >> keywords.js
echo "module.exports = function(a) {
a.addKeyword('markdownDescription');
a.addKeyword('deprecationMessage');
}" >> keywords.js
for file in ./bundle/internal/schema/testdata/pass/*.yml; do
ajv test -s schema.json -d $file --valid -c=./keywords.js

1
acceptance/bundle/templates/dbt-sql/script
View File

@ -6,3 +6,4 @@ trace $CLI bundle validate -t prod
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore
rm .databricks/.gitignore

2
acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff
View File

@ -48,7 +48,7 @@
- catalog: main
+ ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
+ # catalog: catalog_name
target: my_default_python_${bundle.target}
schema: my_default_python_${bundle.target}
- serverless: true
libraries:
- notebook:

2
acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml
View File

@ -5,7 +5,7 @@ resources:
name: my_default_python_pipeline
## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
# catalog: catalog_name
target: my_default_python_${bundle.target}
schema: my_default_python_${bundle.target}
libraries:
- notebook:
path: ../src/dlt_pipeline.ipynb

1
acceptance/bundle/templates/default-python/classic/script
View File

@ -6,6 +6,7 @@ trace $CLI bundle validate -t prod
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore
rm .databricks/.gitignore
cd ../../

2
acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt
View File

@ -18,5 +18,5 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde
- ## Catalog is required for serverless compute
- catalog: main
+ catalog: customcatalog
target: my_default_python_${bundle.target}
schema: my_default_python_${bundle.target}
serverless: true

2
acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml
View File

@ -5,7 +5,7 @@ resources:
name: my_default_python_pipeline
## Catalog is required for serverless compute
catalog: main
target: my_default_python_${bundle.target}
schema: my_default_python_${bundle.target}
serverless: true
libraries:
- notebook:

1
acceptance/bundle/templates/default-python/serverless/script
View File

@ -6,3 +6,4 @@ trace $CLI bundle validate -t prod
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore
rm .databricks/.gitignore

1
acceptance/bundle/templates/default-sql/output/my_default_sql/.databricks/out.gitignore Normal file
View File

@ -0,0 +1 @@
*

3
acceptance/bundle/templates/default-sql/script
View File

@ -6,3 +6,6 @@ trace $CLI bundle validate -t prod
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore
# Only for this test (default-sql), record .databricks/.gitignore in the output
mv .databricks/.gitignore .databricks/out.gitignore

1
acceptance/bundle/templates/experimental-jobs-as-code/script
View File

@ -11,3 +11,4 @@ rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info
# Do not affect this repository's git behaviour #2318
mv .gitignore out.gitignore
rm .databricks/.gitignore

2
bundle/bundle.go
View File

@ -21,6 +21,7 @@ import (
"github.com/databricks/cli/libs/fileset"
"github.com/databricks/cli/libs/locker"
"github.com/databricks/cli/libs/log"
libsync "github.com/databricks/cli/libs/sync"
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/terraform"
"github.com/databricks/cli/libs/vfs"
@ -198,6 +199,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error)
return "", err
}
libsync.WriteGitIgnore(ctx, b.BundleRootPath)
return dir, nil
}

2
bundle/config/root.go
View File

@ -52,7 +52,7 @@ type Root struct {
Targets map[string]*Target `json:"targets,omitempty"`
// DEPRECATED. Left for backward compatibility with Targets
Environments map[string]*Target `json:"environments,omitempty" bundle:"deprecated"`
Environments map[string]*Target `json:"environments,omitempty"`
// Sync section specifies options for files synchronization
Sync Sync `json:"sync,omitempty"`

1
bundle/internal/annotation/descriptor.go
View File

@ -7,6 +7,7 @@ type Descriptor struct {
Default any `json:"default,omitempty"`
Enum []any `json:"enum,omitempty"`
MarkdownExamples string `json:"markdown_examples,omitempty"`
DeprecationMessage string `json:"deprecation_message,omitempty"`
}
const Placeholder = "PLACEHOLDER"

6
bundle/internal/schema/annotations.go
View File

@ -127,6 +127,12 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
if a.Default != nil {
s.Default = a.Default
}
if a.DeprecationMessage != "" {
s.Deprecated = true
s.DeprecationMessage = a.DeprecationMessage
}
s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
s.Title = a.Title
s.Enum = a.Enum

9
bundle/internal/schema/annotations.yml
View File

@ -61,6 +61,8 @@ github.com/databricks/cli/bundle/config.Experimental:
"pydabs":
"description": |-
The PyDABs configuration.
"deprecation_message": |-
Deprecated: please use python instead
"python":
"description": |-
Configures loading of Python code defined with 'databricks-bundles' package.
@ -220,6 +222,11 @@ github.com/databricks/cli/bundle/config.Root:
The bundle attributes when deploying to this target.
"markdown_description": |-
The bundle attributes when deploying to this target,
"environments":
"description": |-
PLACEHOLDER
"deprecation_message": |-
Deprecated: please use targets instead
"experimental":
"description": |-
Defines attributes for experimental features.
@ -308,6 +315,8 @@ github.com/databricks/cli/bundle/config.Target:
"compute_id":
"description": |-
Deprecated. The ID of the compute to use for this target.
"deprecation_message": |-
Deprecated: please use cluster_id instead
"default":
"description": |-
Whether this target is the default target.

13
bundle/schema/jsonschema.json generated
View File

@ -1217,7 +1217,9 @@
"properties": {
"pydabs": {
"description": "The PyDABs configuration.",
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs"
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs",
"deprecationMessage": "Deprecated: please use python instead",
"deprecated": true
},
"python": {
"description": "Configures loading of Python code defined with 'databricks-bundles' package.",
@ -1506,7 +1508,9 @@
},
"compute_id": {
"description": "Deprecated. The ID of the compute to use for this target.",
"$ref": "#/$defs/string"
"$ref": "#/$defs/string",
"deprecationMessage": "Deprecated: please use cluster_id instead",
"deprecated": true
},
"default": {
"description": "Whether this target is the default target.",
@ -7409,6 +7413,11 @@
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle",
"markdownDescription": "The bundle attributes when deploying to this target,"
},
"environments": {
"$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target",
"deprecationMessage": "Deprecated: please use targets instead",
"deprecated": true
},
"experimental": {
"description": "Defines attributes for experimental features.",
"$ref": "#/$defs/github.com/databricks/cli/bundle/config.Experimental"

2
integration/bundle/testdata/default_python/bundle_summary.txt vendored
View File

@ -164,7 +164,7 @@
}
],
"name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline",
"target": "project_name_$UNIQUE_PRJ_dev",
"schema": "project_name_$UNIQUE_PRJ_dev",
"url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]"
}
}

75
integration/cmd/sync/sync_test.go
View File

@ -224,22 +224,21 @@ func (a *syncTest) snapshotContains(files []string) {
_, ok := s.LastModifiedTimes[filePath]
assert.True(a.t, ok, "%s not in snapshot file: %v", filePath, s.LastModifiedTimes)
}
assert.Equal(a.t, len(files), len(s.LastModifiedTimes))
assert.Equal(a.t, len(files), len(s.LastModifiedTimes), "files=%s s.LastModifiedTimes=%s", files, s.LastModifiedTimes)
}
func TestSyncFullFileSync(t *testing.T) {
ctx, assertSync := setupSyncTest(t, "--full", "--watch")
// .gitignore is created by the sync process to enforce .databricks is not synced
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
// New file
localFilePath := filepath.Join(assertSync.localRoot, "foo.txt")
f := testfile.CreateFile(t, localFilePath)
defer f.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt"))
assertSync.remoteFileContent(ctx, "foo.txt", "")
// Write to file
@ -255,24 +254,23 @@ func TestSyncFullFileSync(t *testing.T) {
// delete
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
}
func TestSyncIncrementalFileSync(t *testing.T) {
ctx, assertSync := setupSyncTest(t, "--watch")
// .gitignore is created by the sync process to enforce .databricks is not synced
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
// New file
localFilePath := filepath.Join(assertSync.localRoot, "foo.txt")
f := testfile.CreateFile(t, localFilePath)
defer f.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt", ".gitignore"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt"))
assertSync.remoteFileContent(ctx, "foo.txt", "")
assertSync.snapshotContains(append(repoFiles, "foo.txt", ".gitignore"))
assertSync.snapshotContains(append(repoFiles, "foo.txt"))
// Write to file
f.Overwrite(t, `{"statement": "Mi Gente"}`)
@ -287,16 +285,15 @@ func TestSyncIncrementalFileSync(t *testing.T) {
// delete
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.snapshotContains(append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
assertSync.snapshotContains(repoFiles)
}
func TestSyncNestedFolderSync(t *testing.T) {
ctx, assertSync := setupSyncTest(t, "--watch")
// .gitignore is created by the sync process to enforce .databricks is not synced
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
// New file
localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt")
@ -305,25 +302,24 @@ func TestSyncNestedFolderSync(t *testing.T) {
f := testfile.CreateFile(t, localFilePath)
defer f.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "dir1"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1"))
assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"})
assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"})
assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"})
assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/dir2/dir3/foo.txt"))
assertSync.snapshotContains(append(repoFiles, "dir1/dir2/dir3/foo.txt"))
// delete
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteNotExist(ctx, "dir1")
assertSync.snapshotContains(append(repoFiles, ".gitignore"))
assertSync.snapshotContains(repoFiles)
}
func TestSyncNestedFolderDoesntFailOnNonEmptyDirectory(t *testing.T) {
ctx, assertSync := setupSyncTest(t, "--watch")
// .gitignore is created by the sync process to enforce .databricks is not synced
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
// New file
localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt")
@ -353,9 +349,8 @@ func TestSyncNestedFolderDoesntFailOnNonEmptyDirectory(t *testing.T) {
func TestSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) {
ctx, assertSync := setupSyncTest(t, "--watch")
// .gitignore is created by the sync process to enforce .databricks is not synced
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
// New file
localFilePath := filepath.Join(assertSync.localRoot, "dir1/a b+c/c+d e/e+f g#i.txt")
@ -364,17 +359,17 @@ func TestSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) {
f := testfile.CreateFile(t, localFilePath)
defer f.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "dir1"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1"))
assertSync.remoteDirContent(ctx, "dir1", []string{"a b+c"})
assertSync.remoteDirContent(ctx, "dir1/a b+c", []string{"c+d e"})
assertSync.remoteDirContent(ctx, "dir1/a b+c/c+d e", []string{"e+f g#i.txt"})
assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/a b+c/c+d e/e+f g#i.txt"))
assertSync.snapshotContains(append(repoFiles, "dir1/a b+c/c+d e/e+f g#i.txt"))
// delete
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteNotExist(ctx, "dir1/a b+c/c+d e")
assertSync.snapshotContains(append(repoFiles, ".gitignore"))
assertSync.snapshotContains(repoFiles)
}
// This is a check for the edge case when a user does the following:
@ -395,23 +390,23 @@ func TestSyncIncrementalFileOverwritesFolder(t *testing.T) {
f := testfile.CreateFile(t, localFilePath)
defer f.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo"))
assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"})
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo/bar.txt"))
assertSync.snapshotContains(append(repoFiles, "foo/bar.txt"))
// delete foo/bar.txt
f.Remove(t)
os.Remove(filepath.Join(assertSync.localRoot, "foo"))
assertSync.waitForCompletionMarker()
assertSync.remoteNotExist(ctx, "foo")
assertSync.snapshotContains(append(repoFiles, ".gitignore"))
assertSync.snapshotContains(repoFiles)
f2 := testfile.CreateFile(t, filepath.Join(assertSync.localRoot, "foo"))
defer f2.Close(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo"))
assertSync.objectType(ctx, "foo", "FILE")
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo"))
assertSync.snapshotContains(append(repoFiles, "foo"))
}
func TestSyncIncrementalSyncPythonNotebookToFile(t *testing.T) {
@ -425,23 +420,23 @@ func TestSyncIncrementalSyncPythonNotebookToFile(t *testing.T) {
// notebook was uploaded properly
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo"))
assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON")
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
assertSync.snapshotContains(append(repoFiles, "foo.py"))
// convert to vanilla python file
f.Overwrite(t, "# No longer a python notebook")
assertSync.waitForCompletionMarker()
assertSync.objectType(ctx, "foo.py", "FILE")
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py"))
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py"))
assertSync.snapshotContains(append(repoFiles, "foo.py"))
// delete the vanilla python file
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.snapshotContains(append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
assertSync.snapshotContains(repoFiles)
}
func TestSyncIncrementalSyncFileToPythonNotebook(t *testing.T) {
@ -454,17 +449,17 @@ func TestSyncIncrementalSyncFileToPythonNotebook(t *testing.T) {
assertSync.waitForCompletionMarker()
// assert file upload
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo.py"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py"))
assertSync.objectType(ctx, "foo.py", "FILE")
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
assertSync.snapshotContains(append(repoFiles, "foo.py"))
// convert to notebook
f.Overwrite(t, "# Databricks notebook source")
assertSync.waitForCompletionMarker()
assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON")
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo.py"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo"))
assertSync.snapshotContains(append(repoFiles, "foo.py"))
}
func TestSyncIncrementalSyncPythonNotebookDelete(t *testing.T) {
@ -478,14 +473,14 @@ func TestSyncIncrementalSyncPythonNotebookDelete(t *testing.T) {
assertSync.waitForCompletionMarker()
// notebook was uploaded properly
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo"))
assertSync.objectType(ctx, "foo", "NOTEBOOK")
assertSync.language(ctx, "foo", "PYTHON")
// Delete notebook
f.Remove(t)
assertSync.waitForCompletionMarker()
assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore"))
assertSync.remoteDirContent(ctx, "", repoFiles)
}
func TestSyncEnsureRemotePathIsUsableIfRepoDoesntExist(t *testing.T) {

4
libs/git/fileset.go
View File

@ -43,7 +43,3 @@ func (f *FileSet) Files() ([]fileset.File, error) {
f.view.repo.taintIgnoreRules()
return f.fileset.Files()
}
func (f *FileSet) EnsureValidGitIgnoreExists() error {
return f.view.EnsureValidGitIgnoreExists()
}

33
libs/git/fileset_test.go
View File

@ -1,10 +1,8 @@
package git
import (
"os"
"path"
"path/filepath"
"strings"
"testing"
"github.com/databricks/cli/libs/vfs"
@ -51,34 +49,3 @@ func TestFileSetNonCleanRoot(t *testing.T) {
require.NoError(t, err)
assert.Len(t, files, 3)
}
func TestFileSetAddsCacheDirToGitIgnore(t *testing.T) {
projectDir := t.TempDir()
fileSet, err := NewFileSetAtRoot(vfs.MustNew(projectDir))
require.NoError(t, err)
err = fileSet.EnsureValidGitIgnoreExists()
require.NoError(t, err)
gitIgnorePath := filepath.Join(projectDir, ".gitignore")
assert.FileExists(t, gitIgnorePath)
fileBytes, err := os.ReadFile(gitIgnorePath)
assert.NoError(t, err)
assert.Contains(t, string(fileBytes), ".databricks")
}
func TestFileSetDoesNotCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) {
projectDir := t.TempDir()
gitIgnorePath := filepath.Join(projectDir, ".gitignore")
fileSet, err := NewFileSetAtRoot(vfs.MustNew(projectDir))
require.NoError(t, err)
err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644)
require.NoError(t, err)
err = fileSet.EnsureValidGitIgnoreExists()
require.NoError(t, err)
b, err := os.ReadFile(gitIgnorePath)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(string(b), ".databricks"))
}

33
libs/git/view.go
View File

@ -90,46 +90,25 @@ func NewView(worktreeRoot, root vfs.Path) (*View, error) {
target = strings.TrimPrefix(target, string(os.PathSeparator))
target = path.Clean(filepath.ToSlash(target))
return &View{
result := &View{
repo: repo,
targetPath: target,
}, nil
}
result.SetupDefaults()
return result, nil
}
func NewViewAtRoot(root vfs.Path) (*View, error) {
return NewView(root, root)
}
func (v *View) EnsureValidGitIgnoreExists() error {
ign, err := v.IgnoreDirectory(".databricks")
if err != nil {
return err
}
// return early if .databricks is already being ignored
if ign {
return nil
}
// Create .gitignore with .databricks entry
gitIgnorePath := filepath.Join(v.repo.Root(), v.targetPath, ".gitignore")
file, err := os.OpenFile(gitIgnorePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
if err != nil {
return err
}
defer file.Close()
func (v *View) SetupDefaults() {
// Hard code .databricks ignore pattern so that we never sync it (irrespective)
// of .gitignore patterns
v.repo.addIgnoreRule(newStringIgnoreRules([]string{
".databricks",
}))
_, err = file.WriteString("\n.databricks\n")
if err != nil {
return err
}
v.repo.taintIgnoreRules()
return nil
}

88
libs/git/view_test.go
View File

@ -209,100 +209,12 @@ func TestViewABInTempDir(t *testing.T) {
assert.False(t, tv.Ignore("newfile"))
}
func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredAtRoot(t *testing.T) {
expected, err := os.ReadFile("./testdata_view_ignore/.gitignore")
require.NoError(t, err)
repoPath := createFakeRepo(t, "testdata_view_ignore")
// Since root .gitignore already has .databricks, there should be no edits
// to root .gitignore
v, err := NewViewAtRoot(vfs.MustNew(repoPath))
require.NoError(t, err)
err = v.EnsureValidGitIgnoreExists()
require.NoError(t, err)
actual, err := os.ReadFile(filepath.Join(repoPath, ".gitignore"))
require.NoError(t, err)
assert.Equal(t, string(expected), string(actual))
}
func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredInSubdir(t *testing.T) {
expected, err := os.ReadFile("./testdata_view_ignore/a/.gitignore")
require.NoError(t, err)
repoPath := createFakeRepo(t, "testdata_view_ignore")
// Since root .gitignore already has .databricks, there should be no edits
// to a/.gitignore
v, err := NewView(vfs.MustNew(repoPath), vfs.MustNew(filepath.Join(repoPath, "a")))
require.NoError(t, err)
err = v.EnsureValidGitIgnoreExists()
require.NoError(t, err)
actual, err := os.ReadFile(filepath.Join(repoPath, v.targetPath, ".gitignore"))
require.NoError(t, err)
assert.Equal(t, string(expected), string(actual))
}
func TestViewAddsGitignoreWithCacheDir(t *testing.T) {
repoPath := createFakeRepo(t, "testdata")
err := os.Remove(filepath.Join(repoPath, ".gitignore"))
assert.NoError(t, err)
// Since root .gitignore was deleted, new view adds .databricks to root .gitignore
v, err := NewViewAtRoot(vfs.MustNew(repoPath))
require.NoError(t, err)
err = v.EnsureValidGitIgnoreExists()
require.NoError(t, err)
actual, err := os.ReadFile(filepath.Join(repoPath, ".gitignore"))
require.NoError(t, err)
assert.Contains(t, string(actual), "\n.databricks\n")
}
func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) {
repoPath := createFakeRepo(t, "testdata")
err := os.Remove(filepath.Join(repoPath, ".gitignore"))
require.NoError(t, err)
// Since root .gitignore was deleted, new view adds .databricks to a/.gitignore
v, err := NewView(vfs.MustNew(repoPath), vfs.MustNew(filepath.Join(repoPath, "a")))
require.NoError(t, err)
err = v.EnsureValidGitIgnoreExists()
require.NoError(t, err)
actual, err := os.ReadFile(filepath.Join(repoPath, v.targetPath, ".gitignore"))
require.NoError(t, err)
// created .gitignore has cache dir listed
assert.Contains(t, string(actual), "\n.databricks\n")
assert.NoFileExists(t, filepath.Join(repoPath, ".gitignore"))
}
func TestViewAlwaysIgnoresCacheDir(t *testing.T) {
repoPath := createFakeRepo(t, "testdata")
v, err := NewViewAtRoot(vfs.MustNew(repoPath))
require.NoError(t, err)
err = v.EnsureValidGitIgnoreExists()
require.NoError(t, err)
// Delete root .gitignore which contains .databricks entry
err = os.Remove(filepath.Join(repoPath, ".gitignore"))
require.NoError(t, err)
// taint rules to reload .gitignore
v.repo.taintIgnoreRules()
// assert .databricks is still being ignored
ign1, err := v.IgnoreDirectory(".databricks")
require.NoError(t, err)

4
libs/jsonschema/extension.go
View File

@ -40,4 +40,8 @@ type Extension struct {
// https://code.visualstudio.com/docs/languages/json#_use-rich-formatting-in-hovers
// Also it can be used in documentation generation.
MarkdownDescription string `json:"markdownDescription,omitempty"`
// This field is not in the JSON schema spec, but it is supported in VSCode
// It is used to provide a warning for deprectated fields
DeprecationMessage string `json:"deprecationMessage,omitempty"`
}

8
libs/jsonschema/from_type.go
View File

@ -18,10 +18,6 @@ var skipTags = []string{
// Annotation for internal bundle fields that should not be exposed to customers.
// Fields can be tagged as "internal" to remove them from the generated schema.
"internal",
// Annotation for bundle fields that have been deprecated.
// Fields tagged as "deprecated" are omitted from the generated schema.
"deprecated",
}
type constructor struct {
@ -259,8 +255,8 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) {
structFields := getStructFields(typ)
for _, structField := range structFields {
bundleTags := strings.Split(structField.Tag.Get("bundle"), ",")
// Fields marked as "readonly", "internal" or "deprecated" are skipped
// while generating the schema
// Fields marked as "readonly" or "internal" are skipped while generating
// the schema
skip := false
for _, tag := range skipTags {
if slices.Contains(bundleTags, tag) {

9
libs/jsonschema/from_type_test.go
View File

@ -17,11 +17,10 @@ func TestFromTypeBasic(t *testing.T) {
TriplePointer ***int `json:"triple_pointer,omitempty"`
// These fields should be ignored in the resulting schema.
NotAnnotated string
DashedTag string `json:"-"`
InternalTagged string `json:"internal_tagged" bundle:"internal"`
DeprecatedTagged string `json:"deprecated_tagged" bundle:"deprecated"`
ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"`
NotAnnotated string
DashedTag string `json:"-"`
InternalTagged string `json:"internal_tagged" bundle:"internal"`
ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"`
}
strRef := "#/$defs/string"

5
libs/jsonschema/schema.go
View File

@ -80,6 +80,11 @@ type Schema struct {
// Examples of the value for properties in the schema.
// https://json-schema.org/understanding-json-schema/reference/annotations
Examples any `json:"examples,omitempty"`
// A boolean that indicates the field should not be used and may be removed
// in the future.
// https://json-schema.org/understanding-json-schema/reference/annotations
Deprecated bool `json:"deprecated,omitempty"`
}
// Default value defined in a JSON Schema, represented as a string.

26
libs/sync/gitignore.go Normal file
View File

@ -0,0 +1,26 @@
package sync
import (
"context"
"os"
"path/filepath"
"github.com/databricks/cli/libs/log"
)
func WriteGitIgnore(ctx context.Context, dir string) {
gitignorePath := filepath.Join(dir, ".databricks", ".gitignore")
file, err := os.OpenFile(gitignorePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
if err != nil {
if os.IsExist(err) {
return
}
log.Debugf(ctx, "Failed to create %s: %s", gitignorePath, err)
}
defer file.Close()
_, err = file.WriteString("*\n")
if err != nil {
log.Debugf(ctx, "Error writing to %s: %s", gitignorePath, err)
}
}

5
libs/sync/sync.go
View File

@ -69,10 +69,7 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) {
return nil, err
}
err = fileSet.EnsureValidGitIgnoreExists()
if err != nil {
return nil, err
}
WriteGitIgnore(ctx, opts.LocalRoot.Native())
includeFileSet, err := fileset.NewGlobSet(opts.LocalRoot, opts.Include)
if err != nil {

17
libs/sync/sync_test.go
View File

@ -40,9 +40,6 @@ func TestGetFileSet(t *testing.T) {
fileSet, err := git.NewFileSetAtRoot(root)
require.NoError(t, err)
err = fileSet.EnsureValidGitIgnoreExists()
require.NoError(t, err)
inc, err := fileset.NewGlobSet(root, []string{})
require.NoError(t, err)
@ -59,7 +56,7 @@ func TestGetFileSet(t *testing.T) {
fileList, err := s.GetFileList(ctx)
require.NoError(t, err)
require.Len(t, fileList, 10)
require.Len(t, fileList, 9)
inc, err = fileset.NewGlobSet(root, []string{})
require.NoError(t, err)
@ -77,7 +74,7 @@ func TestGetFileSet(t *testing.T) {
fileList, err = s.GetFileList(ctx)
require.NoError(t, err)
require.Len(t, fileList, 2)
require.Len(t, fileList, 1)
inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"})
require.NoError(t, err)
@ -95,7 +92,7 @@ func TestGetFileSet(t *testing.T) {
fileList, err = s.GetFileList(ctx)
require.NoError(t, err)
require.Len(t, fileList, 11)
require.Len(t, fileList, 10)
}
func TestRecursiveExclude(t *testing.T) {
@ -106,9 +103,6 @@ func TestRecursiveExclude(t *testing.T) {
fileSet, err := git.NewFileSetAtRoot(root)
require.NoError(t, err)
err = fileSet.EnsureValidGitIgnoreExists()
require.NoError(t, err)
inc, err := fileset.NewGlobSet(root, []string{})
require.NoError(t, err)
@ -125,7 +119,7 @@ func TestRecursiveExclude(t *testing.T) {
fileList, err := s.GetFileList(ctx)
require.NoError(t, err)
require.Len(t, fileList, 7)
require.Len(t, fileList, 6)
}
func TestNegateExclude(t *testing.T) {
@ -136,9 +130,6 @@ func TestNegateExclude(t *testing.T) {
fileSet, err := git.NewFileSetAtRoot(root)
require.NoError(t, err)
err = fileSet.EnsureValidGitIgnoreExists()
require.NoError(t, err)
inc, err := fileset.NewGlobSet(root, []string{})
require.NoError(t, err)

6
libs/telemetry/protos/databricks_cli_log.go
View File

@ -24,10 +24,12 @@ type ExecutionContext struct {
FromWebTerminal bool `json:"from_web_terminal,omitempty"`
// Time taken for the CLI command to execute.
ExecutionTimeMs int64 `json:"execution_time_ms,omitempty"`
// We want to serialize the zero value as well so the omitempty tag is not set.
ExecutionTimeMs int64 `json:"execution_time_ms"`
// Exit code of the CLI command.
ExitCode int64 `json:"exit_code,omitempty"`
// We want to serialize the zero value as well so the omitempty tag is not set.
ExitCode int64 `json:"exit_code"`
}
type CliTestEvent struct {

2
libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl
View File

@ -13,7 +13,7 @@ resources:
{{- else}}
catalog: {{default_catalog}}
{{- end}}
target: {{.project_name}}_${bundle.target}
schema: {{.project_name}}_${bundle.target}
{{- if $with_serverless }}
serverless: true
{{- end}}