Merge bbcbffd7f4 into abfd1713e0

## Changes

Users can configure the bundle to not synchronize any files with:
```yaml
sync:
  paths: []
```

If it is explicitly configured as an empty list, the validate command
must not warn about not having any files to synchronize. The warning
exists to alert users who are unintentionally not synchronizing any
files (they might have a `.gitignore` pattern that matches everything).

Closes #1663.

## Tests

* New unit test.

.github/workflows/push.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       - name: Set go env
         run: |

bundle/config/validate/files_to_sync.go

@@ -21,6 +21,12 @@ func (v *filesToSync) Name() string {
 }
 
 func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
+	// The user may be intentional about not synchronizing any files.
+	// In this case, we should not show any warnings.
+	if len(rb.Config().Sync.Paths) == 0 {
+		return nil
+	}
+
 	sync, err := files.GetSync(ctx, rb)
 	if err != nil {
 		return diag.FromErr(err)
@@ -31,6 +37,7 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.
 		return diag.FromErr(err)
 	}
 
+	// If there are files to sync, we don't need to show any warnings.
 	if len(fl) != 0 {
 		return nil
 	}

bundle/config/validate/files_to_sync_test.go

@@ -0,0 +1,105 @@
+package validate
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/vfs"
+	sdkconfig "github.com/databricks/databricks-sdk-go/config"
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilesToSync_NoPaths(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Sync: config.Sync{
+				Paths: []string{},
+			},
+		},
+	}
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	assert.Empty(t, diags)
+}
+
+func setupBundleForFilesToSyncTest(t *testing.T) *bundle.Bundle {
+	dir := t.TempDir()
+
+	testutil.Touch(t, dir, "file1")
+	testutil.Touch(t, dir, "file2")
+
+	b := &bundle.Bundle{
+		BundleRootPath: dir,
+		BundleRoot:     vfs.MustNew(dir),
+		SyncRootPath:   dir,
+		SyncRoot:       vfs.MustNew(dir),
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Target: "default",
+			},
+			Workspace: config.Workspace{
+				FilePath: "/this/doesnt/matter",
+				CurrentUser: &config.User{
+					User: &iam.User{},
+				},
+			},
+			Sync: config.Sync{
+				// Paths are relative to [SyncRootPath].
+				Paths: []string{"."},
+			},
+		},
+	}
+
+	m := mocks.NewMockWorkspaceClient(t)
+	m.WorkspaceClient.Config = &sdkconfig.Config{
+		Host: "https://foo.com",
+	}
+
+	// The initialization logic in [sync.New] performs a check on the destination path.
+	// Removing this check at initialization time is tbd...
+	m.GetMockWorkspaceAPI().EXPECT().GetStatusByPath(mock.Anything, "/this/doesnt/matter").Return(&workspace.ObjectInfo{
+		ObjectType: workspace.ObjectTypeDirectory,
+	}, nil)
+
+	b.SetWorkpaceClient(m.WorkspaceClient)
+	return b
+}
+
+func TestFilesToSync_EverythingIgnored(t *testing.T) {
+	b := setupBundleForFilesToSyncTest(t)
+
+	// Ignore all files.
+	testutil.WriteFile(t, "*\n.*\n", b.BundleRootPath, ".gitignore")
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	require.Equal(t, 1, len(diags))
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+	assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary)
+}
+
+func TestFilesToSync_EverythingExcluded(t *testing.T) {
+	b := setupBundleForFilesToSyncTest(t)
+
+	// Exclude all files.
+	b.Config.Sync.Exclude = []string{"*"}
+
+	ctx := context.Background()
+	rb := bundle.ReadOnly(b)
+	diags := bundle.ApplyReadOnly(ctx, rb, FilesToSync())
+	require.Equal(t, 1, len(diags))
+	assert.Equal(t, diag.Warning, diags[0].Severity)
+	assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary)
+}

bundle/config/variable/lookup.go

@@ -22,6 +22,8 @@ type Lookup struct {
 
 	Metastore string `json:"metastore,omitempty"`
 
+	NotificationDestination string `json:"notification_destination,omitempty"`
+
 	Pipeline string `json:"pipeline,omitempty"`
 
 	Query string `json:"query,omitempty"`
@@ -63,6 +65,9 @@ func (l *Lookup) constructResolver() (resolver, error) {
 	if l.Metastore != "" {
 		resolvers = append(resolvers, resolveMetastore{name: l.Metastore})
 	}
+	if l.NotificationDestination != "" {
+		resolvers = append(resolvers, resolveNotificationDestination{name: l.NotificationDestination})
+	}
 	if l.Pipeline != "" {
 		resolvers = append(resolvers, resolvePipeline{name: l.Pipeline})
 	}

bundle/config/variable/resolve_notification_destination.go

@@ -0,0 +1,46 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/settings"
+)
+
+type resolveNotificationDestination struct {
+	name string
+}
+
+func (l resolveNotificationDestination) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	result, err := w.NotificationDestinations.ListAll(ctx, settings.ListNotificationDestinationsRequest{
+		// The default page size for this API is 20.
+		// We use a higher value to make fewer API calls.
+		PageSize: 200,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	// Collect all notification destinations with the given name.
+	var entities []settings.ListNotificationDestinationsResult
+	for _, entity := range result {
+		if entity.DisplayName == l.name {
+			entities = append(entities, entity)
+		}
+	}
+
+	// Return the ID of the first matching notification destination.
+	switch len(entities) {
+	case 0:
+		return "", fmt.Errorf("notification destination named %q does not exist", l.name)
+	case 1:
+		return entities[0].Id, nil
+	default:
+		return "", fmt.Errorf("there are %d instances of clusters named %q", len(entities), l.name)
+	}
+}
+
+func (l resolveNotificationDestination) String() string {
+	return fmt.Sprintf("notification-destination: %s", l.name)
+}

bundle/config/variable/resolve_notification_destination_test.go

@@ -0,0 +1,82 @@
+package variable
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/settings"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveNotificationDestination_ResolveSuccess(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{
+			{Id: "1234", DisplayName: "destination"},
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	result, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.NoError(t, err)
+	assert.Equal(t, "1234", result)
+}
+
+func TestResolveNotificationDestination_ResolveError(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return(nil, fmt.Errorf("bad"))
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	assert.ErrorContains(t, err, "bad")
+}
+
+func TestResolveNotificationDestination_ResolveNotFound(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `notification destination named "destination" does not exist`)
+}
+
+func TestResolveNotificationDestination_ResolveMultiple(t *testing.T) {
+	m := mocks.NewMockWorkspaceClient(t)
+
+	api := m.GetMockNotificationDestinationsAPI()
+	api.EXPECT().
+		ListAll(mock.Anything, mock.Anything).
+		Return([]settings.ListNotificationDestinationsResult{
+			{Id: "1234", DisplayName: "destination"},
+			{Id: "5678", DisplayName: "destination"},
+		}, nil)
+
+	ctx := context.Background()
+	l := resolveNotificationDestination{name: "destination"}
+	_, err := l.Resolve(ctx, m.WorkspaceClient)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, `there are 2 instances of clusters named "destination"`)
+}
+
+func TestResolveNotificationDestination_String(t *testing.T) {
+	l := resolveNotificationDestination{name: "name"}
+	assert.Equal(t, "notification-destination: name", l.String())
+}

bundle/tests/python_import/dataclass_no_wheel/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: python-import-dataclass-no-wheel
+
+experimental:
+  pydabs:
+    enabled: true
+    import:
+    - "my_job"
+
+variables:
+  default_cluster_spec:
+    type: complex
+    value:
+      num_workers: 1
+      spark_version: "15.4.x-scala2.12"

bundle/tests/python_import/dataclass_no_wheel/notebooks/my_notebook.py

@@ -0,0 +1,2 @@
+# Databricks Notebook Source
+1+1

bundle/tests/python_import/dataclass_no_wheel/src/my_job.py

@@ -0,0 +1,22 @@
+from databricks.bundles.jobs import Job, Task, NotebookTask, JobCluster
+from databricks.bundles.variables import Bundle
+
+my_job = Job(
+    name="Test Job",
+    resource_name="my_job",
+    job_clusters=[
+        JobCluster(
+            job_cluster_key="my_cluster",
+            new_cluster=Bundle.variables.default_cluster_spec,
+        ),
+    ],
+    tasks=[
+        Task(
+            task_key="my_notebook_task",
+            job_cluster_key="my_cluster",
+            notebook_task=NotebookTask(
+                notebook_path="notebooks/my_notebook.py",
+            ),
+        ),
+    ],
+)

bundle/tests/python_import_test.go

@@ -0,0 +1,95 @@
+package config_tests
+
+import (
+	"os"
+	"os/exec"
+	pathlib "path"
+	"runtime"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPythonImport_dataclass_no_wheel(t *testing.T) {
+	activateVEnv(t)
+	setPythonPath(t, "python_import/dataclass_no_wheel/src")
+
+	expected := &resources.Job{
+		JobSettings: &jobs.JobSettings{
+			Name: "Test Job",
+			JobClusters: []jobs.JobCluster{
+				{
+					JobClusterKey: "my_cluster",
+				},
+			},
+			Tasks: []jobs.Task{
+				{
+					NotebookTask: &jobs.NotebookTask{
+						NotebookPath: "notebooks/my_notebook.py",
+					},
+					JobClusterKey: "my_cluster",
+					TaskKey:       "my_notebook_task",
+				},
+			},
+		},
+	}
+
+	b := load(t, "./python_import/dataclass_no_wheel")
+
+	assert.Equal(t, []string{"my_job"}, maps.Keys(b.Config.Resources.Jobs))
+
+	myJob := b.Config.Resources.Jobs["my_job"]
+	assert.Equal(t, expected, myJob)
+
+	// NewCluster is reference to a variable and needs to be checked separately
+	err := b.Config.Mutate(func(value dyn.Value) (dyn.Value, error) {
+		path := dyn.MustPathFromString("resources.jobs.my_job.job_clusters[0].new_cluster")
+		value, err := dyn.GetByPath(value, path)
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		assert.Equal(t, "${var.default_cluster_spec}", value.AsAny())
+
+		return value, nil
+	})
+	require.NoError(t, err)
+}
+
+func setPythonPath(t *testing.T, path string) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+	t.Setenv("PYTHONPATH", pathlib.Join(wd, path))
+}
+
+func activateVEnv(t *testing.T) {
+	dir := t.TempDir()
+	venvDir := pathlib.Join(dir, "venv")
+
+	err := exec.Command("python3", "-m", "venv", venvDir).Run()
+	require.NoError(t, err, "failed to create venv")
+
+	// we don't have shell to activate venv, updating PATH is enough
+
+	var venvBinDir string
+	if runtime.GOOS == "windows" {
+		venvBinDir = pathlib.Join(venvDir, "Scripts")
+		t.Setenv("PATH", venvBinDir+";"+os.Getenv("PATH"))
+	} else {
+		venvBinDir = pathlib.Join(venvDir, "bin")
+		t.Setenv("PATH", venvBinDir+":"+os.Getenv("PATH"))
+	}
+
+	err = exec.Command(
+		pathlib.Join(venvBinDir, "pip"),
+		"install",
+		"databricks-pydabs==0.5.1",
+	).Run()
+	require.NoError(t, err, "failed to install databricks-pydabs")
+}