Support relative paths in artifact files source section and always upload all artifact files (#1247)

Support relative paths in artifact files source section and always
upload all artifact files

Fixes #1156

## Tests
Added unit tests

bundle/artifacts/artifacts.go

@@ -121,13 +121,6 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
 	for i := range a.Files {
 		f := &a.Files[i]
 
-		// Lookup all tasks that reference this file.
-		libs, ok := filesToLibraries[f.Source]
-		if !ok {
-			log.Debugf(ctx, "No tasks reference %s. Skipping upload.", f.Source)
-			continue
-		}
-
 		filename := filepath.Base(f.Source)
 		cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename))
 
@@ -139,6 +132,13 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
 		log.Infof(ctx, "Upload succeeded")
 		f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source))
 
+		// Lookup all tasks that reference this file.
+		libs, ok := filesToLibraries[f.Source]
+		if !ok {
+			log.Debugf(ctx, "No tasks reference %s", f.Source)
+			continue
+		}
+
 		// Update all tasks that reference this file.
 		for _, lib := range libs {
 			wsfsBase := "/Workspace"

bundle/artifacts/build.go

@@ -49,7 +49,8 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error {
 	}
 
 	if !filepath.IsAbs(artifact.Path) {
-		artifact.Path = filepath.Join(b.Config.Path, artifact.Path)
+		dirPath := filepath.Dir(artifact.ConfigFilePath)
+		artifact.Path = filepath.Join(dirPath, artifact.Path)
 	}
 
 	return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name))

bundle/artifacts/upload.go

@@ -3,8 +3,10 @@ package artifacts
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
@@ -41,6 +43,35 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error {
 		return fmt.Errorf("artifact source is not configured: %s", m.name)
 	}
 
+	// Check if source paths are absolute, if not, make them absolute
+	for k := range artifact.Files {
+		f := &artifact.Files[k]
+		if !filepath.IsAbs(f.Source) {
+			dirPath := filepath.Dir(artifact.ConfigFilePath)
+			f.Source = filepath.Join(dirPath, f.Source)
+		}
+	}
+
+	// Expand any glob reference in files source path
+	files := make([]config.ArtifactFile, 0, len(artifact.Files))
+	for _, f := range artifact.Files {
+		matches, err := filepath.Glob(f.Source)
+		if err != nil {
+			return fmt.Errorf("unable to find files for %s: %w", f.Source, err)
+		}
+
+		if len(matches) == 0 {
+			return fmt.Errorf("no files found for %s", f.Source)
+		}
+
+		for _, match := range matches {
+			files = append(files, config.ArtifactFile{
+				Source: match,
+			})
+		}
+	}
+
+	artifact.Files = files
 	return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name))
 }
 

bundle/artifacts/upload_test.go

@@ -0,0 +1,98 @@
+package artifacts
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/testfile"
+	"github.com/stretchr/testify/require"
+)
+
+type noop struct{}
+
+func (n *noop) Apply(context.Context, *bundle.Bundle) error {
+	return nil
+}
+
+func (n *noop) Name() string {
+	return "noop"
+}
+
+func TestExpandGlobFilesSource(t *testing.T) {
+	rootPath := t.TempDir()
+	err := os.Mkdir(filepath.Join(rootPath, "test"), 0755)
+	require.NoError(t, err)
+
+	t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar"))
+	t1.Close(t)
+
+	t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar"))
+	t2.Close(t)
+
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Path: rootPath,
+			Artifacts: map[string]*config.Artifact{
+				"test": {
+					Type: "custom",
+					Files: []config.ArtifactFile{
+						{
+							Source: filepath.Join("..", "test", "*.jar"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml"))
+
+	u := &upload{"test"}
+	uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator {
+		return &noop{}
+	}
+
+	err = bundle.Apply(context.Background(), b, u)
+	require.NoError(t, err)
+
+	require.Equal(t, 2, len(b.Config.Artifacts["test"].Files))
+	require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source)
+	require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source)
+}
+
+func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) {
+	rootPath := t.TempDir()
+	err := os.Mkdir(filepath.Join(rootPath, "test"), 0755)
+	require.NoError(t, err)
+
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Path: rootPath,
+			Artifacts: map[string]*config.Artifact{
+				"test": {
+					Type: "custom",
+					Files: []config.ArtifactFile{
+						{
+							Source: filepath.Join("..", "test", "myjar.jar"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml"))
+
+	u := &upload{"test"}
+	uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator {
+		return &noop{}
+	}
+
+	err = bundle.Apply(context.Background(), b, u)
+	require.ErrorContains(t, err, "no files found for")
+}