diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index e474240d..ce2e165b 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -121,13 +121,6 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u for i := range a.Files { f := &a.Files[i] - // Lookup all tasks that reference this file. - libs, ok := filesToLibraries[f.Source] - if !ok { - log.Debugf(ctx, "No tasks reference %s. Skipping upload.", f.Source) - continue - } - filename := filepath.Base(f.Source) cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) @@ -139,6 +132,13 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u log.Infof(ctx, "Upload succeeded") f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source)) + // Lookup all tasks that reference this file. + libs, ok := filesToLibraries[f.Source] + if !ok { + log.Debugf(ctx, "No tasks reference %s", f.Source) + continue + } + // Update all tasks that reference this file. for _, lib := range libs { wsfsBase := "/Workspace" diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 6b1aac82..a78958e6 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -49,7 +49,8 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error { } if !filepath.IsAbs(artifact.Path) { - artifact.Path = filepath.Join(b.Config.Path, artifact.Path) + dirPath := filepath.Dir(artifact.ConfigFilePath) + artifact.Path = filepath.Join(dirPath, artifact.Path) } return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 990718aa..61e65208 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -3,8 +3,10 @@ package artifacts import ( "context" "fmt" + "path/filepath" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/databricks-sdk-go/service/workspace" ) @@ -41,6 +43,35 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error { return fmt.Errorf("artifact source is not configured: %s", m.name) } + // Check if source paths are absolute, if not, make them absolute + for k := range artifact.Files { + f := &artifact.Files[k] + if !filepath.IsAbs(f.Source) { + dirPath := filepath.Dir(artifact.ConfigFilePath) + f.Source = filepath.Join(dirPath, f.Source) + } + } + + // Expand any glob reference in files source path + files := make([]config.ArtifactFile, 0, len(artifact.Files)) + for _, f := range artifact.Files { + matches, err := filepath.Glob(f.Source) + if err != nil { + return fmt.Errorf("unable to find files for %s: %w", f.Source, err) + } + + if len(matches) == 0 { + return fmt.Errorf("no files found for %s", f.Source) + } + + for _, match := range matches { + files = append(files, config.ArtifactFile{ + Source: match, + }) + } + } + + artifact.Files = files return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name)) } diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go new file mode 100644 index 00000000..6dea1c14 --- /dev/null +++ b/bundle/artifacts/upload_test.go @@ -0,0 +1,98 @@ +package artifacts + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/testfile" + "github.com/stretchr/testify/require" +) + +type noop struct{} + +func (n *noop) Apply(context.Context, *bundle.Bundle) error { + return nil +} + +func (n *noop) Name() string { + return "noop" +} + +func TestExpandGlobFilesSource(t *testing.T) { + rootPath := t.TempDir() + err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) + require.NoError(t, err) + + t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar")) + t1.Close(t) + + t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar")) + t2.Close(t) + + b := &bundle.Bundle{ + Config: config.Root{ + Path: rootPath, + Artifacts: map[string]*config.Artifact{ + "test": { + Type: "custom", + Files: []config.ArtifactFile{ + { + Source: filepath.Join("..", "test", "*.jar"), + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) + + u := &upload{"test"} + uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { + return &noop{} + } + + err = bundle.Apply(context.Background(), b, u) + require.NoError(t, err) + + require.Equal(t, 2, len(b.Config.Artifacts["test"].Files)) + require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source) + require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source) +} + +func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { + rootPath := t.TempDir() + err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) + require.NoError(t, err) + + b := &bundle.Bundle{ + Config: config.Root{ + Path: rootPath, + Artifacts: map[string]*config.Artifact{ + "test": { + Type: "custom", + Files: []config.ArtifactFile{ + { + Source: filepath.Join("..", "test", "myjar.jar"), + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) + + u := &upload{"test"} + uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { + return &noop{} + } + + err = bundle.Apply(context.Background(), b, u) + require.ErrorContains(t, err, "no files found for") +}