Support relative paths in artifact files source section and always upload all artifact files (#1247)

Support relative paths in artifact files source section and always
upload all artifact files

Fixes #1156

## Tests
Added unit tests
This commit is contained in:
Andrew Nester 2024-03-04 21:28:15 +01:00 committed by GitHub
parent 09d1846e13
commit ecf9c52f61
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 138 additions and 8 deletions

View File

@ -121,13 +121,6 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
for i := range a.Files {
f := &a.Files[i]
// Lookup all tasks that reference this file.
libs, ok := filesToLibraries[f.Source]
if !ok {
log.Debugf(ctx, "No tasks reference %s. Skipping upload.", f.Source)
continue
}
filename := filepath.Base(f.Source)
cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename))
@ -139,6 +132,13 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
log.Infof(ctx, "Upload succeeded")
f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source))
// Lookup all tasks that reference this file.
libs, ok := filesToLibraries[f.Source]
if !ok {
log.Debugf(ctx, "No tasks reference %s", f.Source)
continue
}
// Update all tasks that reference this file.
for _, lib := range libs {
wsfsBase := "/Workspace"

View File

@ -49,7 +49,8 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error {
}
if !filepath.IsAbs(artifact.Path) {
artifact.Path = filepath.Join(b.Config.Path, artifact.Path)
dirPath := filepath.Dir(artifact.ConfigFilePath)
artifact.Path = filepath.Join(dirPath, artifact.Path)
}
return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name))

View File

@ -3,8 +3,10 @@ package artifacts
import (
"context"
"fmt"
"path/filepath"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/databricks-sdk-go/service/workspace"
)
@ -41,6 +43,35 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error {
return fmt.Errorf("artifact source is not configured: %s", m.name)
}
// Check if source paths are absolute, if not, make them absolute
for k := range artifact.Files {
f := &artifact.Files[k]
if !filepath.IsAbs(f.Source) {
dirPath := filepath.Dir(artifact.ConfigFilePath)
f.Source = filepath.Join(dirPath, f.Source)
}
}
// Expand any glob reference in files source path
files := make([]config.ArtifactFile, 0, len(artifact.Files))
for _, f := range artifact.Files {
matches, err := filepath.Glob(f.Source)
if err != nil {
return fmt.Errorf("unable to find files for %s: %w", f.Source, err)
}
if len(matches) == 0 {
return fmt.Errorf("no files found for %s", f.Source)
}
for _, match := range matches {
files = append(files, config.ArtifactFile{
Source: match,
})
}
}
artifact.Files = files
return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name))
}

View File

@ -0,0 +1,98 @@
package artifacts
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/internal/bundletest"
"github.com/databricks/cli/libs/testfile"
"github.com/stretchr/testify/require"
)
type noop struct{}
func (n *noop) Apply(context.Context, *bundle.Bundle) error {
return nil
}
func (n *noop) Name() string {
return "noop"
}
func TestExpandGlobFilesSource(t *testing.T) {
rootPath := t.TempDir()
err := os.Mkdir(filepath.Join(rootPath, "test"), 0755)
require.NoError(t, err)
t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar"))
t1.Close(t)
t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar"))
t2.Close(t)
b := &bundle.Bundle{
Config: config.Root{
Path: rootPath,
Artifacts: map[string]*config.Artifact{
"test": {
Type: "custom",
Files: []config.ArtifactFile{
{
Source: filepath.Join("..", "test", "*.jar"),
},
},
},
},
},
}
bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml"))
u := &upload{"test"}
uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator {
return &noop{}
}
err = bundle.Apply(context.Background(), b, u)
require.NoError(t, err)
require.Equal(t, 2, len(b.Config.Artifacts["test"].Files))
require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source)
require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source)
}
func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) {
rootPath := t.TempDir()
err := os.Mkdir(filepath.Join(rootPath, "test"), 0755)
require.NoError(t, err)
b := &bundle.Bundle{
Config: config.Root{
Path: rootPath,
Artifacts: map[string]*config.Artifact{
"test": {
Type: "custom",
Files: []config.ArtifactFile{
{
Source: filepath.Join("..", "test", "myjar.jar"),
},
},
},
},
},
}
bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml"))
u := &upload{"test"}
uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator {
return &noop{}
}
err = bundle.Apply(context.Background(), b, u)
require.ErrorContains(t, err, "no files found for")
}