diff --git a/bundle/artifacts/artifacts_test.go b/bundle/artifacts/artifacts_test.go index 4c0a18f38..bbae44efa 100644 --- a/bundle/artifacts/artifacts_test.go +++ b/bundle/artifacts/artifacts_test.go @@ -105,6 +105,7 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) { b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{}) artifact := &config.Artifact{ + Type: "whl", Files: []config.ArtifactFile{ { Source: whlPath, @@ -118,4 +119,5 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) { err := uploadArtifact(context.Background(), artifact, b) require.NoError(t, err) require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath) + require.Regexp(t, regexp.MustCompile("/Workspace/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].Libraries[0].Whl) } diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index 1955e265d..d7048a02e 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -78,9 +78,13 @@ func (a *Artifact) NormalisePaths() { remotePath := path.Join(wsfsBase, f.RemotePath) for i := range f.Libraries { lib := f.Libraries[i] - switch a.Type { - case ArtifactPythonWheel: + if lib.Whl != "" { lib.Whl = remotePath + continue + } + if lib.Jar != "" { + lib.Jar = remotePath + continue } } diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index d9a257db8..8e2e504c5 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -78,6 +78,17 @@ func IsTaskWithLocalLibraries(task *jobs.Task) bool { return false } +func IsTaskWithWorkspaceLibraries(task *jobs.Task) bool { + for _, l := range task.Libraries { + path := libPath(&l) + if isWorkspacePath(path) { + return true + } + } + + return false +} + func isMissingRequiredLibraries(task *jobs.Task) bool { if task.Libraries != nil { return false diff --git a/bundle/python/transform.go b/bundle/python/transform.go index 3d744df9d..d8eb33f54 100644 --- a/bundle/python/transform.go +++ b/bundle/python/transform.go @@ -73,8 +73,11 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { for i := range tasks { task := &tasks[i] - // Keep only Python wheel tasks with local libraries referenced - if task.PythonWheelTask == nil || !libraries.IsTaskWithLocalLibraries(task) { + // Keep only Python wheel tasks with workspace libraries referenced. + // At this point of moment we don't have local paths in Libraries sections anymore + // Local paths have been replaced with the remote when the artifacts where uploaded + // in artifacts.UploadAll mutator. + if task.PythonWheelTask == nil || !needsTrampoline(task) { continue } @@ -87,6 +90,10 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { return result } +func needsTrampoline(task *jobs.Task) bool { + return libraries.IsTaskWithWorkspaceLibraries(task) +} + func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) { params, err := t.generateParameters(task.PythonWheelTask) if err != nil { diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index 99d3129d8..a7448f234 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -84,7 +84,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { TaskKey: "key1", PythonWheelTask: &jobs.PythonWheelTask{}, Libraries: []compute.Library{ - {Whl: "./dist/test.whl"}, + {Whl: "/Workspace/Users/test@test.com/bundle/dist/test.whl"}, }, }, {