Apply Python wheel trampoline if workspace library is used (#755)

## Changes
Workspace library will be detected by trampoline in 2 cases:
- User defined to use local wheel file
- User defined to use remote wheel file from Workspace file system

In both of these cases we should correctly apply Python trampoline

## Tests
Added a regression test (also covered by Python e2e test)
This commit is contained in:
Andrew Nester 2023-09-08 15:45:21 +02:00 committed by GitHub
parent 7c96270db8
commit 18a5b05d82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 5 deletions

View File

@ -105,6 +105,7 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{})
artifact := &config.Artifact{
Type: "whl",
Files: []config.ArtifactFile{
{
Source: whlPath,
@ -118,4 +119,5 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
err := uploadArtifact(context.Background(), artifact, b)
require.NoError(t, err)
require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath)
require.Regexp(t, regexp.MustCompile("/Workspace/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].Libraries[0].Whl)
}

View File

@ -78,9 +78,13 @@ func (a *Artifact) NormalisePaths() {
remotePath := path.Join(wsfsBase, f.RemotePath)
for i := range f.Libraries {
lib := f.Libraries[i]
switch a.Type {
case ArtifactPythonWheel:
if lib.Whl != "" {
lib.Whl = remotePath
continue
}
if lib.Jar != "" {
lib.Jar = remotePath
continue
}
}

View File

@ -78,6 +78,17 @@ func IsTaskWithLocalLibraries(task *jobs.Task) bool {
return false
}
func IsTaskWithWorkspaceLibraries(task *jobs.Task) bool {
for _, l := range task.Libraries {
path := libPath(&l)
if isWorkspacePath(path) {
return true
}
}
return false
}
func isMissingRequiredLibraries(task *jobs.Task) bool {
if task.Libraries != nil {
return false

View File

@ -73,8 +73,11 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
for i := range tasks {
task := &tasks[i]
// Keep only Python wheel tasks with local libraries referenced
if task.PythonWheelTask == nil || !libraries.IsTaskWithLocalLibraries(task) {
// Keep only Python wheel tasks with workspace libraries referenced.
// At this point of moment we don't have local paths in Libraries sections anymore
// Local paths have been replaced with the remote when the artifacts where uploaded
// in artifacts.UploadAll mutator.
if task.PythonWheelTask == nil || !needsTrampoline(task) {
continue
}
@ -87,6 +90,10 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
return result
}
func needsTrampoline(task *jobs.Task) bool {
return libraries.IsTaskWithWorkspaceLibraries(task)
}
func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) {
params, err := t.generateParameters(task.PythonWheelTask)
if err != nil {

View File

@ -84,7 +84,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) {
TaskKey: "key1",
PythonWheelTask: &jobs.PythonWheelTask{},
Libraries: []compute.Library{
{Whl: "./dist/test.whl"},
{Whl: "/Workspace/Users/test@test.com/bundle/dist/test.whl"},
},
},
{