Apply Python wheel trampoline if workspace library is used (#755)

## Changes
Workspace library will be detected by trampoline in 2 cases:
- User defined to use local wheel file
- User defined to use remote wheel file from Workspace file system

In both of these cases we should correctly apply Python trampoline

## Tests
Added a regression test (also covered by Python e2e test)
This commit is contained in:
Andrew Nester 2023-09-08 15:45:21 +02:00 committed by GitHub
parent 7c96270db8
commit 18a5b05d82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 5 deletions

View File

@ -105,6 +105,7 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{}) b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{})
artifact := &config.Artifact{ artifact := &config.Artifact{
Type: "whl",
Files: []config.ArtifactFile{ Files: []config.ArtifactFile{
{ {
Source: whlPath, Source: whlPath,
@ -118,4 +119,5 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
err := uploadArtifact(context.Background(), artifact, b) err := uploadArtifact(context.Background(), artifact, b)
require.NoError(t, err) require.NoError(t, err)
require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath) require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath)
require.Regexp(t, regexp.MustCompile("/Workspace/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].Libraries[0].Whl)
} }

View File

@ -78,9 +78,13 @@ func (a *Artifact) NormalisePaths() {
remotePath := path.Join(wsfsBase, f.RemotePath) remotePath := path.Join(wsfsBase, f.RemotePath)
for i := range f.Libraries { for i := range f.Libraries {
lib := f.Libraries[i] lib := f.Libraries[i]
switch a.Type { if lib.Whl != "" {
case ArtifactPythonWheel:
lib.Whl = remotePath lib.Whl = remotePath
continue
}
if lib.Jar != "" {
lib.Jar = remotePath
continue
} }
} }

View File

@ -78,6 +78,17 @@ func IsTaskWithLocalLibraries(task *jobs.Task) bool {
return false return false
} }
func IsTaskWithWorkspaceLibraries(task *jobs.Task) bool {
for _, l := range task.Libraries {
path := libPath(&l)
if isWorkspacePath(path) {
return true
}
}
return false
}
func isMissingRequiredLibraries(task *jobs.Task) bool { func isMissingRequiredLibraries(task *jobs.Task) bool {
if task.Libraries != nil { if task.Libraries != nil {
return false return false

View File

@ -73,8 +73,11 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
for i := range tasks { for i := range tasks {
task := &tasks[i] task := &tasks[i]
// Keep only Python wheel tasks with local libraries referenced // Keep only Python wheel tasks with workspace libraries referenced.
if task.PythonWheelTask == nil || !libraries.IsTaskWithLocalLibraries(task) { // At this point of moment we don't have local paths in Libraries sections anymore
// Local paths have been replaced with the remote when the artifacts where uploaded
// in artifacts.UploadAll mutator.
if task.PythonWheelTask == nil || !needsTrampoline(task) {
continue continue
} }
@ -87,6 +90,10 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
return result return result
} }
func needsTrampoline(task *jobs.Task) bool {
return libraries.IsTaskWithWorkspaceLibraries(task)
}
func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) { func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) {
params, err := t.generateParameters(task.PythonWheelTask) params, err := t.generateParameters(task.PythonWheelTask)
if err != nil { if err != nil {

View File

@ -84,7 +84,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) {
TaskKey: "key1", TaskKey: "key1",
PythonWheelTask: &jobs.PythonWheelTask{}, PythonWheelTask: &jobs.PythonWheelTask{},
Libraries: []compute.Library{ Libraries: []compute.Library{
{Whl: "./dist/test.whl"}, {Whl: "/Workspace/Users/test@test.com/bundle/dist/test.whl"},
}, },
}, },
{ {