mirror of https://github.com/databricks/cli.git
Apply Python wheel trampoline if workspace library is used (#755)
## Changes Workspace library will be detected by trampoline in 2 cases: - User defined to use local wheel file - User defined to use remote wheel file from Workspace file system In both of these cases we should correctly apply Python trampoline ## Tests Added a regression test (also covered by Python e2e test)
This commit is contained in:
parent
7c96270db8
commit
18a5b05d82
|
@ -105,6 +105,7 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
|
||||||
|
|
||||||
b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{})
|
b.WorkspaceClient().Workspace.WithImpl(MockWorkspaceService{})
|
||||||
artifact := &config.Artifact{
|
artifact := &config.Artifact{
|
||||||
|
Type: "whl",
|
||||||
Files: []config.ArtifactFile{
|
Files: []config.ArtifactFile{
|
||||||
{
|
{
|
||||||
Source: whlPath,
|
Source: whlPath,
|
||||||
|
@ -118,4 +119,5 @@ func TestUploadArtifactFileToCorrectRemotePath(t *testing.T) {
|
||||||
err := uploadArtifact(context.Background(), artifact, b)
|
err := uploadArtifact(context.Background(), artifact, b)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath)
|
require.Regexp(t, regexp.MustCompile("/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].RemotePath)
|
||||||
|
require.Regexp(t, regexp.MustCompile("/Workspace/Users/test@databricks.com/whatever/.internal/[a-z0-9]+/test.whl"), artifact.Files[0].Libraries[0].Whl)
|
||||||
}
|
}
|
||||||
|
|
|
@ -78,9 +78,13 @@ func (a *Artifact) NormalisePaths() {
|
||||||
remotePath := path.Join(wsfsBase, f.RemotePath)
|
remotePath := path.Join(wsfsBase, f.RemotePath)
|
||||||
for i := range f.Libraries {
|
for i := range f.Libraries {
|
||||||
lib := f.Libraries[i]
|
lib := f.Libraries[i]
|
||||||
switch a.Type {
|
if lib.Whl != "" {
|
||||||
case ArtifactPythonWheel:
|
|
||||||
lib.Whl = remotePath
|
lib.Whl = remotePath
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if lib.Jar != "" {
|
||||||
|
lib.Jar = remotePath
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -78,6 +78,17 @@ func IsTaskWithLocalLibraries(task *jobs.Task) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func IsTaskWithWorkspaceLibraries(task *jobs.Task) bool {
|
||||||
|
for _, l := range task.Libraries {
|
||||||
|
path := libPath(&l)
|
||||||
|
if isWorkspacePath(path) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func isMissingRequiredLibraries(task *jobs.Task) bool {
|
func isMissingRequiredLibraries(task *jobs.Task) bool {
|
||||||
if task.Libraries != nil {
|
if task.Libraries != nil {
|
||||||
return false
|
return false
|
||||||
|
|
|
@ -73,8 +73,11 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
|
||||||
for i := range tasks {
|
for i := range tasks {
|
||||||
task := &tasks[i]
|
task := &tasks[i]
|
||||||
|
|
||||||
// Keep only Python wheel tasks with local libraries referenced
|
// Keep only Python wheel tasks with workspace libraries referenced.
|
||||||
if task.PythonWheelTask == nil || !libraries.IsTaskWithLocalLibraries(task) {
|
// At this point of moment we don't have local paths in Libraries sections anymore
|
||||||
|
// Local paths have been replaced with the remote when the artifacts where uploaded
|
||||||
|
// in artifacts.UploadAll mutator.
|
||||||
|
if task.PythonWheelTask == nil || !needsTrampoline(task) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -87,6 +90,10 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func needsTrampoline(task *jobs.Task) bool {
|
||||||
|
return libraries.IsTaskWithWorkspaceLibraries(task)
|
||||||
|
}
|
||||||
|
|
||||||
func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) {
|
func (t *pythonTrampoline) GetTemplateData(task *jobs.Task) (map[string]any, error) {
|
||||||
params, err := t.generateParameters(task.PythonWheelTask)
|
params, err := t.generateParameters(task.PythonWheelTask)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -84,7 +84,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) {
|
||||||
TaskKey: "key1",
|
TaskKey: "key1",
|
||||||
PythonWheelTask: &jobs.PythonWheelTask{},
|
PythonWheelTask: &jobs.PythonWheelTask{},
|
||||||
Libraries: []compute.Library{
|
Libraries: []compute.Library{
|
||||||
{Whl: "./dist/test.whl"},
|
{Whl: "/Workspace/Users/test@test.com/bundle/dist/test.whl"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue